269914d64a887e346f0d680e869c7e99033960d7
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs.cpp
25 *
26 * This file drives the GLSL IR -> LIR translation, contains the
27 * optimizations on the LIR, and drives the generation of native code
28 * from the LIR.
29 */
30
31 #include <sys/types.h>
32
33 #include "util/hash_table.h"
34 #include "main/macros.h"
35 #include "main/shaderobj.h"
36 #include "main/fbobject.h"
37 #include "program/prog_parameter.h"
38 #include "program/prog_print.h"
39 #include "util/register_allocate.h"
40 #include "program/hash_table.h"
41 #include "brw_context.h"
42 #include "brw_eu.h"
43 #include "brw_wm.h"
44 #include "brw_fs.h"
45 #include "brw_cfg.h"
46 #include "brw_dead_control_flow.h"
47 #include "main/uniforms.h"
48 #include "brw_fs_live_variables.h"
49 #include "glsl/glsl_types.h"
50 #include "program/sampler.h"
51
52 using namespace brw;
53
54 void
55 fs_inst::init(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
56 const fs_reg *src, unsigned sources)
57 {
58 memset(this, 0, sizeof(*this));
59
60 this->src = new fs_reg[MAX2(sources, 3)];
61 for (unsigned i = 0; i < sources; i++)
62 this->src[i] = src[i];
63
64 this->opcode = opcode;
65 this->dst = dst;
66 this->sources = sources;
67 this->exec_size = exec_size;
68
69 assert(dst.file != IMM && dst.file != UNIFORM);
70
71 assert(this->exec_size != 0);
72
73 this->conditional_mod = BRW_CONDITIONAL_NONE;
74
75 /* This will be the case for almost all instructions. */
76 switch (dst.file) {
77 case GRF:
78 case HW_REG:
79 case MRF:
80 case ATTR:
81 this->regs_written = DIV_ROUND_UP(dst.component_size(exec_size),
82 REG_SIZE);
83 break;
84 case BAD_FILE:
85 this->regs_written = 0;
86 break;
87 case IMM:
88 case UNIFORM:
89 unreachable("Invalid destination register file");
90 default:
91 unreachable("Invalid register file");
92 }
93
94 this->writes_accumulator = false;
95 }
96
97 fs_inst::fs_inst()
98 {
99 init(BRW_OPCODE_NOP, 8, dst, NULL, 0);
100 }
101
102 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size)
103 {
104 init(opcode, exec_size, reg_undef, NULL, 0);
105 }
106
107 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst)
108 {
109 init(opcode, exec_size, dst, NULL, 0);
110 }
111
112 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
113 const fs_reg &src0)
114 {
115 const fs_reg src[1] = { src0 };
116 init(opcode, exec_size, dst, src, 1);
117 }
118
119 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
120 const fs_reg &src0, const fs_reg &src1)
121 {
122 const fs_reg src[2] = { src0, src1 };
123 init(opcode, exec_size, dst, src, 2);
124 }
125
126 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
127 const fs_reg &src0, const fs_reg &src1, const fs_reg &src2)
128 {
129 const fs_reg src[3] = { src0, src1, src2 };
130 init(opcode, exec_size, dst, src, 3);
131 }
132
133 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_width, const fs_reg &dst,
134 const fs_reg src[], unsigned sources)
135 {
136 init(opcode, exec_width, dst, src, sources);
137 }
138
139 fs_inst::fs_inst(const fs_inst &that)
140 {
141 memcpy(this, &that, sizeof(that));
142
143 this->src = new fs_reg[MAX2(that.sources, 3)];
144
145 for (unsigned i = 0; i < that.sources; i++)
146 this->src[i] = that.src[i];
147 }
148
149 fs_inst::~fs_inst()
150 {
151 delete[] this->src;
152 }
153
154 void
155 fs_inst::resize_sources(uint8_t num_sources)
156 {
157 if (this->sources != num_sources) {
158 fs_reg *src = new fs_reg[MAX2(num_sources, 3)];
159
160 for (unsigned i = 0; i < MIN2(this->sources, num_sources); ++i)
161 src[i] = this->src[i];
162
163 delete[] this->src;
164 this->src = src;
165 this->sources = num_sources;
166 }
167 }
168
169 void
170 fs_visitor::VARYING_PULL_CONSTANT_LOAD(const fs_builder &bld,
171 const fs_reg &dst,
172 const fs_reg &surf_index,
173 const fs_reg &varying_offset,
174 uint32_t const_offset)
175 {
176 /* We have our constant surface use a pitch of 4 bytes, so our index can
177 * be any component of a vector, and then we load 4 contiguous
178 * components starting from that.
179 *
180 * We break down the const_offset to a portion added to the variable
181 * offset and a portion done using reg_offset, which means that if you
182 * have GLSL using something like "uniform vec4 a[20]; gl_FragColor =
183 * a[i]", we'll temporarily generate 4 vec4 loads from offset i * 4, and
184 * CSE can later notice that those loads are all the same and eliminate
185 * the redundant ones.
186 */
187 fs_reg vec4_offset = vgrf(glsl_type::int_type);
188 bld.ADD(vec4_offset, varying_offset, fs_reg(const_offset & ~3));
189
190 int scale = 1;
191 if (devinfo->gen == 4 && bld.dispatch_width() == 8) {
192 /* Pre-gen5, we can either use a SIMD8 message that requires (header,
193 * u, v, r) as parameters, or we can just use the SIMD16 message
194 * consisting of (header, u). We choose the second, at the cost of a
195 * longer return length.
196 */
197 scale = 2;
198 }
199
200 enum opcode op;
201 if (devinfo->gen >= 7)
202 op = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7;
203 else
204 op = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD;
205
206 int regs_written = 4 * (bld.dispatch_width() / 8) * scale;
207 fs_reg vec4_result = fs_reg(GRF, alloc.allocate(regs_written), dst.type);
208 fs_inst *inst = bld.emit(op, vec4_result, surf_index, vec4_offset);
209 inst->regs_written = regs_written;
210
211 if (devinfo->gen < 7) {
212 inst->base_mrf = 13;
213 inst->header_size = 1;
214 if (devinfo->gen == 4)
215 inst->mlen = 3;
216 else
217 inst->mlen = 1 + bld.dispatch_width() / 8;
218 }
219
220 bld.MOV(dst, offset(vec4_result, bld, (const_offset & 3) * scale));
221 }
222
223 /**
224 * A helper for MOV generation for fixing up broken hardware SEND dependency
225 * handling.
226 */
227 void
228 fs_visitor::DEP_RESOLVE_MOV(const fs_builder &bld, int grf)
229 {
230 /* The caller always wants uncompressed to emit the minimal extra
231 * dependencies, and to avoid having to deal with aligning its regs to 2.
232 */
233 const fs_builder ubld = bld.annotate("send dependency resolve")
234 .half(0);
235
236 ubld.MOV(ubld.null_reg_f(), fs_reg(GRF, grf, BRW_REGISTER_TYPE_F));
237 }
238
239 bool
240 fs_inst::equals(fs_inst *inst) const
241 {
242 return (opcode == inst->opcode &&
243 dst.equals(inst->dst) &&
244 src[0].equals(inst->src[0]) &&
245 src[1].equals(inst->src[1]) &&
246 src[2].equals(inst->src[2]) &&
247 saturate == inst->saturate &&
248 predicate == inst->predicate &&
249 conditional_mod == inst->conditional_mod &&
250 mlen == inst->mlen &&
251 base_mrf == inst->base_mrf &&
252 target == inst->target &&
253 eot == inst->eot &&
254 header_size == inst->header_size &&
255 shadow_compare == inst->shadow_compare &&
256 exec_size == inst->exec_size &&
257 offset == inst->offset);
258 }
259
260 bool
261 fs_inst::overwrites_reg(const fs_reg &reg) const
262 {
263 return reg.in_range(dst, regs_written);
264 }
265
266 bool
267 fs_inst::is_send_from_grf() const
268 {
269 switch (opcode) {
270 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7:
271 case SHADER_OPCODE_SHADER_TIME_ADD:
272 case FS_OPCODE_INTERPOLATE_AT_CENTROID:
273 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
274 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
275 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
276 case SHADER_OPCODE_UNTYPED_ATOMIC:
277 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
278 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
279 case SHADER_OPCODE_TYPED_ATOMIC:
280 case SHADER_OPCODE_TYPED_SURFACE_READ:
281 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
282 case SHADER_OPCODE_URB_WRITE_SIMD8:
283 return true;
284 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
285 return src[1].file == GRF;
286 case FS_OPCODE_FB_WRITE:
287 return src[0].file == GRF;
288 default:
289 if (is_tex())
290 return src[0].file == GRF;
291
292 return false;
293 }
294 }
295
296 bool
297 fs_inst::is_copy_payload(const brw::simple_allocator &grf_alloc) const
298 {
299 if (this->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
300 return false;
301
302 fs_reg reg = this->src[0];
303 if (reg.file != GRF || reg.reg_offset != 0 || reg.stride == 0)
304 return false;
305
306 if (grf_alloc.sizes[reg.reg] != this->regs_written)
307 return false;
308
309 for (int i = 0; i < this->sources; i++) {
310 reg.type = this->src[i].type;
311 if (!this->src[i].equals(reg))
312 return false;
313
314 if (i < this->header_size) {
315 reg.reg_offset += 1;
316 } else {
317 reg.reg_offset += this->exec_size / 8;
318 }
319 }
320
321 return true;
322 }
323
324 bool
325 fs_inst::can_do_source_mods(const struct brw_device_info *devinfo)
326 {
327 if (devinfo->gen == 6 && is_math())
328 return false;
329
330 if (is_send_from_grf())
331 return false;
332
333 if (!backend_instruction::can_do_source_mods())
334 return false;
335
336 return true;
337 }
338
339 bool
340 fs_inst::has_side_effects() const
341 {
342 return this->eot || backend_instruction::has_side_effects();
343 }
344
345 void
346 fs_reg::init()
347 {
348 memset(this, 0, sizeof(*this));
349 stride = 1;
350 }
351
352 /** Generic unset register constructor. */
353 fs_reg::fs_reg()
354 {
355 init();
356 this->file = BAD_FILE;
357 }
358
359 /** Immediate value constructor. */
360 fs_reg::fs_reg(float f)
361 {
362 init();
363 this->file = IMM;
364 this->type = BRW_REGISTER_TYPE_F;
365 this->stride = 0;
366 this->fixed_hw_reg.dw1.f = f;
367 }
368
369 /** Immediate value constructor. */
370 fs_reg::fs_reg(int32_t i)
371 {
372 init();
373 this->file = IMM;
374 this->type = BRW_REGISTER_TYPE_D;
375 this->stride = 0;
376 this->fixed_hw_reg.dw1.d = i;
377 }
378
379 /** Immediate value constructor. */
380 fs_reg::fs_reg(uint32_t u)
381 {
382 init();
383 this->file = IMM;
384 this->type = BRW_REGISTER_TYPE_UD;
385 this->stride = 0;
386 this->fixed_hw_reg.dw1.ud = u;
387 }
388
389 /** Vector float immediate value constructor. */
390 fs_reg::fs_reg(uint8_t vf[4])
391 {
392 init();
393 this->file = IMM;
394 this->type = BRW_REGISTER_TYPE_VF;
395 memcpy(&this->fixed_hw_reg.dw1.ud, vf, sizeof(unsigned));
396 }
397
398 /** Vector float immediate value constructor. */
399 fs_reg::fs_reg(uint8_t vf0, uint8_t vf1, uint8_t vf2, uint8_t vf3)
400 {
401 init();
402 this->file = IMM;
403 this->type = BRW_REGISTER_TYPE_VF;
404 this->fixed_hw_reg.dw1.ud = (vf0 << 0) |
405 (vf1 << 8) |
406 (vf2 << 16) |
407 (vf3 << 24);
408 }
409
410 /** Fixed brw_reg. */
411 fs_reg::fs_reg(struct brw_reg fixed_hw_reg)
412 {
413 init();
414 this->file = HW_REG;
415 this->fixed_hw_reg = fixed_hw_reg;
416 this->type = fixed_hw_reg.type;
417 }
418
419 bool
420 fs_reg::equals(const fs_reg &r) const
421 {
422 return (file == r.file &&
423 reg == r.reg &&
424 reg_offset == r.reg_offset &&
425 subreg_offset == r.subreg_offset &&
426 type == r.type &&
427 negate == r.negate &&
428 abs == r.abs &&
429 !reladdr && !r.reladdr &&
430 ((file != HW_REG && file != IMM) ||
431 memcmp(&fixed_hw_reg, &r.fixed_hw_reg,
432 sizeof(fixed_hw_reg)) == 0) &&
433 stride == r.stride);
434 }
435
436 fs_reg &
437 fs_reg::set_smear(unsigned subreg)
438 {
439 assert(file != HW_REG && file != IMM);
440 subreg_offset = subreg * type_sz(type);
441 stride = 0;
442 return *this;
443 }
444
445 bool
446 fs_reg::is_contiguous() const
447 {
448 return stride == 1;
449 }
450
451 unsigned
452 fs_reg::component_size(unsigned width) const
453 {
454 const unsigned stride = (file != HW_REG ? this->stride :
455 fixed_hw_reg.hstride == 0 ? 0 :
456 1 << (fixed_hw_reg.hstride - 1));
457 return MAX2(width * stride, 1) * type_sz(type);
458 }
459
460 extern "C" int
461 type_size_scalar(const struct glsl_type *type)
462 {
463 unsigned int size, i;
464
465 switch (type->base_type) {
466 case GLSL_TYPE_UINT:
467 case GLSL_TYPE_INT:
468 case GLSL_TYPE_FLOAT:
469 case GLSL_TYPE_BOOL:
470 return type->components();
471 case GLSL_TYPE_ARRAY:
472 return type_size_scalar(type->fields.array) * type->length;
473 case GLSL_TYPE_STRUCT:
474 size = 0;
475 for (i = 0; i < type->length; i++) {
476 size += type_size_scalar(type->fields.structure[i].type);
477 }
478 return size;
479 case GLSL_TYPE_SAMPLER:
480 /* Samplers take up no register space, since they're baked in at
481 * link time.
482 */
483 return 0;
484 case GLSL_TYPE_ATOMIC_UINT:
485 return 0;
486 case GLSL_TYPE_SUBROUTINE:
487 return 1;
488 case GLSL_TYPE_IMAGE:
489 return BRW_IMAGE_PARAM_SIZE;
490 case GLSL_TYPE_VOID:
491 case GLSL_TYPE_ERROR:
492 case GLSL_TYPE_INTERFACE:
493 case GLSL_TYPE_DOUBLE:
494 unreachable("not reached");
495 }
496
497 return 0;
498 }
499
500 /**
501 * Create a MOV to read the timestamp register.
502 *
503 * The caller is responsible for emitting the MOV. The return value is
504 * the destination of the MOV, with extra parameters set.
505 */
506 fs_reg
507 fs_visitor::get_timestamp(const fs_builder &bld)
508 {
509 assert(devinfo->gen >= 7);
510
511 fs_reg ts = fs_reg(retype(brw_vec4_reg(BRW_ARCHITECTURE_REGISTER_FILE,
512 BRW_ARF_TIMESTAMP,
513 0),
514 BRW_REGISTER_TYPE_UD));
515
516 fs_reg dst = fs_reg(GRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
517
518 /* We want to read the 3 fields we care about even if it's not enabled in
519 * the dispatch.
520 */
521 bld.group(4, 0).exec_all().MOV(dst, ts);
522
523 /* The caller wants the low 32 bits of the timestamp. Since it's running
524 * at the GPU clock rate of ~1.2ghz, it will roll over every ~3 seconds,
525 * which is plenty of time for our purposes. It is identical across the
526 * EUs, but since it's tracking GPU core speed it will increment at a
527 * varying rate as render P-states change.
528 *
529 * The caller could also check if render P-states have changed (or anything
530 * else that might disrupt timing) by setting smear to 2 and checking if
531 * that field is != 0.
532 */
533 dst.set_smear(0);
534
535 return dst;
536 }
537
538 void
539 fs_visitor::emit_shader_time_begin()
540 {
541 shader_start_time = get_timestamp(bld.annotate("shader time start"));
542 }
543
544 void
545 fs_visitor::emit_shader_time_end()
546 {
547 /* Insert our code just before the final SEND with EOT. */
548 exec_node *end = this->instructions.get_tail();
549 assert(end && ((fs_inst *) end)->eot);
550 const fs_builder ibld = bld.annotate("shader time end")
551 .exec_all().at(NULL, end);
552
553 fs_reg shader_end_time = get_timestamp(ibld);
554
555 /* Check that there weren't any timestamp reset events (assuming these
556 * were the only two timestamp reads that happened).
557 */
558 fs_reg reset = shader_end_time;
559 reset.set_smear(2);
560 set_condmod(BRW_CONDITIONAL_Z,
561 ibld.AND(ibld.null_reg_ud(), reset, fs_reg(1u)));
562 ibld.IF(BRW_PREDICATE_NORMAL);
563
564 fs_reg start = shader_start_time;
565 start.negate = true;
566 fs_reg diff = fs_reg(GRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
567 diff.set_smear(0);
568
569 const fs_builder cbld = ibld.group(1, 0);
570 cbld.group(1, 0).ADD(diff, start, shader_end_time);
571
572 /* If there were no instructions between the two timestamp gets, the diff
573 * is 2 cycles. Remove that overhead, so I can forget about that when
574 * trying to determine the time taken for single instructions.
575 */
576 cbld.ADD(diff, diff, fs_reg(-2u));
577 SHADER_TIME_ADD(cbld, 0, diff);
578 SHADER_TIME_ADD(cbld, 1, fs_reg(1u));
579 ibld.emit(BRW_OPCODE_ELSE);
580 SHADER_TIME_ADD(cbld, 2, fs_reg(1u));
581 ibld.emit(BRW_OPCODE_ENDIF);
582 }
583
584 void
585 fs_visitor::SHADER_TIME_ADD(const fs_builder &bld,
586 int shader_time_subindex,
587 fs_reg value)
588 {
589 int index = shader_time_index * 3 + shader_time_subindex;
590 fs_reg offset = fs_reg(index * SHADER_TIME_STRIDE);
591
592 fs_reg payload;
593 if (dispatch_width == 8)
594 payload = vgrf(glsl_type::uvec2_type);
595 else
596 payload = vgrf(glsl_type::uint_type);
597
598 bld.emit(SHADER_OPCODE_SHADER_TIME_ADD, fs_reg(), payload, offset, value);
599 }
600
601 void
602 fs_visitor::vfail(const char *format, va_list va)
603 {
604 char *msg;
605
606 if (failed)
607 return;
608
609 failed = true;
610
611 msg = ralloc_vasprintf(mem_ctx, format, va);
612 msg = ralloc_asprintf(mem_ctx, "%s compile failed: %s\n", stage_abbrev, msg);
613
614 this->fail_msg = msg;
615
616 if (debug_enabled) {
617 fprintf(stderr, "%s", msg);
618 }
619 }
620
621 void
622 fs_visitor::fail(const char *format, ...)
623 {
624 va_list va;
625
626 va_start(va, format);
627 vfail(format, va);
628 va_end(va);
629 }
630
631 /**
632 * Mark this program as impossible to compile in SIMD16 mode.
633 *
634 * During the SIMD8 compile (which happens first), we can detect and flag
635 * things that are unsupported in SIMD16 mode, so the compiler can skip
636 * the SIMD16 compile altogether.
637 *
638 * During a SIMD16 compile (if one happens anyway), this just calls fail().
639 */
640 void
641 fs_visitor::no16(const char *msg)
642 {
643 if (dispatch_width == 16) {
644 fail("%s", msg);
645 } else {
646 simd16_unsupported = true;
647
648 compiler->shader_perf_log(log_data,
649 "SIMD16 shader failed to compile: %s", msg);
650 }
651 }
652
653 /**
654 * Returns true if the instruction has a flag that means it won't
655 * update an entire destination register.
656 *
657 * For example, dead code elimination and live variable analysis want to know
658 * when a write to a variable screens off any preceding values that were in
659 * it.
660 */
661 bool
662 fs_inst::is_partial_write() const
663 {
664 return ((this->predicate && this->opcode != BRW_OPCODE_SEL) ||
665 (this->exec_size * type_sz(this->dst.type)) < 32 ||
666 !this->dst.is_contiguous());
667 }
668
669 unsigned
670 fs_inst::components_read(unsigned i) const
671 {
672 switch (opcode) {
673 case FS_OPCODE_LINTERP:
674 if (i == 0)
675 return 2;
676 else
677 return 1;
678
679 case FS_OPCODE_PIXEL_X:
680 case FS_OPCODE_PIXEL_Y:
681 assert(i == 0);
682 return 2;
683
684 case FS_OPCODE_FB_WRITE_LOGICAL:
685 assert(src[6].file == IMM);
686 /* First/second FB write color. */
687 if (i < 2)
688 return src[6].fixed_hw_reg.dw1.ud;
689 else
690 return 1;
691
692 case SHADER_OPCODE_TEX_LOGICAL:
693 case SHADER_OPCODE_TXD_LOGICAL:
694 case SHADER_OPCODE_TXF_LOGICAL:
695 case SHADER_OPCODE_TXL_LOGICAL:
696 case SHADER_OPCODE_TXS_LOGICAL:
697 case FS_OPCODE_TXB_LOGICAL:
698 case SHADER_OPCODE_TXF_CMS_LOGICAL:
699 case SHADER_OPCODE_TXF_UMS_LOGICAL:
700 case SHADER_OPCODE_TXF_MCS_LOGICAL:
701 case SHADER_OPCODE_LOD_LOGICAL:
702 case SHADER_OPCODE_TG4_LOGICAL:
703 case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
704 assert(src[8].file == IMM && src[9].file == IMM);
705 /* Texture coordinates. */
706 if (i == 0)
707 return src[8].fixed_hw_reg.dw1.ud;
708 /* Texture derivatives. */
709 else if ((i == 2 || i == 3) && opcode == SHADER_OPCODE_TXD_LOGICAL)
710 return src[9].fixed_hw_reg.dw1.ud;
711 /* Texture offset. */
712 else if (i == 7)
713 return 2;
714 else
715 return 1;
716
717 case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
718 case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
719 assert(src[3].file == IMM);
720 /* Surface coordinates. */
721 if (i == 0)
722 return src[3].fixed_hw_reg.dw1.ud;
723 /* Surface operation source (ignored for reads). */
724 else if (i == 1)
725 return 0;
726 else
727 return 1;
728
729 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
730 case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
731 assert(src[3].file == IMM &&
732 src[4].file == IMM);
733 /* Surface coordinates. */
734 if (i == 0)
735 return src[3].fixed_hw_reg.dw1.ud;
736 /* Surface operation source. */
737 else if (i == 1)
738 return src[4].fixed_hw_reg.dw1.ud;
739 else
740 return 1;
741
742 case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
743 case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: {
744 assert(src[3].file == IMM &&
745 src[4].file == IMM);
746 const unsigned op = src[4].fixed_hw_reg.dw1.ud;
747 /* Surface coordinates. */
748 if (i == 0)
749 return src[3].fixed_hw_reg.dw1.ud;
750 /* Surface operation source. */
751 else if (i == 1 && op == BRW_AOP_CMPWR)
752 return 2;
753 else if (i == 1 && (op == BRW_AOP_INC || op == BRW_AOP_DEC ||
754 op == BRW_AOP_PREDEC))
755 return 0;
756 else
757 return 1;
758 }
759
760 default:
761 return 1;
762 }
763 }
764
765 int
766 fs_inst::regs_read(int arg) const
767 {
768 switch (opcode) {
769 case FS_OPCODE_FB_WRITE:
770 case SHADER_OPCODE_URB_WRITE_SIMD8:
771 case SHADER_OPCODE_UNTYPED_ATOMIC:
772 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
773 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
774 case SHADER_OPCODE_TYPED_ATOMIC:
775 case SHADER_OPCODE_TYPED_SURFACE_READ:
776 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
777 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
778 if (arg == 0)
779 return mlen;
780 break;
781
782 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
783 /* The payload is actually stored in src1 */
784 if (arg == 1)
785 return mlen;
786 break;
787
788 case FS_OPCODE_LINTERP:
789 if (arg == 1)
790 return 1;
791 break;
792
793 case SHADER_OPCODE_LOAD_PAYLOAD:
794 if (arg < this->header_size)
795 return 1;
796 break;
797
798 case CS_OPCODE_CS_TERMINATE:
799 return 1;
800
801 default:
802 if (is_tex() && arg == 0 && src[0].file == GRF)
803 return mlen;
804 break;
805 }
806
807 switch (src[arg].file) {
808 case BAD_FILE:
809 return 0;
810 case UNIFORM:
811 case IMM:
812 return 1;
813 case GRF:
814 case ATTR:
815 case HW_REG:
816 return DIV_ROUND_UP(components_read(arg) *
817 src[arg].component_size(exec_size),
818 REG_SIZE);
819 case MRF:
820 unreachable("MRF registers are not allowed as sources");
821 default:
822 unreachable("Invalid register file");
823 }
824 }
825
826 bool
827 fs_inst::reads_flag() const
828 {
829 return predicate;
830 }
831
832 bool
833 fs_inst::writes_flag() const
834 {
835 return (conditional_mod && (opcode != BRW_OPCODE_SEL &&
836 opcode != BRW_OPCODE_IF &&
837 opcode != BRW_OPCODE_WHILE)) ||
838 opcode == FS_OPCODE_MOV_DISPATCH_TO_FLAGS;
839 }
840
841 /**
842 * Returns how many MRFs an FS opcode will write over.
843 *
844 * Note that this is not the 0 or 1 implied writes in an actual gen
845 * instruction -- the FS opcodes often generate MOVs in addition.
846 */
847 int
848 fs_visitor::implied_mrf_writes(fs_inst *inst)
849 {
850 if (inst->mlen == 0)
851 return 0;
852
853 if (inst->base_mrf == -1)
854 return 0;
855
856 switch (inst->opcode) {
857 case SHADER_OPCODE_RCP:
858 case SHADER_OPCODE_RSQ:
859 case SHADER_OPCODE_SQRT:
860 case SHADER_OPCODE_EXP2:
861 case SHADER_OPCODE_LOG2:
862 case SHADER_OPCODE_SIN:
863 case SHADER_OPCODE_COS:
864 return 1 * dispatch_width / 8;
865 case SHADER_OPCODE_POW:
866 case SHADER_OPCODE_INT_QUOTIENT:
867 case SHADER_OPCODE_INT_REMAINDER:
868 return 2 * dispatch_width / 8;
869 case SHADER_OPCODE_TEX:
870 case FS_OPCODE_TXB:
871 case SHADER_OPCODE_TXD:
872 case SHADER_OPCODE_TXF:
873 case SHADER_OPCODE_TXF_CMS:
874 case SHADER_OPCODE_TXF_MCS:
875 case SHADER_OPCODE_TG4:
876 case SHADER_OPCODE_TG4_OFFSET:
877 case SHADER_OPCODE_TXL:
878 case SHADER_OPCODE_TXS:
879 case SHADER_OPCODE_LOD:
880 return 1;
881 case FS_OPCODE_FB_WRITE:
882 return 2;
883 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
884 case SHADER_OPCODE_GEN4_SCRATCH_READ:
885 return 1;
886 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD:
887 return inst->mlen;
888 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
889 return inst->mlen;
890 case SHADER_OPCODE_UNTYPED_ATOMIC:
891 case SHADER_OPCODE_UNTYPED_SURFACE_READ:
892 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE:
893 case SHADER_OPCODE_TYPED_ATOMIC:
894 case SHADER_OPCODE_TYPED_SURFACE_READ:
895 case SHADER_OPCODE_TYPED_SURFACE_WRITE:
896 case SHADER_OPCODE_URB_WRITE_SIMD8:
897 case FS_OPCODE_INTERPOLATE_AT_CENTROID:
898 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
899 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
900 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
901 return 0;
902 default:
903 unreachable("not reached");
904 }
905 }
906
907 fs_reg
908 fs_visitor::vgrf(const glsl_type *const type)
909 {
910 int reg_width = dispatch_width / 8;
911 return fs_reg(GRF, alloc.allocate(type_size_scalar(type) * reg_width),
912 brw_type_for_base_type(type));
913 }
914
915 /** Fixed HW reg constructor. */
916 fs_reg::fs_reg(enum register_file file, int reg)
917 {
918 init();
919 this->file = file;
920 this->reg = reg;
921 this->type = BRW_REGISTER_TYPE_F;
922 this->stride = (file == UNIFORM ? 0 : 1);
923 }
924
925 /** Fixed HW reg constructor. */
926 fs_reg::fs_reg(enum register_file file, int reg, enum brw_reg_type type)
927 {
928 init();
929 this->file = file;
930 this->reg = reg;
931 this->type = type;
932 this->stride = (file == UNIFORM ? 0 : 1);
933 }
934
935 /* For SIMD16, we need to follow from the uniform setup of SIMD8 dispatch.
936 * This brings in those uniform definitions
937 */
938 void
939 fs_visitor::import_uniforms(fs_visitor *v)
940 {
941 this->push_constant_loc = v->push_constant_loc;
942 this->pull_constant_loc = v->pull_constant_loc;
943 this->uniforms = v->uniforms;
944 this->param_size = v->param_size;
945 }
946
947 void
948 fs_visitor::setup_vec4_uniform_value(unsigned param_offset,
949 const gl_constant_value *values,
950 unsigned n)
951 {
952 static const gl_constant_value zero = { 0 };
953
954 for (unsigned i = 0; i < n; ++i)
955 stage_prog_data->param[param_offset + i] = &values[i];
956
957 for (unsigned i = n; i < 4; ++i)
958 stage_prog_data->param[param_offset + i] = &zero;
959 }
960
961 fs_reg *
962 fs_visitor::emit_fragcoord_interpolation(bool pixel_center_integer,
963 bool origin_upper_left)
964 {
965 assert(stage == MESA_SHADER_FRAGMENT);
966 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
967 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::vec4_type));
968 fs_reg wpos = *reg;
969 bool flip = !origin_upper_left ^ key->render_to_fbo;
970
971 /* gl_FragCoord.x */
972 if (pixel_center_integer) {
973 bld.MOV(wpos, this->pixel_x);
974 } else {
975 bld.ADD(wpos, this->pixel_x, fs_reg(0.5f));
976 }
977 wpos = offset(wpos, bld, 1);
978
979 /* gl_FragCoord.y */
980 if (!flip && pixel_center_integer) {
981 bld.MOV(wpos, this->pixel_y);
982 } else {
983 fs_reg pixel_y = this->pixel_y;
984 float offset = (pixel_center_integer ? 0.0f : 0.5f);
985
986 if (flip) {
987 pixel_y.negate = true;
988 offset += key->drawable_height - 1.0f;
989 }
990
991 bld.ADD(wpos, pixel_y, fs_reg(offset));
992 }
993 wpos = offset(wpos, bld, 1);
994
995 /* gl_FragCoord.z */
996 if (devinfo->gen >= 6) {
997 bld.MOV(wpos, fs_reg(brw_vec8_grf(payload.source_depth_reg, 0)));
998 } else {
999 bld.emit(FS_OPCODE_LINTERP, wpos,
1000 this->delta_xy[BRW_WM_PERSPECTIVE_PIXEL_BARYCENTRIC],
1001 interp_reg(VARYING_SLOT_POS, 2));
1002 }
1003 wpos = offset(wpos, bld, 1);
1004
1005 /* gl_FragCoord.w: Already set up in emit_interpolation */
1006 bld.MOV(wpos, this->wpos_w);
1007
1008 return reg;
1009 }
1010
1011 fs_inst *
1012 fs_visitor::emit_linterp(const fs_reg &attr, const fs_reg &interp,
1013 glsl_interp_qualifier interpolation_mode,
1014 bool is_centroid, bool is_sample)
1015 {
1016 brw_wm_barycentric_interp_mode barycoord_mode;
1017 if (devinfo->gen >= 6) {
1018 if (is_centroid) {
1019 if (interpolation_mode == INTERP_QUALIFIER_SMOOTH)
1020 barycoord_mode = BRW_WM_PERSPECTIVE_CENTROID_BARYCENTRIC;
1021 else
1022 barycoord_mode = BRW_WM_NONPERSPECTIVE_CENTROID_BARYCENTRIC;
1023 } else if (is_sample) {
1024 if (interpolation_mode == INTERP_QUALIFIER_SMOOTH)
1025 barycoord_mode = BRW_WM_PERSPECTIVE_SAMPLE_BARYCENTRIC;
1026 else
1027 barycoord_mode = BRW_WM_NONPERSPECTIVE_SAMPLE_BARYCENTRIC;
1028 } else {
1029 if (interpolation_mode == INTERP_QUALIFIER_SMOOTH)
1030 barycoord_mode = BRW_WM_PERSPECTIVE_PIXEL_BARYCENTRIC;
1031 else
1032 barycoord_mode = BRW_WM_NONPERSPECTIVE_PIXEL_BARYCENTRIC;
1033 }
1034 } else {
1035 /* On Ironlake and below, there is only one interpolation mode.
1036 * Centroid interpolation doesn't mean anything on this hardware --
1037 * there is no multisampling.
1038 */
1039 barycoord_mode = BRW_WM_PERSPECTIVE_PIXEL_BARYCENTRIC;
1040 }
1041 return bld.emit(FS_OPCODE_LINTERP, attr,
1042 this->delta_xy[barycoord_mode], interp);
1043 }
1044
1045 void
1046 fs_visitor::emit_general_interpolation(fs_reg attr, const char *name,
1047 const glsl_type *type,
1048 glsl_interp_qualifier interpolation_mode,
1049 int location, bool mod_centroid,
1050 bool mod_sample)
1051 {
1052 attr.type = brw_type_for_base_type(type->get_scalar_type());
1053
1054 assert(stage == MESA_SHADER_FRAGMENT);
1055 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
1056 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
1057
1058 unsigned int array_elements;
1059
1060 if (type->is_array()) {
1061 array_elements = type->length;
1062 if (array_elements == 0) {
1063 fail("dereferenced array '%s' has length 0\n", name);
1064 }
1065 type = type->fields.array;
1066 } else {
1067 array_elements = 1;
1068 }
1069
1070 if (interpolation_mode == INTERP_QUALIFIER_NONE) {
1071 bool is_gl_Color =
1072 location == VARYING_SLOT_COL0 || location == VARYING_SLOT_COL1;
1073 if (key->flat_shade && is_gl_Color) {
1074 interpolation_mode = INTERP_QUALIFIER_FLAT;
1075 } else {
1076 interpolation_mode = INTERP_QUALIFIER_SMOOTH;
1077 }
1078 }
1079
1080 for (unsigned int i = 0; i < array_elements; i++) {
1081 for (unsigned int j = 0; j < type->matrix_columns; j++) {
1082 if (prog_data->urb_setup[location] == -1) {
1083 /* If there's no incoming setup data for this slot, don't
1084 * emit interpolation for it.
1085 */
1086 attr = offset(attr, bld, type->vector_elements);
1087 location++;
1088 continue;
1089 }
1090
1091 if (interpolation_mode == INTERP_QUALIFIER_FLAT) {
1092 /* Constant interpolation (flat shading) case. The SF has
1093 * handed us defined values in only the constant offset
1094 * field of the setup reg.
1095 */
1096 for (unsigned int k = 0; k < type->vector_elements; k++) {
1097 struct brw_reg interp = interp_reg(location, k);
1098 interp = suboffset(interp, 3);
1099 interp.type = attr.type;
1100 bld.emit(FS_OPCODE_CINTERP, attr, fs_reg(interp));
1101 attr = offset(attr, bld, 1);
1102 }
1103 } else {
1104 /* Smooth/noperspective interpolation case. */
1105 for (unsigned int k = 0; k < type->vector_elements; k++) {
1106 struct brw_reg interp = interp_reg(location, k);
1107 if (devinfo->needs_unlit_centroid_workaround && mod_centroid) {
1108 /* Get the pixel/sample mask into f0 so that we know
1109 * which pixels are lit. Then, for each channel that is
1110 * unlit, replace the centroid data with non-centroid
1111 * data.
1112 */
1113 bld.emit(FS_OPCODE_MOV_DISPATCH_TO_FLAGS);
1114
1115 fs_inst *inst;
1116 inst = emit_linterp(attr, fs_reg(interp), interpolation_mode,
1117 false, false);
1118 inst->predicate = BRW_PREDICATE_NORMAL;
1119 inst->predicate_inverse = true;
1120 if (devinfo->has_pln)
1121 inst->no_dd_clear = true;
1122
1123 inst = emit_linterp(attr, fs_reg(interp), interpolation_mode,
1124 mod_centroid && !key->persample_shading,
1125 mod_sample || key->persample_shading);
1126 inst->predicate = BRW_PREDICATE_NORMAL;
1127 inst->predicate_inverse = false;
1128 if (devinfo->has_pln)
1129 inst->no_dd_check = true;
1130
1131 } else {
1132 emit_linterp(attr, fs_reg(interp), interpolation_mode,
1133 mod_centroid && !key->persample_shading,
1134 mod_sample || key->persample_shading);
1135 }
1136 if (devinfo->gen < 6 && interpolation_mode == INTERP_QUALIFIER_SMOOTH) {
1137 bld.MUL(attr, attr, this->pixel_w);
1138 }
1139 attr = offset(attr, bld, 1);
1140 }
1141
1142 }
1143 location++;
1144 }
1145 }
1146 }
1147
1148 fs_reg *
1149 fs_visitor::emit_frontfacing_interpolation()
1150 {
1151 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::bool_type));
1152
1153 if (devinfo->gen >= 6) {
1154 /* Bit 15 of g0.0 is 0 if the polygon is front facing. We want to create
1155 * a boolean result from this (~0/true or 0/false).
1156 *
1157 * We can use the fact that bit 15 is the MSB of g0.0:W to accomplish
1158 * this task in only one instruction:
1159 * - a negation source modifier will flip the bit; and
1160 * - a W -> D type conversion will sign extend the bit into the high
1161 * word of the destination.
1162 *
1163 * An ASR 15 fills the low word of the destination.
1164 */
1165 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
1166 g0.negate = true;
1167
1168 bld.ASR(*reg, g0, fs_reg(15));
1169 } else {
1170 /* Bit 31 of g1.6 is 0 if the polygon is front facing. We want to create
1171 * a boolean result from this (1/true or 0/false).
1172 *
1173 * Like in the above case, since the bit is the MSB of g1.6:UD we can use
1174 * the negation source modifier to flip it. Unfortunately the SHR
1175 * instruction only operates on UD (or D with an abs source modifier)
1176 * sources without negation.
1177 *
1178 * Instead, use ASR (which will give ~0/true or 0/false).
1179 */
1180 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
1181 g1_6.negate = true;
1182
1183 bld.ASR(*reg, g1_6, fs_reg(31));
1184 }
1185
1186 return reg;
1187 }
1188
1189 void
1190 fs_visitor::compute_sample_position(fs_reg dst, fs_reg int_sample_pos)
1191 {
1192 assert(stage == MESA_SHADER_FRAGMENT);
1193 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
1194 assert(dst.type == BRW_REGISTER_TYPE_F);
1195
1196 if (key->compute_pos_offset) {
1197 /* Convert int_sample_pos to floating point */
1198 bld.MOV(dst, int_sample_pos);
1199 /* Scale to the range [0, 1] */
1200 bld.MUL(dst, dst, fs_reg(1 / 16.0f));
1201 }
1202 else {
1203 /* From ARB_sample_shading specification:
1204 * "When rendering to a non-multisample buffer, or if multisample
1205 * rasterization is disabled, gl_SamplePosition will always be
1206 * (0.5, 0.5).
1207 */
1208 bld.MOV(dst, fs_reg(0.5f));
1209 }
1210 }
1211
1212 fs_reg *
1213 fs_visitor::emit_samplepos_setup()
1214 {
1215 assert(devinfo->gen >= 6);
1216
1217 const fs_builder abld = bld.annotate("compute sample position");
1218 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::vec2_type));
1219 fs_reg pos = *reg;
1220 fs_reg int_sample_x = vgrf(glsl_type::int_type);
1221 fs_reg int_sample_y = vgrf(glsl_type::int_type);
1222
1223 /* WM will be run in MSDISPMODE_PERSAMPLE. So, only one of SIMD8 or SIMD16
1224 * mode will be enabled.
1225 *
1226 * From the Ivy Bridge PRM, volume 2 part 1, page 344:
1227 * R31.1:0 Position Offset X/Y for Slot[3:0]
1228 * R31.3:2 Position Offset X/Y for Slot[7:4]
1229 * .....
1230 *
1231 * The X, Y sample positions come in as bytes in thread payload. So, read
1232 * the positions using vstride=16, width=8, hstride=2.
1233 */
1234 struct brw_reg sample_pos_reg =
1235 stride(retype(brw_vec1_grf(payload.sample_pos_reg, 0),
1236 BRW_REGISTER_TYPE_B), 16, 8, 2);
1237
1238 if (dispatch_width == 8) {
1239 abld.MOV(int_sample_x, fs_reg(sample_pos_reg));
1240 } else {
1241 abld.half(0).MOV(half(int_sample_x, 0), fs_reg(sample_pos_reg));
1242 abld.half(1).MOV(half(int_sample_x, 1),
1243 fs_reg(suboffset(sample_pos_reg, 16)));
1244 }
1245 /* Compute gl_SamplePosition.x */
1246 compute_sample_position(pos, int_sample_x);
1247 pos = offset(pos, abld, 1);
1248 if (dispatch_width == 8) {
1249 abld.MOV(int_sample_y, fs_reg(suboffset(sample_pos_reg, 1)));
1250 } else {
1251 abld.half(0).MOV(half(int_sample_y, 0),
1252 fs_reg(suboffset(sample_pos_reg, 1)));
1253 abld.half(1).MOV(half(int_sample_y, 1),
1254 fs_reg(suboffset(sample_pos_reg, 17)));
1255 }
1256 /* Compute gl_SamplePosition.y */
1257 compute_sample_position(pos, int_sample_y);
1258 return reg;
1259 }
1260
1261 fs_reg *
1262 fs_visitor::emit_sampleid_setup()
1263 {
1264 assert(stage == MESA_SHADER_FRAGMENT);
1265 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
1266 assert(devinfo->gen >= 6);
1267
1268 const fs_builder abld = bld.annotate("compute sample id");
1269 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::int_type));
1270
1271 if (key->compute_sample_id) {
1272 fs_reg t1 = vgrf(glsl_type::int_type);
1273 fs_reg t2 = vgrf(glsl_type::int_type);
1274 t2.type = BRW_REGISTER_TYPE_UW;
1275
1276 /* The PS will be run in MSDISPMODE_PERSAMPLE. For example with
1277 * 8x multisampling, subspan 0 will represent sample N (where N
1278 * is 0, 2, 4 or 6), subspan 1 will represent sample 1, 3, 5 or
1279 * 7. We can find the value of N by looking at R0.0 bits 7:6
1280 * ("Starting Sample Pair Index (SSPI)") and multiplying by two
1281 * (since samples are always delivered in pairs). That is, we
1282 * compute 2*((R0.0 & 0xc0) >> 6) == (R0.0 & 0xc0) >> 5. Then
1283 * we need to add N to the sequence (0, 0, 0, 0, 1, 1, 1, 1) in
1284 * case of SIMD8 and sequence (0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2,
1285 * 2, 3, 3, 3, 3) in case of SIMD16. We compute this sequence by
1286 * populating a temporary variable with the sequence (0, 1, 2, 3),
1287 * and then reading from it using vstride=1, width=4, hstride=0.
1288 * These computations hold good for 4x multisampling as well.
1289 *
1290 * For 2x MSAA and SIMD16, we want to use the sequence (0, 1, 0, 1):
1291 * the first four slots are sample 0 of subspan 0; the next four
1292 * are sample 1 of subspan 0; the third group is sample 0 of
1293 * subspan 1, and finally sample 1 of subspan 1.
1294 */
1295 abld.exec_all()
1296 .AND(t1, fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)),
1297 fs_reg(0xc0));
1298 abld.exec_all().SHR(t1, t1, fs_reg(5));
1299
1300 /* This works for both SIMD8 and SIMD16 */
1301 abld.exec_all()
1302 .MOV(t2, brw_imm_v(key->persample_2x ? 0x1010 : 0x3210));
1303
1304 /* This special instruction takes care of setting vstride=1,
1305 * width=4, hstride=0 of t2 during an ADD instruction.
1306 */
1307 abld.emit(FS_OPCODE_SET_SAMPLE_ID, *reg, t1, t2);
1308 } else {
1309 /* As per GL_ARB_sample_shading specification:
1310 * "When rendering to a non-multisample buffer, or if multisample
1311 * rasterization is disabled, gl_SampleID will always be zero."
1312 */
1313 abld.MOV(*reg, fs_reg(0));
1314 }
1315
1316 return reg;
1317 }
1318
1319 fs_reg
1320 fs_visitor::resolve_source_modifiers(const fs_reg &src)
1321 {
1322 if (!src.abs && !src.negate)
1323 return src;
1324
1325 fs_reg temp = bld.vgrf(src.type);
1326 bld.MOV(temp, src);
1327
1328 return temp;
1329 }
1330
1331 void
1332 fs_visitor::emit_discard_jump()
1333 {
1334 assert(((brw_wm_prog_data*) this->prog_data)->uses_kill);
1335
1336 /* For performance, after a discard, jump to the end of the
1337 * shader if all relevant channels have been discarded.
1338 */
1339 fs_inst *discard_jump = bld.emit(FS_OPCODE_DISCARD_JUMP);
1340 discard_jump->flag_subreg = 1;
1341
1342 discard_jump->predicate = (dispatch_width == 8)
1343 ? BRW_PREDICATE_ALIGN1_ANY8H
1344 : BRW_PREDICATE_ALIGN1_ANY16H;
1345 discard_jump->predicate_inverse = true;
1346 }
1347
1348 void
1349 fs_visitor::assign_curb_setup()
1350 {
1351 if (dispatch_width == 8) {
1352 prog_data->dispatch_grf_start_reg = payload.num_regs;
1353 } else {
1354 if (stage == MESA_SHADER_FRAGMENT) {
1355 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
1356 prog_data->dispatch_grf_start_reg_16 = payload.num_regs;
1357 } else if (stage == MESA_SHADER_COMPUTE) {
1358 brw_cs_prog_data *prog_data = (brw_cs_prog_data*) this->prog_data;
1359 prog_data->dispatch_grf_start_reg_16 = payload.num_regs;
1360 } else {
1361 unreachable("Unsupported shader type!");
1362 }
1363 }
1364
1365 prog_data->curb_read_length = ALIGN(stage_prog_data->nr_params, 8) / 8;
1366
1367 /* Map the offsets in the UNIFORM file to fixed HW regs. */
1368 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1369 for (unsigned int i = 0; i < inst->sources; i++) {
1370 if (inst->src[i].file == UNIFORM) {
1371 int uniform_nr = inst->src[i].reg + inst->src[i].reg_offset;
1372 int constant_nr;
1373 if (uniform_nr >= 0 && uniform_nr < (int) uniforms) {
1374 constant_nr = push_constant_loc[uniform_nr];
1375 } else {
1376 /* Section 5.11 of the OpenGL 4.1 spec says:
1377 * "Out-of-bounds reads return undefined values, which include
1378 * values from other variables of the active program or zero."
1379 * Just return the first push constant.
1380 */
1381 constant_nr = 0;
1382 }
1383
1384 struct brw_reg brw_reg = brw_vec1_grf(payload.num_regs +
1385 constant_nr / 8,
1386 constant_nr % 8);
1387
1388 assert(inst->src[i].stride == 0);
1389 inst->src[i].file = HW_REG;
1390 inst->src[i].fixed_hw_reg = byte_offset(
1391 retype(brw_reg, inst->src[i].type),
1392 inst->src[i].subreg_offset);
1393 }
1394 }
1395 }
1396 }
1397
1398 void
1399 fs_visitor::calculate_urb_setup()
1400 {
1401 assert(stage == MESA_SHADER_FRAGMENT);
1402 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
1403 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
1404
1405 memset(prog_data->urb_setup, -1,
1406 sizeof(prog_data->urb_setup[0]) * VARYING_SLOT_MAX);
1407
1408 int urb_next = 0;
1409 /* Figure out where each of the incoming setup attributes lands. */
1410 if (devinfo->gen >= 6) {
1411 if (_mesa_bitcount_64(prog->InputsRead &
1412 BRW_FS_VARYING_INPUT_MASK) <= 16) {
1413 /* The SF/SBE pipeline stage can do arbitrary rearrangement of the
1414 * first 16 varying inputs, so we can put them wherever we want.
1415 * Just put them in order.
1416 *
1417 * This is useful because it means that (a) inputs not used by the
1418 * fragment shader won't take up valuable register space, and (b) we
1419 * won't have to recompile the fragment shader if it gets paired with
1420 * a different vertex (or geometry) shader.
1421 */
1422 for (unsigned int i = 0; i < VARYING_SLOT_MAX; i++) {
1423 if (prog->InputsRead & BRW_FS_VARYING_INPUT_MASK &
1424 BITFIELD64_BIT(i)) {
1425 prog_data->urb_setup[i] = urb_next++;
1426 }
1427 }
1428 } else {
1429 /* We have enough input varyings that the SF/SBE pipeline stage can't
1430 * arbitrarily rearrange them to suit our whim; we have to put them
1431 * in an order that matches the output of the previous pipeline stage
1432 * (geometry or vertex shader).
1433 */
1434 struct brw_vue_map prev_stage_vue_map;
1435 brw_compute_vue_map(devinfo, &prev_stage_vue_map,
1436 key->input_slots_valid);
1437 int first_slot = 2 * BRW_SF_URB_ENTRY_READ_OFFSET;
1438 assert(prev_stage_vue_map.num_slots <= first_slot + 32);
1439 for (int slot = first_slot; slot < prev_stage_vue_map.num_slots;
1440 slot++) {
1441 int varying = prev_stage_vue_map.slot_to_varying[slot];
1442 /* Note that varying == BRW_VARYING_SLOT_COUNT when a slot is
1443 * unused.
1444 */
1445 if (varying != BRW_VARYING_SLOT_COUNT &&
1446 (prog->InputsRead & BRW_FS_VARYING_INPUT_MASK &
1447 BITFIELD64_BIT(varying))) {
1448 prog_data->urb_setup[varying] = slot - first_slot;
1449 }
1450 }
1451 urb_next = prev_stage_vue_map.num_slots - first_slot;
1452 }
1453 } else {
1454 /* FINISHME: The sf doesn't map VS->FS inputs for us very well. */
1455 for (unsigned int i = 0; i < VARYING_SLOT_MAX; i++) {
1456 /* Point size is packed into the header, not as a general attribute */
1457 if (i == VARYING_SLOT_PSIZ)
1458 continue;
1459
1460 if (key->input_slots_valid & BITFIELD64_BIT(i)) {
1461 /* The back color slot is skipped when the front color is
1462 * also written to. In addition, some slots can be
1463 * written in the vertex shader and not read in the
1464 * fragment shader. So the register number must always be
1465 * incremented, mapped or not.
1466 */
1467 if (_mesa_varying_slot_in_fs((gl_varying_slot) i))
1468 prog_data->urb_setup[i] = urb_next;
1469 urb_next++;
1470 }
1471 }
1472
1473 /*
1474 * It's a FS only attribute, and we did interpolation for this attribute
1475 * in SF thread. So, count it here, too.
1476 *
1477 * See compile_sf_prog() for more info.
1478 */
1479 if (prog->InputsRead & BITFIELD64_BIT(VARYING_SLOT_PNTC))
1480 prog_data->urb_setup[VARYING_SLOT_PNTC] = urb_next++;
1481 }
1482
1483 prog_data->num_varying_inputs = urb_next;
1484 }
1485
1486 void
1487 fs_visitor::assign_urb_setup()
1488 {
1489 assert(stage == MESA_SHADER_FRAGMENT);
1490 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
1491
1492 int urb_start = payload.num_regs + prog_data->base.curb_read_length;
1493
1494 /* Offset all the urb_setup[] index by the actual position of the
1495 * setup regs, now that the location of the constants has been chosen.
1496 */
1497 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1498 if (inst->opcode == FS_OPCODE_LINTERP) {
1499 assert(inst->src[1].file == HW_REG);
1500 inst->src[1].fixed_hw_reg.nr += urb_start;
1501 }
1502
1503 if (inst->opcode == FS_OPCODE_CINTERP) {
1504 assert(inst->src[0].file == HW_REG);
1505 inst->src[0].fixed_hw_reg.nr += urb_start;
1506 }
1507 }
1508
1509 /* Each attribute is 4 setup channels, each of which is half a reg. */
1510 this->first_non_payload_grf =
1511 urb_start + prog_data->num_varying_inputs * 2;
1512 }
1513
1514 void
1515 fs_visitor::assign_vs_urb_setup()
1516 {
1517 brw_vs_prog_data *vs_prog_data = (brw_vs_prog_data *) prog_data;
1518 int grf, count, slot, channel, attr;
1519
1520 assert(stage == MESA_SHADER_VERTEX);
1521 count = _mesa_bitcount_64(vs_prog_data->inputs_read);
1522 if (vs_prog_data->uses_vertexid || vs_prog_data->uses_instanceid)
1523 count++;
1524
1525 /* Each attribute is 4 regs. */
1526 this->first_non_payload_grf =
1527 payload.num_regs + prog_data->curb_read_length + count * 4;
1528
1529 unsigned vue_entries =
1530 MAX2(count, vs_prog_data->base.vue_map.num_slots);
1531
1532 vs_prog_data->base.urb_entry_size = ALIGN(vue_entries, 4) / 4;
1533 vs_prog_data->base.urb_read_length = (count + 1) / 2;
1534
1535 assert(vs_prog_data->base.urb_read_length <= 15);
1536
1537 /* Rewrite all ATTR file references to the hw grf that they land in. */
1538 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1539 for (int i = 0; i < inst->sources; i++) {
1540 if (inst->src[i].file == ATTR) {
1541
1542 if (inst->src[i].reg == VERT_ATTRIB_MAX) {
1543 slot = count - 1;
1544 } else {
1545 /* Attributes come in in a contiguous block, ordered by their
1546 * gl_vert_attrib value. That means we can compute the slot
1547 * number for an attribute by masking out the enabled
1548 * attributes before it and counting the bits.
1549 */
1550 attr = inst->src[i].reg + inst->src[i].reg_offset / 4;
1551 slot = _mesa_bitcount_64(vs_prog_data->inputs_read &
1552 BITFIELD64_MASK(attr));
1553 }
1554
1555 channel = inst->src[i].reg_offset & 3;
1556
1557 grf = payload.num_regs +
1558 prog_data->curb_read_length +
1559 slot * 4 + channel;
1560
1561 inst->src[i].file = HW_REG;
1562 inst->src[i].fixed_hw_reg =
1563 retype(brw_vec8_grf(grf, 0), inst->src[i].type);
1564 }
1565 }
1566 }
1567 }
1568
1569 /**
1570 * Split large virtual GRFs into separate components if we can.
1571 *
1572 * This is mostly duplicated with what brw_fs_vector_splitting does,
1573 * but that's really conservative because it's afraid of doing
1574 * splitting that doesn't result in real progress after the rest of
1575 * the optimization phases, which would cause infinite looping in
1576 * optimization. We can do it once here, safely. This also has the
1577 * opportunity to split interpolated values, or maybe even uniforms,
1578 * which we don't have at the IR level.
1579 *
1580 * We want to split, because virtual GRFs are what we register
1581 * allocate and spill (due to contiguousness requirements for some
1582 * instructions), and they're what we naturally generate in the
1583 * codegen process, but most virtual GRFs don't actually need to be
1584 * contiguous sets of GRFs. If we split, we'll end up with reduced
1585 * live intervals and better dead code elimination and coalescing.
1586 */
1587 void
1588 fs_visitor::split_virtual_grfs()
1589 {
1590 int num_vars = this->alloc.count;
1591
1592 /* Count the total number of registers */
1593 int reg_count = 0;
1594 int vgrf_to_reg[num_vars];
1595 for (int i = 0; i < num_vars; i++) {
1596 vgrf_to_reg[i] = reg_count;
1597 reg_count += alloc.sizes[i];
1598 }
1599
1600 /* An array of "split points". For each register slot, this indicates
1601 * if this slot can be separated from the previous slot. Every time an
1602 * instruction uses multiple elements of a register (as a source or
1603 * destination), we mark the used slots as inseparable. Then we go
1604 * through and split the registers into the smallest pieces we can.
1605 */
1606 bool split_points[reg_count];
1607 memset(split_points, 0, sizeof(split_points));
1608
1609 /* Mark all used registers as fully splittable */
1610 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1611 if (inst->dst.file == GRF) {
1612 int reg = vgrf_to_reg[inst->dst.reg];
1613 for (unsigned j = 1; j < this->alloc.sizes[inst->dst.reg]; j++)
1614 split_points[reg + j] = true;
1615 }
1616
1617 for (int i = 0; i < inst->sources; i++) {
1618 if (inst->src[i].file == GRF) {
1619 int reg = vgrf_to_reg[inst->src[i].reg];
1620 for (unsigned j = 1; j < this->alloc.sizes[inst->src[i].reg]; j++)
1621 split_points[reg + j] = true;
1622 }
1623 }
1624 }
1625
1626 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1627 if (inst->dst.file == GRF) {
1628 int reg = vgrf_to_reg[inst->dst.reg] + inst->dst.reg_offset;
1629 for (int j = 1; j < inst->regs_written; j++)
1630 split_points[reg + j] = false;
1631 }
1632 for (int i = 0; i < inst->sources; i++) {
1633 if (inst->src[i].file == GRF) {
1634 int reg = vgrf_to_reg[inst->src[i].reg] + inst->src[i].reg_offset;
1635 for (int j = 1; j < inst->regs_read(i); j++)
1636 split_points[reg + j] = false;
1637 }
1638 }
1639 }
1640
1641 int new_virtual_grf[reg_count];
1642 int new_reg_offset[reg_count];
1643
1644 int reg = 0;
1645 for (int i = 0; i < num_vars; i++) {
1646 /* The first one should always be 0 as a quick sanity check. */
1647 assert(split_points[reg] == false);
1648
1649 /* j = 0 case */
1650 new_reg_offset[reg] = 0;
1651 reg++;
1652 int offset = 1;
1653
1654 /* j > 0 case */
1655 for (unsigned j = 1; j < alloc.sizes[i]; j++) {
1656 /* If this is a split point, reset the offset to 0 and allocate a
1657 * new virtual GRF for the previous offset many registers
1658 */
1659 if (split_points[reg]) {
1660 assert(offset <= MAX_VGRF_SIZE);
1661 int grf = alloc.allocate(offset);
1662 for (int k = reg - offset; k < reg; k++)
1663 new_virtual_grf[k] = grf;
1664 offset = 0;
1665 }
1666 new_reg_offset[reg] = offset;
1667 offset++;
1668 reg++;
1669 }
1670
1671 /* The last one gets the original register number */
1672 assert(offset <= MAX_VGRF_SIZE);
1673 alloc.sizes[i] = offset;
1674 for (int k = reg - offset; k < reg; k++)
1675 new_virtual_grf[k] = i;
1676 }
1677 assert(reg == reg_count);
1678
1679 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1680 if (inst->dst.file == GRF) {
1681 reg = vgrf_to_reg[inst->dst.reg] + inst->dst.reg_offset;
1682 inst->dst.reg = new_virtual_grf[reg];
1683 inst->dst.reg_offset = new_reg_offset[reg];
1684 assert((unsigned)new_reg_offset[reg] < alloc.sizes[new_virtual_grf[reg]]);
1685 }
1686 for (int i = 0; i < inst->sources; i++) {
1687 if (inst->src[i].file == GRF) {
1688 reg = vgrf_to_reg[inst->src[i].reg] + inst->src[i].reg_offset;
1689 inst->src[i].reg = new_virtual_grf[reg];
1690 inst->src[i].reg_offset = new_reg_offset[reg];
1691 assert((unsigned)new_reg_offset[reg] < alloc.sizes[new_virtual_grf[reg]]);
1692 }
1693 }
1694 }
1695 invalidate_live_intervals();
1696 }
1697
1698 /**
1699 * Remove unused virtual GRFs and compact the virtual_grf_* arrays.
1700 *
1701 * During code generation, we create tons of temporary variables, many of
1702 * which get immediately killed and are never used again. Yet, in later
1703 * optimization and analysis passes, such as compute_live_intervals, we need
1704 * to loop over all the virtual GRFs. Compacting them can save a lot of
1705 * overhead.
1706 */
1707 bool
1708 fs_visitor::compact_virtual_grfs()
1709 {
1710 bool progress = false;
1711 int remap_table[this->alloc.count];
1712 memset(remap_table, -1, sizeof(remap_table));
1713
1714 /* Mark which virtual GRFs are used. */
1715 foreach_block_and_inst(block, const fs_inst, inst, cfg) {
1716 if (inst->dst.file == GRF)
1717 remap_table[inst->dst.reg] = 0;
1718
1719 for (int i = 0; i < inst->sources; i++) {
1720 if (inst->src[i].file == GRF)
1721 remap_table[inst->src[i].reg] = 0;
1722 }
1723 }
1724
1725 /* Compact the GRF arrays. */
1726 int new_index = 0;
1727 for (unsigned i = 0; i < this->alloc.count; i++) {
1728 if (remap_table[i] == -1) {
1729 /* We just found an unused register. This means that we are
1730 * actually going to compact something.
1731 */
1732 progress = true;
1733 } else {
1734 remap_table[i] = new_index;
1735 alloc.sizes[new_index] = alloc.sizes[i];
1736 invalidate_live_intervals();
1737 ++new_index;
1738 }
1739 }
1740
1741 this->alloc.count = new_index;
1742
1743 /* Patch all the instructions to use the newly renumbered registers */
1744 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1745 if (inst->dst.file == GRF)
1746 inst->dst.reg = remap_table[inst->dst.reg];
1747
1748 for (int i = 0; i < inst->sources; i++) {
1749 if (inst->src[i].file == GRF)
1750 inst->src[i].reg = remap_table[inst->src[i].reg];
1751 }
1752 }
1753
1754 /* Patch all the references to delta_xy, since they're used in register
1755 * allocation. If they're unused, switch them to BAD_FILE so we don't
1756 * think some random VGRF is delta_xy.
1757 */
1758 for (unsigned i = 0; i < ARRAY_SIZE(delta_xy); i++) {
1759 if (delta_xy[i].file == GRF) {
1760 if (remap_table[delta_xy[i].reg] != -1) {
1761 delta_xy[i].reg = remap_table[delta_xy[i].reg];
1762 } else {
1763 delta_xy[i].file = BAD_FILE;
1764 }
1765 }
1766 }
1767
1768 return progress;
1769 }
1770
1771 /**
1772 * Assign UNIFORM file registers to either push constants or pull constants.
1773 *
1774 * We allow a fragment shader to have more than the specified minimum
1775 * maximum number of fragment shader uniform components (64). If
1776 * there are too many of these, they'd fill up all of register space.
1777 * So, this will push some of them out to the pull constant buffer and
1778 * update the program to load them. We also use pull constants for all
1779 * indirect constant loads because we don't support indirect accesses in
1780 * registers yet.
1781 */
1782 void
1783 fs_visitor::assign_constant_locations()
1784 {
1785 /* Only the first compile (SIMD8 mode) gets to decide on locations. */
1786 if (dispatch_width != 8)
1787 return;
1788
1789 unsigned int num_pull_constants = 0;
1790
1791 pull_constant_loc = ralloc_array(mem_ctx, int, uniforms);
1792 memset(pull_constant_loc, -1, sizeof(pull_constant_loc[0]) * uniforms);
1793
1794 bool is_live[uniforms];
1795 memset(is_live, 0, sizeof(is_live));
1796
1797 /* First, we walk through the instructions and do two things:
1798 *
1799 * 1) Figure out which uniforms are live.
1800 *
1801 * 2) Find all indirect access of uniform arrays and flag them as needing
1802 * to go into the pull constant buffer.
1803 *
1804 * Note that we don't move constant-indexed accesses to arrays. No
1805 * testing has been done of the performance impact of this choice.
1806 */
1807 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
1808 for (int i = 0 ; i < inst->sources; i++) {
1809 if (inst->src[i].file != UNIFORM)
1810 continue;
1811
1812 if (inst->src[i].reladdr) {
1813 int uniform = inst->src[i].reg;
1814
1815 /* If this array isn't already present in the pull constant buffer,
1816 * add it.
1817 */
1818 if (pull_constant_loc[uniform] == -1) {
1819 assert(param_size[uniform]);
1820 for (int j = 0; j < param_size[uniform]; j++)
1821 pull_constant_loc[uniform + j] = num_pull_constants++;
1822 }
1823 } else {
1824 /* Mark the the one accessed uniform as live */
1825 int constant_nr = inst->src[i].reg + inst->src[i].reg_offset;
1826 if (constant_nr >= 0 && constant_nr < (int) uniforms)
1827 is_live[constant_nr] = true;
1828 }
1829 }
1830 }
1831
1832 /* Only allow 16 registers (128 uniform components) as push constants.
1833 *
1834 * Just demote the end of the list. We could probably do better
1835 * here, demoting things that are rarely used in the program first.
1836 *
1837 * If changing this value, note the limitation about total_regs in
1838 * brw_curbe.c.
1839 */
1840 unsigned int max_push_components = 16 * 8;
1841 unsigned int num_push_constants = 0;
1842
1843 push_constant_loc = ralloc_array(mem_ctx, int, uniforms);
1844
1845 for (unsigned int i = 0; i < uniforms; i++) {
1846 if (!is_live[i] || pull_constant_loc[i] != -1) {
1847 /* This UNIFORM register is either dead, or has already been demoted
1848 * to a pull const. Mark it as no longer living in the param[] array.
1849 */
1850 push_constant_loc[i] = -1;
1851 continue;
1852 }
1853
1854 if (num_push_constants < max_push_components) {
1855 /* Retain as a push constant. Record the location in the params[]
1856 * array.
1857 */
1858 push_constant_loc[i] = num_push_constants++;
1859 } else {
1860 /* Demote to a pull constant. */
1861 push_constant_loc[i] = -1;
1862 pull_constant_loc[i] = num_pull_constants++;
1863 }
1864 }
1865
1866 stage_prog_data->nr_params = num_push_constants;
1867 stage_prog_data->nr_pull_params = num_pull_constants;
1868
1869 /* Up until now, the param[] array has been indexed by reg + reg_offset
1870 * of UNIFORM registers. Move pull constants into pull_param[] and
1871 * condense param[] to only contain the uniforms we chose to push.
1872 *
1873 * NOTE: Because we are condensing the params[] array, we know that
1874 * push_constant_loc[i] <= i and we can do it in one smooth loop without
1875 * having to make a copy.
1876 */
1877 for (unsigned int i = 0; i < uniforms; i++) {
1878 const gl_constant_value *value = stage_prog_data->param[i];
1879
1880 if (pull_constant_loc[i] != -1) {
1881 stage_prog_data->pull_param[pull_constant_loc[i]] = value;
1882 } else if (push_constant_loc[i] != -1) {
1883 stage_prog_data->param[push_constant_loc[i]] = value;
1884 }
1885 }
1886 }
1887
1888 /**
1889 * Replace UNIFORM register file access with either UNIFORM_PULL_CONSTANT_LOAD
1890 * or VARYING_PULL_CONSTANT_LOAD instructions which load values into VGRFs.
1891 */
1892 void
1893 fs_visitor::demote_pull_constants()
1894 {
1895 foreach_block_and_inst (block, fs_inst, inst, cfg) {
1896 for (int i = 0; i < inst->sources; i++) {
1897 if (inst->src[i].file != UNIFORM)
1898 continue;
1899
1900 int pull_index;
1901 unsigned location = inst->src[i].reg + inst->src[i].reg_offset;
1902 if (location >= uniforms) /* Out of bounds access */
1903 pull_index = -1;
1904 else
1905 pull_index = pull_constant_loc[location];
1906
1907 if (pull_index == -1)
1908 continue;
1909
1910 /* Set up the annotation tracking for new generated instructions. */
1911 const fs_builder ibld(this, block, inst);
1912 fs_reg surf_index(stage_prog_data->binding_table.pull_constants_start);
1913 fs_reg dst = vgrf(glsl_type::float_type);
1914
1915 assert(inst->src[i].stride == 0);
1916
1917 /* Generate a pull load into dst. */
1918 if (inst->src[i].reladdr) {
1919 VARYING_PULL_CONSTANT_LOAD(ibld, dst,
1920 surf_index,
1921 *inst->src[i].reladdr,
1922 pull_index);
1923 inst->src[i].reladdr = NULL;
1924 inst->src[i].stride = 1;
1925 } else {
1926 const fs_builder ubld = ibld.exec_all().group(8, 0);
1927 fs_reg offset = fs_reg((unsigned)(pull_index * 4) & ~15);
1928 ubld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
1929 dst, surf_index, offset);
1930 inst->src[i].set_smear(pull_index & 3);
1931 }
1932
1933 /* Rewrite the instruction to use the temporary VGRF. */
1934 inst->src[i].file = GRF;
1935 inst->src[i].reg = dst.reg;
1936 inst->src[i].reg_offset = 0;
1937 }
1938 }
1939 invalidate_live_intervals();
1940 }
1941
1942 bool
1943 fs_visitor::opt_algebraic()
1944 {
1945 bool progress = false;
1946
1947 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1948 switch (inst->opcode) {
1949 case BRW_OPCODE_MOV:
1950 if (inst->src[0].file != IMM)
1951 break;
1952
1953 if (inst->saturate) {
1954 if (inst->dst.type != inst->src[0].type)
1955 assert(!"unimplemented: saturate mixed types");
1956
1957 if (brw_saturate_immediate(inst->dst.type,
1958 &inst->src[0].fixed_hw_reg)) {
1959 inst->saturate = false;
1960 progress = true;
1961 }
1962 }
1963 break;
1964
1965 case BRW_OPCODE_MUL:
1966 if (inst->src[1].file != IMM)
1967 continue;
1968
1969 /* a * 1.0 = a */
1970 if (inst->src[1].is_one()) {
1971 inst->opcode = BRW_OPCODE_MOV;
1972 inst->src[1] = reg_undef;
1973 progress = true;
1974 break;
1975 }
1976
1977 /* a * -1.0 = -a */
1978 if (inst->src[1].is_negative_one()) {
1979 inst->opcode = BRW_OPCODE_MOV;
1980 inst->src[0].negate = !inst->src[0].negate;
1981 inst->src[1] = reg_undef;
1982 progress = true;
1983 break;
1984 }
1985
1986 /* a * 0.0 = 0.0 */
1987 if (inst->src[1].is_zero()) {
1988 inst->opcode = BRW_OPCODE_MOV;
1989 inst->src[0] = inst->src[1];
1990 inst->src[1] = reg_undef;
1991 progress = true;
1992 break;
1993 }
1994
1995 if (inst->src[0].file == IMM) {
1996 assert(inst->src[0].type == BRW_REGISTER_TYPE_F);
1997 inst->opcode = BRW_OPCODE_MOV;
1998 inst->src[0].fixed_hw_reg.dw1.f *= inst->src[1].fixed_hw_reg.dw1.f;
1999 inst->src[1] = reg_undef;
2000 progress = true;
2001 break;
2002 }
2003 break;
2004 case BRW_OPCODE_ADD:
2005 if (inst->src[1].file != IMM)
2006 continue;
2007
2008 /* a + 0.0 = a */
2009 if (inst->src[1].is_zero()) {
2010 inst->opcode = BRW_OPCODE_MOV;
2011 inst->src[1] = reg_undef;
2012 progress = true;
2013 break;
2014 }
2015
2016 if (inst->src[0].file == IMM) {
2017 assert(inst->src[0].type == BRW_REGISTER_TYPE_F);
2018 inst->opcode = BRW_OPCODE_MOV;
2019 inst->src[0].fixed_hw_reg.dw1.f += inst->src[1].fixed_hw_reg.dw1.f;
2020 inst->src[1] = reg_undef;
2021 progress = true;
2022 break;
2023 }
2024 break;
2025 case BRW_OPCODE_OR:
2026 if (inst->src[0].equals(inst->src[1])) {
2027 inst->opcode = BRW_OPCODE_MOV;
2028 inst->src[1] = reg_undef;
2029 progress = true;
2030 break;
2031 }
2032 break;
2033 case BRW_OPCODE_LRP:
2034 if (inst->src[1].equals(inst->src[2])) {
2035 inst->opcode = BRW_OPCODE_MOV;
2036 inst->src[0] = inst->src[1];
2037 inst->src[1] = reg_undef;
2038 inst->src[2] = reg_undef;
2039 progress = true;
2040 break;
2041 }
2042 break;
2043 case BRW_OPCODE_CMP:
2044 if (inst->conditional_mod == BRW_CONDITIONAL_GE &&
2045 inst->src[0].abs &&
2046 inst->src[0].negate &&
2047 inst->src[1].is_zero()) {
2048 inst->src[0].abs = false;
2049 inst->src[0].negate = false;
2050 inst->conditional_mod = BRW_CONDITIONAL_Z;
2051 progress = true;
2052 break;
2053 }
2054 break;
2055 case BRW_OPCODE_SEL:
2056 if (inst->src[0].equals(inst->src[1])) {
2057 inst->opcode = BRW_OPCODE_MOV;
2058 inst->src[1] = reg_undef;
2059 inst->predicate = BRW_PREDICATE_NONE;
2060 inst->predicate_inverse = false;
2061 progress = true;
2062 } else if (inst->saturate && inst->src[1].file == IMM) {
2063 switch (inst->conditional_mod) {
2064 case BRW_CONDITIONAL_LE:
2065 case BRW_CONDITIONAL_L:
2066 switch (inst->src[1].type) {
2067 case BRW_REGISTER_TYPE_F:
2068 if (inst->src[1].fixed_hw_reg.dw1.f >= 1.0f) {
2069 inst->opcode = BRW_OPCODE_MOV;
2070 inst->src[1] = reg_undef;
2071 inst->conditional_mod = BRW_CONDITIONAL_NONE;
2072 progress = true;
2073 }
2074 break;
2075 default:
2076 break;
2077 }
2078 break;
2079 case BRW_CONDITIONAL_GE:
2080 case BRW_CONDITIONAL_G:
2081 switch (inst->src[1].type) {
2082 case BRW_REGISTER_TYPE_F:
2083 if (inst->src[1].fixed_hw_reg.dw1.f <= 0.0f) {
2084 inst->opcode = BRW_OPCODE_MOV;
2085 inst->src[1] = reg_undef;
2086 inst->conditional_mod = BRW_CONDITIONAL_NONE;
2087 progress = true;
2088 }
2089 break;
2090 default:
2091 break;
2092 }
2093 default:
2094 break;
2095 }
2096 }
2097 break;
2098 case BRW_OPCODE_MAD:
2099 if (inst->src[1].is_zero() || inst->src[2].is_zero()) {
2100 inst->opcode = BRW_OPCODE_MOV;
2101 inst->src[1] = reg_undef;
2102 inst->src[2] = reg_undef;
2103 progress = true;
2104 } else if (inst->src[0].is_zero()) {
2105 inst->opcode = BRW_OPCODE_MUL;
2106 inst->src[0] = inst->src[2];
2107 inst->src[2] = reg_undef;
2108 progress = true;
2109 } else if (inst->src[1].is_one()) {
2110 inst->opcode = BRW_OPCODE_ADD;
2111 inst->src[1] = inst->src[2];
2112 inst->src[2] = reg_undef;
2113 progress = true;
2114 } else if (inst->src[2].is_one()) {
2115 inst->opcode = BRW_OPCODE_ADD;
2116 inst->src[2] = reg_undef;
2117 progress = true;
2118 } else if (inst->src[1].file == IMM && inst->src[2].file == IMM) {
2119 inst->opcode = BRW_OPCODE_ADD;
2120 inst->src[1].fixed_hw_reg.dw1.f *= inst->src[2].fixed_hw_reg.dw1.f;
2121 inst->src[2] = reg_undef;
2122 progress = true;
2123 }
2124 break;
2125 case SHADER_OPCODE_RCP: {
2126 fs_inst *prev = (fs_inst *)inst->prev;
2127 if (prev->opcode == SHADER_OPCODE_SQRT) {
2128 if (inst->src[0].equals(prev->dst)) {
2129 inst->opcode = SHADER_OPCODE_RSQ;
2130 inst->src[0] = prev->src[0];
2131 progress = true;
2132 }
2133 }
2134 break;
2135 }
2136 case SHADER_OPCODE_BROADCAST:
2137 if (is_uniform(inst->src[0])) {
2138 inst->opcode = BRW_OPCODE_MOV;
2139 inst->sources = 1;
2140 inst->force_writemask_all = true;
2141 progress = true;
2142 } else if (inst->src[1].file == IMM) {
2143 inst->opcode = BRW_OPCODE_MOV;
2144 inst->src[0] = component(inst->src[0],
2145 inst->src[1].fixed_hw_reg.dw1.ud);
2146 inst->sources = 1;
2147 inst->force_writemask_all = true;
2148 progress = true;
2149 }
2150 break;
2151
2152 default:
2153 break;
2154 }
2155
2156 /* Swap if src[0] is immediate. */
2157 if (progress && inst->is_commutative()) {
2158 if (inst->src[0].file == IMM) {
2159 fs_reg tmp = inst->src[1];
2160 inst->src[1] = inst->src[0];
2161 inst->src[0] = tmp;
2162 }
2163 }
2164 }
2165 return progress;
2166 }
2167
2168 /**
2169 * Optimize sample messages that have constant zero values for the trailing
2170 * texture coordinates. We can just reduce the message length for these
2171 * instructions instead of reserving a register for it. Trailing parameters
2172 * that aren't sent default to zero anyway. This will cause the dead code
2173 * eliminator to remove the MOV instruction that would otherwise be emitted to
2174 * set up the zero value.
2175 */
2176 bool
2177 fs_visitor::opt_zero_samples()
2178 {
2179 /* Gen4 infers the texturing opcode based on the message length so we can't
2180 * change it.
2181 */
2182 if (devinfo->gen < 5)
2183 return false;
2184
2185 bool progress = false;
2186
2187 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2188 if (!inst->is_tex())
2189 continue;
2190
2191 fs_inst *load_payload = (fs_inst *) inst->prev;
2192
2193 if (load_payload->is_head_sentinel() ||
2194 load_payload->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
2195 continue;
2196
2197 /* We don't want to remove the message header or the first parameter.
2198 * Removing the first parameter is not allowed, see the Haswell PRM
2199 * volume 7, page 149:
2200 *
2201 * "Parameter 0 is required except for the sampleinfo message, which
2202 * has no parameter 0"
2203 */
2204 while (inst->mlen > inst->header_size + inst->exec_size / 8 &&
2205 load_payload->src[(inst->mlen - inst->header_size) /
2206 (inst->exec_size / 8) +
2207 inst->header_size - 1].is_zero()) {
2208 inst->mlen -= inst->exec_size / 8;
2209 progress = true;
2210 }
2211 }
2212
2213 if (progress)
2214 invalidate_live_intervals();
2215
2216 return progress;
2217 }
2218
2219 /**
2220 * Optimize sample messages which are followed by the final RT write.
2221 *
2222 * CHV, and GEN9+ can mark a texturing SEND instruction with EOT to have its
2223 * results sent directly to the framebuffer, bypassing the EU. Recognize the
2224 * final texturing results copied to the framebuffer write payload and modify
2225 * them to write to the framebuffer directly.
2226 */
2227 bool
2228 fs_visitor::opt_sampler_eot()
2229 {
2230 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
2231
2232 if (stage != MESA_SHADER_FRAGMENT)
2233 return false;
2234
2235 if (devinfo->gen < 9 && !devinfo->is_cherryview)
2236 return false;
2237
2238 /* FINISHME: It should be possible to implement this optimization when there
2239 * are multiple drawbuffers.
2240 */
2241 if (key->nr_color_regions != 1)
2242 return false;
2243
2244 /* Look for a texturing instruction immediately before the final FB_WRITE. */
2245 bblock_t *block = cfg->blocks[cfg->num_blocks - 1];
2246 fs_inst *fb_write = (fs_inst *)block->end();
2247 assert(fb_write->eot);
2248 assert(fb_write->opcode == FS_OPCODE_FB_WRITE);
2249
2250 fs_inst *tex_inst = (fs_inst *) fb_write->prev;
2251
2252 /* There wasn't one; nothing to do. */
2253 if (unlikely(tex_inst->is_head_sentinel()) || !tex_inst->is_tex())
2254 return false;
2255
2256 /* This optimisation doesn't seem to work for textureGather for some
2257 * reason. I can't find any documentation or known workarounds to indicate
2258 * that this is expected, but considering that it is probably pretty
2259 * unlikely that a shader would directly write out the results from
2260 * textureGather we might as well just disable it.
2261 */
2262 if (tex_inst->opcode == SHADER_OPCODE_TG4 ||
2263 tex_inst->opcode == SHADER_OPCODE_TG4_OFFSET)
2264 return false;
2265
2266 /* If there's no header present, we need to munge the LOAD_PAYLOAD as well.
2267 * It's very likely to be the previous instruction.
2268 */
2269 fs_inst *load_payload = (fs_inst *) tex_inst->prev;
2270 if (load_payload->is_head_sentinel() ||
2271 load_payload->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
2272 return false;
2273
2274 assert(!tex_inst->eot); /* We can't get here twice */
2275 assert((tex_inst->offset & (0xff << 24)) == 0);
2276
2277 const fs_builder ibld(this, block, tex_inst);
2278
2279 tex_inst->offset |= fb_write->target << 24;
2280 tex_inst->eot = true;
2281 tex_inst->dst = ibld.null_reg_ud();
2282 fb_write->remove(cfg->blocks[cfg->num_blocks - 1]);
2283
2284 /* If a header is present, marking the eot is sufficient. Otherwise, we need
2285 * to create a new LOAD_PAYLOAD command with the same sources and a space
2286 * saved for the header. Using a new destination register not only makes sure
2287 * we have enough space, but it will make sure the dead code eliminator kills
2288 * the instruction that this will replace.
2289 */
2290 if (tex_inst->header_size != 0)
2291 return true;
2292
2293 fs_reg send_header = ibld.vgrf(BRW_REGISTER_TYPE_F,
2294 load_payload->sources + 1);
2295 fs_reg *new_sources =
2296 ralloc_array(mem_ctx, fs_reg, load_payload->sources + 1);
2297
2298 new_sources[0] = fs_reg();
2299 for (int i = 0; i < load_payload->sources; i++)
2300 new_sources[i+1] = load_payload->src[i];
2301
2302 /* The LOAD_PAYLOAD helper seems like the obvious choice here. However, it
2303 * requires a lot of information about the sources to appropriately figure
2304 * out the number of registers needed to be used. Given this stage in our
2305 * optimization, we may not have the appropriate GRFs required by
2306 * LOAD_PAYLOAD at this point (copy propagation). Therefore, we need to
2307 * manually emit the instruction.
2308 */
2309 fs_inst *new_load_payload = new(mem_ctx) fs_inst(SHADER_OPCODE_LOAD_PAYLOAD,
2310 load_payload->exec_size,
2311 send_header,
2312 new_sources,
2313 load_payload->sources + 1);
2314
2315 new_load_payload->regs_written = load_payload->regs_written + 1;
2316 new_load_payload->header_size = 1;
2317 tex_inst->mlen++;
2318 tex_inst->header_size = 1;
2319 tex_inst->insert_before(cfg->blocks[cfg->num_blocks - 1], new_load_payload);
2320 tex_inst->src[0] = send_header;
2321
2322 return true;
2323 }
2324
2325 bool
2326 fs_visitor::opt_register_renaming()
2327 {
2328 bool progress = false;
2329 int depth = 0;
2330
2331 int remap[alloc.count];
2332 memset(remap, -1, sizeof(int) * alloc.count);
2333
2334 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2335 if (inst->opcode == BRW_OPCODE_IF || inst->opcode == BRW_OPCODE_DO) {
2336 depth++;
2337 } else if (inst->opcode == BRW_OPCODE_ENDIF ||
2338 inst->opcode == BRW_OPCODE_WHILE) {
2339 depth--;
2340 }
2341
2342 /* Rewrite instruction sources. */
2343 for (int i = 0; i < inst->sources; i++) {
2344 if (inst->src[i].file == GRF &&
2345 remap[inst->src[i].reg] != -1 &&
2346 remap[inst->src[i].reg] != inst->src[i].reg) {
2347 inst->src[i].reg = remap[inst->src[i].reg];
2348 progress = true;
2349 }
2350 }
2351
2352 const int dst = inst->dst.reg;
2353
2354 if (depth == 0 &&
2355 inst->dst.file == GRF &&
2356 alloc.sizes[inst->dst.reg] == inst->exec_size / 8 &&
2357 !inst->is_partial_write()) {
2358 if (remap[dst] == -1) {
2359 remap[dst] = dst;
2360 } else {
2361 remap[dst] = alloc.allocate(inst->exec_size / 8);
2362 inst->dst.reg = remap[dst];
2363 progress = true;
2364 }
2365 } else if (inst->dst.file == GRF &&
2366 remap[dst] != -1 &&
2367 remap[dst] != dst) {
2368 inst->dst.reg = remap[dst];
2369 progress = true;
2370 }
2371 }
2372
2373 if (progress) {
2374 invalidate_live_intervals();
2375
2376 for (unsigned i = 0; i < ARRAY_SIZE(delta_xy); i++) {
2377 if (delta_xy[i].file == GRF && remap[delta_xy[i].reg] != -1) {
2378 delta_xy[i].reg = remap[delta_xy[i].reg];
2379 }
2380 }
2381 }
2382
2383 return progress;
2384 }
2385
2386 /**
2387 * Remove redundant or useless discard jumps.
2388 *
2389 * For example, we can eliminate jumps in the following sequence:
2390 *
2391 * discard-jump (redundant with the next jump)
2392 * discard-jump (useless; jumps to the next instruction)
2393 * placeholder-halt
2394 */
2395 bool
2396 fs_visitor::opt_redundant_discard_jumps()
2397 {
2398 bool progress = false;
2399
2400 bblock_t *last_bblock = cfg->blocks[cfg->num_blocks - 1];
2401
2402 fs_inst *placeholder_halt = NULL;
2403 foreach_inst_in_block_reverse(fs_inst, inst, last_bblock) {
2404 if (inst->opcode == FS_OPCODE_PLACEHOLDER_HALT) {
2405 placeholder_halt = inst;
2406 break;
2407 }
2408 }
2409
2410 if (!placeholder_halt)
2411 return false;
2412
2413 /* Delete any HALTs immediately before the placeholder halt. */
2414 for (fs_inst *prev = (fs_inst *) placeholder_halt->prev;
2415 !prev->is_head_sentinel() && prev->opcode == FS_OPCODE_DISCARD_JUMP;
2416 prev = (fs_inst *) placeholder_halt->prev) {
2417 prev->remove(last_bblock);
2418 progress = true;
2419 }
2420
2421 if (progress)
2422 invalidate_live_intervals();
2423
2424 return progress;
2425 }
2426
2427 bool
2428 fs_visitor::compute_to_mrf()
2429 {
2430 bool progress = false;
2431 int next_ip = 0;
2432
2433 /* No MRFs on Gen >= 7. */
2434 if (devinfo->gen >= 7)
2435 return false;
2436
2437 calculate_live_intervals();
2438
2439 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
2440 int ip = next_ip;
2441 next_ip++;
2442
2443 if (inst->opcode != BRW_OPCODE_MOV ||
2444 inst->is_partial_write() ||
2445 inst->dst.file != MRF || inst->src[0].file != GRF ||
2446 inst->dst.type != inst->src[0].type ||
2447 inst->src[0].abs || inst->src[0].negate ||
2448 !inst->src[0].is_contiguous() ||
2449 inst->src[0].subreg_offset)
2450 continue;
2451
2452 /* Work out which hardware MRF registers are written by this
2453 * instruction.
2454 */
2455 int mrf_low = inst->dst.reg & ~BRW_MRF_COMPR4;
2456 int mrf_high;
2457 if (inst->dst.reg & BRW_MRF_COMPR4) {
2458 mrf_high = mrf_low + 4;
2459 } else if (inst->exec_size == 16) {
2460 mrf_high = mrf_low + 1;
2461 } else {
2462 mrf_high = mrf_low;
2463 }
2464
2465 /* Can't compute-to-MRF this GRF if someone else was going to
2466 * read it later.
2467 */
2468 if (this->virtual_grf_end[inst->src[0].reg] > ip)
2469 continue;
2470
2471 /* Found a move of a GRF to a MRF. Let's see if we can go
2472 * rewrite the thing that made this GRF to write into the MRF.
2473 */
2474 foreach_inst_in_block_reverse_starting_from(fs_inst, scan_inst, inst, block) {
2475 if (scan_inst->dst.file == GRF &&
2476 scan_inst->dst.reg == inst->src[0].reg) {
2477 /* Found the last thing to write our reg we want to turn
2478 * into a compute-to-MRF.
2479 */
2480
2481 /* If this one instruction didn't populate all the
2482 * channels, bail. We might be able to rewrite everything
2483 * that writes that reg, but it would require smarter
2484 * tracking to delay the rewriting until complete success.
2485 */
2486 if (scan_inst->is_partial_write())
2487 break;
2488
2489 /* Things returning more than one register would need us to
2490 * understand coalescing out more than one MOV at a time.
2491 */
2492 if (scan_inst->regs_written > scan_inst->exec_size / 8)
2493 break;
2494
2495 /* SEND instructions can't have MRF as a destination. */
2496 if (scan_inst->mlen)
2497 break;
2498
2499 if (devinfo->gen == 6) {
2500 /* gen6 math instructions must have the destination be
2501 * GRF, so no compute-to-MRF for them.
2502 */
2503 if (scan_inst->is_math()) {
2504 break;
2505 }
2506 }
2507
2508 if (scan_inst->dst.reg_offset == inst->src[0].reg_offset) {
2509 /* Found the creator of our MRF's source value. */
2510 scan_inst->dst.file = MRF;
2511 scan_inst->dst.reg = inst->dst.reg;
2512 scan_inst->saturate |= inst->saturate;
2513 inst->remove(block);
2514 progress = true;
2515 }
2516 break;
2517 }
2518
2519 /* We don't handle control flow here. Most computation of
2520 * values that end up in MRFs are shortly before the MRF
2521 * write anyway.
2522 */
2523 if (block->start() == scan_inst)
2524 break;
2525
2526 /* You can't read from an MRF, so if someone else reads our
2527 * MRF's source GRF that we wanted to rewrite, that stops us.
2528 */
2529 bool interfered = false;
2530 for (int i = 0; i < scan_inst->sources; i++) {
2531 if (scan_inst->src[i].file == GRF &&
2532 scan_inst->src[i].reg == inst->src[0].reg &&
2533 scan_inst->src[i].reg_offset == inst->src[0].reg_offset) {
2534 interfered = true;
2535 }
2536 }
2537 if (interfered)
2538 break;
2539
2540 if (scan_inst->dst.file == MRF) {
2541 /* If somebody else writes our MRF here, we can't
2542 * compute-to-MRF before that.
2543 */
2544 int scan_mrf_low = scan_inst->dst.reg & ~BRW_MRF_COMPR4;
2545 int scan_mrf_high;
2546
2547 if (scan_inst->dst.reg & BRW_MRF_COMPR4) {
2548 scan_mrf_high = scan_mrf_low + 4;
2549 } else if (scan_inst->exec_size == 16) {
2550 scan_mrf_high = scan_mrf_low + 1;
2551 } else {
2552 scan_mrf_high = scan_mrf_low;
2553 }
2554
2555 if (mrf_low == scan_mrf_low ||
2556 mrf_low == scan_mrf_high ||
2557 mrf_high == scan_mrf_low ||
2558 mrf_high == scan_mrf_high) {
2559 break;
2560 }
2561 }
2562
2563 if (scan_inst->mlen > 0 && scan_inst->base_mrf != -1) {
2564 /* Found a SEND instruction, which means that there are
2565 * live values in MRFs from base_mrf to base_mrf +
2566 * scan_inst->mlen - 1. Don't go pushing our MRF write up
2567 * above it.
2568 */
2569 if (mrf_low >= scan_inst->base_mrf &&
2570 mrf_low < scan_inst->base_mrf + scan_inst->mlen) {
2571 break;
2572 }
2573 if (mrf_high >= scan_inst->base_mrf &&
2574 mrf_high < scan_inst->base_mrf + scan_inst->mlen) {
2575 break;
2576 }
2577 }
2578 }
2579 }
2580
2581 if (progress)
2582 invalidate_live_intervals();
2583
2584 return progress;
2585 }
2586
2587 /**
2588 * Eliminate FIND_LIVE_CHANNEL instructions occurring outside any control
2589 * flow. We could probably do better here with some form of divergence
2590 * analysis.
2591 */
2592 bool
2593 fs_visitor::eliminate_find_live_channel()
2594 {
2595 bool progress = false;
2596 unsigned depth = 0;
2597
2598 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
2599 switch (inst->opcode) {
2600 case BRW_OPCODE_IF:
2601 case BRW_OPCODE_DO:
2602 depth++;
2603 break;
2604
2605 case BRW_OPCODE_ENDIF:
2606 case BRW_OPCODE_WHILE:
2607 depth--;
2608 break;
2609
2610 case FS_OPCODE_DISCARD_JUMP:
2611 /* This can potentially make control flow non-uniform until the end
2612 * of the program.
2613 */
2614 return progress;
2615
2616 case SHADER_OPCODE_FIND_LIVE_CHANNEL:
2617 if (depth == 0) {
2618 inst->opcode = BRW_OPCODE_MOV;
2619 inst->src[0] = fs_reg(0);
2620 inst->sources = 1;
2621 inst->force_writemask_all = true;
2622 progress = true;
2623 }
2624 break;
2625
2626 default:
2627 break;
2628 }
2629 }
2630
2631 return progress;
2632 }
2633
2634 /**
2635 * Once we've generated code, try to convert normal FS_OPCODE_FB_WRITE
2636 * instructions to FS_OPCODE_REP_FB_WRITE.
2637 */
2638 void
2639 fs_visitor::emit_repclear_shader()
2640 {
2641 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
2642 int base_mrf = 1;
2643 int color_mrf = base_mrf + 2;
2644
2645 fs_inst *mov = bld.exec_all().MOV(vec4(brw_message_reg(color_mrf)),
2646 fs_reg(UNIFORM, 0, BRW_REGISTER_TYPE_F));
2647
2648 fs_inst *write;
2649 if (key->nr_color_regions == 1) {
2650 write = bld.emit(FS_OPCODE_REP_FB_WRITE);
2651 write->saturate = key->clamp_fragment_color;
2652 write->base_mrf = color_mrf;
2653 write->target = 0;
2654 write->header_size = 0;
2655 write->mlen = 1;
2656 } else {
2657 assume(key->nr_color_regions > 0);
2658 for (int i = 0; i < key->nr_color_regions; ++i) {
2659 write = bld.emit(FS_OPCODE_REP_FB_WRITE);
2660 write->saturate = key->clamp_fragment_color;
2661 write->base_mrf = base_mrf;
2662 write->target = i;
2663 write->header_size = 2;
2664 write->mlen = 3;
2665 }
2666 }
2667 write->eot = true;
2668
2669 calculate_cfg();
2670
2671 assign_constant_locations();
2672 assign_curb_setup();
2673
2674 /* Now that we have the uniform assigned, go ahead and force it to a vec4. */
2675 assert(mov->src[0].file == HW_REG);
2676 mov->src[0] = brw_vec4_grf(mov->src[0].fixed_hw_reg.nr, 0);
2677 }
2678
2679 /**
2680 * Walks through basic blocks, looking for repeated MRF writes and
2681 * removing the later ones.
2682 */
2683 bool
2684 fs_visitor::remove_duplicate_mrf_writes()
2685 {
2686 fs_inst *last_mrf_move[16];
2687 bool progress = false;
2688
2689 /* Need to update the MRF tracking for compressed instructions. */
2690 if (dispatch_width == 16)
2691 return false;
2692
2693 memset(last_mrf_move, 0, sizeof(last_mrf_move));
2694
2695 foreach_block_and_inst_safe (block, fs_inst, inst, cfg) {
2696 if (inst->is_control_flow()) {
2697 memset(last_mrf_move, 0, sizeof(last_mrf_move));
2698 }
2699
2700 if (inst->opcode == BRW_OPCODE_MOV &&
2701 inst->dst.file == MRF) {
2702 fs_inst *prev_inst = last_mrf_move[inst->dst.reg];
2703 if (prev_inst && inst->equals(prev_inst)) {
2704 inst->remove(block);
2705 progress = true;
2706 continue;
2707 }
2708 }
2709
2710 /* Clear out the last-write records for MRFs that were overwritten. */
2711 if (inst->dst.file == MRF) {
2712 last_mrf_move[inst->dst.reg] = NULL;
2713 }
2714
2715 if (inst->mlen > 0 && inst->base_mrf != -1) {
2716 /* Found a SEND instruction, which will include two or fewer
2717 * implied MRF writes. We could do better here.
2718 */
2719 for (int i = 0; i < implied_mrf_writes(inst); i++) {
2720 last_mrf_move[inst->base_mrf + i] = NULL;
2721 }
2722 }
2723
2724 /* Clear out any MRF move records whose sources got overwritten. */
2725 if (inst->dst.file == GRF) {
2726 for (unsigned int i = 0; i < ARRAY_SIZE(last_mrf_move); i++) {
2727 if (last_mrf_move[i] &&
2728 last_mrf_move[i]->src[0].reg == inst->dst.reg) {
2729 last_mrf_move[i] = NULL;
2730 }
2731 }
2732 }
2733
2734 if (inst->opcode == BRW_OPCODE_MOV &&
2735 inst->dst.file == MRF &&
2736 inst->src[0].file == GRF &&
2737 !inst->is_partial_write()) {
2738 last_mrf_move[inst->dst.reg] = inst;
2739 }
2740 }
2741
2742 if (progress)
2743 invalidate_live_intervals();
2744
2745 return progress;
2746 }
2747
2748 static void
2749 clear_deps_for_inst_src(fs_inst *inst, bool *deps, int first_grf, int grf_len)
2750 {
2751 /* Clear the flag for registers that actually got read (as expected). */
2752 for (int i = 0; i < inst->sources; i++) {
2753 int grf;
2754 if (inst->src[i].file == GRF) {
2755 grf = inst->src[i].reg;
2756 } else if (inst->src[i].file == HW_REG &&
2757 inst->src[i].fixed_hw_reg.file == BRW_GENERAL_REGISTER_FILE) {
2758 grf = inst->src[i].fixed_hw_reg.nr;
2759 } else {
2760 continue;
2761 }
2762
2763 if (grf >= first_grf &&
2764 grf < first_grf + grf_len) {
2765 deps[grf - first_grf] = false;
2766 if (inst->exec_size == 16)
2767 deps[grf - first_grf + 1] = false;
2768 }
2769 }
2770 }
2771
2772 /**
2773 * Implements this workaround for the original 965:
2774 *
2775 * "[DevBW, DevCL] Implementation Restrictions: As the hardware does not
2776 * check for post destination dependencies on this instruction, software
2777 * must ensure that there is no destination hazard for the case of ‘write
2778 * followed by a posted write’ shown in the following example.
2779 *
2780 * 1. mov r3 0
2781 * 2. send r3.xy <rest of send instruction>
2782 * 3. mov r2 r3
2783 *
2784 * Due to no post-destination dependency check on the ‘send’, the above
2785 * code sequence could have two instructions (1 and 2) in flight at the
2786 * same time that both consider ‘r3’ as the target of their final writes.
2787 */
2788 void
2789 fs_visitor::insert_gen4_pre_send_dependency_workarounds(bblock_t *block,
2790 fs_inst *inst)
2791 {
2792 int write_len = inst->regs_written;
2793 int first_write_grf = inst->dst.reg;
2794 bool needs_dep[BRW_MAX_MRF];
2795 assert(write_len < (int)sizeof(needs_dep) - 1);
2796
2797 memset(needs_dep, false, sizeof(needs_dep));
2798 memset(needs_dep, true, write_len);
2799
2800 clear_deps_for_inst_src(inst, needs_dep, first_write_grf, write_len);
2801
2802 /* Walk backwards looking for writes to registers we're writing which
2803 * aren't read since being written. If we hit the start of the program,
2804 * we assume that there are no outstanding dependencies on entry to the
2805 * program.
2806 */
2807 foreach_inst_in_block_reverse_starting_from(fs_inst, scan_inst, inst, block) {
2808 /* If we hit control flow, assume that there *are* outstanding
2809 * dependencies, and force their cleanup before our instruction.
2810 */
2811 if (block->start() == scan_inst) {
2812 for (int i = 0; i < write_len; i++) {
2813 if (needs_dep[i])
2814 DEP_RESOLVE_MOV(fs_builder(this, block, inst),
2815 first_write_grf + i);
2816 }
2817 return;
2818 }
2819
2820 /* We insert our reads as late as possible on the assumption that any
2821 * instruction but a MOV that might have left us an outstanding
2822 * dependency has more latency than a MOV.
2823 */
2824 if (scan_inst->dst.file == GRF) {
2825 for (int i = 0; i < scan_inst->regs_written; i++) {
2826 int reg = scan_inst->dst.reg + i;
2827
2828 if (reg >= first_write_grf &&
2829 reg < first_write_grf + write_len &&
2830 needs_dep[reg - first_write_grf]) {
2831 DEP_RESOLVE_MOV(fs_builder(this, block, inst), reg);
2832 needs_dep[reg - first_write_grf] = false;
2833 if (scan_inst->exec_size == 16)
2834 needs_dep[reg - first_write_grf + 1] = false;
2835 }
2836 }
2837 }
2838
2839 /* Clear the flag for registers that actually got read (as expected). */
2840 clear_deps_for_inst_src(scan_inst, needs_dep, first_write_grf, write_len);
2841
2842 /* Continue the loop only if we haven't resolved all the dependencies */
2843 int i;
2844 for (i = 0; i < write_len; i++) {
2845 if (needs_dep[i])
2846 break;
2847 }
2848 if (i == write_len)
2849 return;
2850 }
2851 }
2852
2853 /**
2854 * Implements this workaround for the original 965:
2855 *
2856 * "[DevBW, DevCL] Errata: A destination register from a send can not be
2857 * used as a destination register until after it has been sourced by an
2858 * instruction with a different destination register.
2859 */
2860 void
2861 fs_visitor::insert_gen4_post_send_dependency_workarounds(bblock_t *block, fs_inst *inst)
2862 {
2863 int write_len = inst->regs_written;
2864 int first_write_grf = inst->dst.reg;
2865 bool needs_dep[BRW_MAX_MRF];
2866 assert(write_len < (int)sizeof(needs_dep) - 1);
2867
2868 memset(needs_dep, false, sizeof(needs_dep));
2869 memset(needs_dep, true, write_len);
2870 /* Walk forwards looking for writes to registers we're writing which aren't
2871 * read before being written.
2872 */
2873 foreach_inst_in_block_starting_from(fs_inst, scan_inst, inst, block) {
2874 /* If we hit control flow, force resolve all remaining dependencies. */
2875 if (block->end() == scan_inst) {
2876 for (int i = 0; i < write_len; i++) {
2877 if (needs_dep[i])
2878 DEP_RESOLVE_MOV(fs_builder(this, block, scan_inst),
2879 first_write_grf + i);
2880 }
2881 return;
2882 }
2883
2884 /* Clear the flag for registers that actually got read (as expected). */
2885 clear_deps_for_inst_src(scan_inst, needs_dep, first_write_grf, write_len);
2886
2887 /* We insert our reads as late as possible since they're reading the
2888 * result of a SEND, which has massive latency.
2889 */
2890 if (scan_inst->dst.file == GRF &&
2891 scan_inst->dst.reg >= first_write_grf &&
2892 scan_inst->dst.reg < first_write_grf + write_len &&
2893 needs_dep[scan_inst->dst.reg - first_write_grf]) {
2894 DEP_RESOLVE_MOV(fs_builder(this, block, scan_inst),
2895 scan_inst->dst.reg);
2896 needs_dep[scan_inst->dst.reg - first_write_grf] = false;
2897 }
2898
2899 /* Continue the loop only if we haven't resolved all the dependencies */
2900 int i;
2901 for (i = 0; i < write_len; i++) {
2902 if (needs_dep[i])
2903 break;
2904 }
2905 if (i == write_len)
2906 return;
2907 }
2908 }
2909
2910 void
2911 fs_visitor::insert_gen4_send_dependency_workarounds()
2912 {
2913 if (devinfo->gen != 4 || devinfo->is_g4x)
2914 return;
2915
2916 bool progress = false;
2917
2918 /* Note that we're done with register allocation, so GRF fs_regs always
2919 * have a .reg_offset of 0.
2920 */
2921
2922 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2923 if (inst->mlen != 0 && inst->dst.file == GRF) {
2924 insert_gen4_pre_send_dependency_workarounds(block, inst);
2925 insert_gen4_post_send_dependency_workarounds(block, inst);
2926 progress = true;
2927 }
2928 }
2929
2930 if (progress)
2931 invalidate_live_intervals();
2932 }
2933
2934 /**
2935 * Turns the generic expression-style uniform pull constant load instruction
2936 * into a hardware-specific series of instructions for loading a pull
2937 * constant.
2938 *
2939 * The expression style allows the CSE pass before this to optimize out
2940 * repeated loads from the same offset, and gives the pre-register-allocation
2941 * scheduling full flexibility, while the conversion to native instructions
2942 * allows the post-register-allocation scheduler the best information
2943 * possible.
2944 *
2945 * Note that execution masking for setting up pull constant loads is special:
2946 * the channels that need to be written are unrelated to the current execution
2947 * mask, since a later instruction will use one of the result channels as a
2948 * source operand for all 8 or 16 of its channels.
2949 */
2950 void
2951 fs_visitor::lower_uniform_pull_constant_loads()
2952 {
2953 foreach_block_and_inst (block, fs_inst, inst, cfg) {
2954 if (inst->opcode != FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD)
2955 continue;
2956
2957 if (devinfo->gen >= 7) {
2958 /* The offset arg before was a vec4-aligned byte offset. We need to
2959 * turn it into a dword offset.
2960 */
2961 fs_reg const_offset_reg = inst->src[1];
2962 assert(const_offset_reg.file == IMM &&
2963 const_offset_reg.type == BRW_REGISTER_TYPE_UD);
2964 const_offset_reg.fixed_hw_reg.dw1.ud /= 4;
2965
2966 fs_reg payload, offset;
2967 if (devinfo->gen >= 9) {
2968 /* We have to use a message header on Skylake to get SIMD4x2
2969 * mode. Reserve space for the register.
2970 */
2971 offset = payload = fs_reg(GRF, alloc.allocate(2));
2972 offset.reg_offset++;
2973 inst->mlen = 2;
2974 } else {
2975 offset = payload = fs_reg(GRF, alloc.allocate(1));
2976 inst->mlen = 1;
2977 }
2978
2979 /* This is actually going to be a MOV, but since only the first dword
2980 * is accessed, we have a special opcode to do just that one. Note
2981 * that this needs to be an operation that will be considered a def
2982 * by live variable analysis, or register allocation will explode.
2983 */
2984 fs_inst *setup = new(mem_ctx) fs_inst(FS_OPCODE_SET_SIMD4X2_OFFSET,
2985 8, offset, const_offset_reg);
2986 setup->force_writemask_all = true;
2987
2988 setup->ir = inst->ir;
2989 setup->annotation = inst->annotation;
2990 inst->insert_before(block, setup);
2991
2992 /* Similarly, this will only populate the first 4 channels of the
2993 * result register (since we only use smear values from 0-3), but we
2994 * don't tell the optimizer.
2995 */
2996 inst->opcode = FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7;
2997 inst->src[1] = payload;
2998 inst->base_mrf = -1;
2999
3000 invalidate_live_intervals();
3001 } else {
3002 /* Before register allocation, we didn't tell the scheduler about the
3003 * MRF we use. We know it's safe to use this MRF because nothing
3004 * else does except for register spill/unspill, which generates and
3005 * uses its MRF within a single IR instruction.
3006 */
3007 inst->base_mrf = 14;
3008 inst->mlen = 1;
3009 }
3010 }
3011 }
3012
3013 bool
3014 fs_visitor::lower_load_payload()
3015 {
3016 bool progress = false;
3017
3018 foreach_block_and_inst_safe (block, fs_inst, inst, cfg) {
3019 if (inst->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
3020 continue;
3021
3022 assert(inst->dst.file == MRF || inst->dst.file == GRF);
3023 assert(inst->saturate == false);
3024 fs_reg dst = inst->dst;
3025
3026 /* Get rid of COMPR4. We'll add it back in if we need it */
3027 if (dst.file == MRF)
3028 dst.reg = dst.reg & ~BRW_MRF_COMPR4;
3029
3030 const fs_builder ibld(this, block, inst);
3031 const fs_builder hbld = ibld.exec_all().group(8, 0);
3032
3033 for (uint8_t i = 0; i < inst->header_size; i++) {
3034 if (inst->src[i].file != BAD_FILE) {
3035 fs_reg mov_dst = retype(dst, BRW_REGISTER_TYPE_UD);
3036 fs_reg mov_src = retype(inst->src[i], BRW_REGISTER_TYPE_UD);
3037 hbld.MOV(mov_dst, mov_src);
3038 }
3039 dst = offset(dst, hbld, 1);
3040 }
3041
3042 if (inst->dst.file == MRF && (inst->dst.reg & BRW_MRF_COMPR4) &&
3043 inst->exec_size > 8) {
3044 /* In this case, the payload portion of the LOAD_PAYLOAD isn't
3045 * a straightforward copy. Instead, the result of the
3046 * LOAD_PAYLOAD is treated as interleaved and the first four
3047 * non-header sources are unpacked as:
3048 *
3049 * m + 0: r0
3050 * m + 1: g0
3051 * m + 2: b0
3052 * m + 3: a0
3053 * m + 4: r1
3054 * m + 5: g1
3055 * m + 6: b1
3056 * m + 7: a1
3057 *
3058 * This is used for gen <= 5 fb writes.
3059 */
3060 assert(inst->exec_size == 16);
3061 assert(inst->header_size + 4 <= inst->sources);
3062 for (uint8_t i = inst->header_size; i < inst->header_size + 4; i++) {
3063 if (inst->src[i].file != BAD_FILE) {
3064 if (devinfo->has_compr4) {
3065 fs_reg compr4_dst = retype(dst, inst->src[i].type);
3066 compr4_dst.reg |= BRW_MRF_COMPR4;
3067 ibld.MOV(compr4_dst, inst->src[i]);
3068 } else {
3069 /* Platform doesn't have COMPR4. We have to fake it */
3070 fs_reg mov_dst = retype(dst, inst->src[i].type);
3071 ibld.half(0).MOV(mov_dst, half(inst->src[i], 0));
3072 mov_dst.reg += 4;
3073 ibld.half(1).MOV(mov_dst, half(inst->src[i], 1));
3074 }
3075 }
3076
3077 dst.reg++;
3078 }
3079
3080 /* The loop above only ever incremented us through the first set
3081 * of 4 registers. However, thanks to the magic of COMPR4, we
3082 * actually wrote to the first 8 registers, so we need to take
3083 * that into account now.
3084 */
3085 dst.reg += 4;
3086
3087 /* The COMPR4 code took care of the first 4 sources. We'll let
3088 * the regular path handle any remaining sources. Yes, we are
3089 * modifying the instruction but we're about to delete it so
3090 * this really doesn't hurt anything.
3091 */
3092 inst->header_size += 4;
3093 }
3094
3095 for (uint8_t i = inst->header_size; i < inst->sources; i++) {
3096 if (inst->src[i].file != BAD_FILE)
3097 ibld.MOV(retype(dst, inst->src[i].type), inst->src[i]);
3098 dst = offset(dst, ibld, 1);
3099 }
3100
3101 inst->remove(block);
3102 progress = true;
3103 }
3104
3105 if (progress)
3106 invalidate_live_intervals();
3107
3108 return progress;
3109 }
3110
3111 bool
3112 fs_visitor::lower_integer_multiplication()
3113 {
3114 bool progress = false;
3115
3116 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
3117 const fs_builder ibld(this, block, inst);
3118
3119 if (inst->opcode == BRW_OPCODE_MUL) {
3120 if (inst->dst.is_accumulator() ||
3121 (inst->dst.type != BRW_REGISTER_TYPE_D &&
3122 inst->dst.type != BRW_REGISTER_TYPE_UD))
3123 continue;
3124
3125 /* Gen8's MUL instruction can do a 32-bit x 32-bit -> 32-bit
3126 * operation directly, but CHV/BXT cannot.
3127 */
3128 if (devinfo->gen >= 8 &&
3129 !devinfo->is_cherryview && !devinfo->is_broxton)
3130 continue;
3131
3132 if (inst->src[1].file == IMM &&
3133 inst->src[1].fixed_hw_reg.dw1.ud < (1 << 16)) {
3134 /* The MUL instruction isn't commutative. On Gen <= 6, only the low
3135 * 16-bits of src0 are read, and on Gen >= 7 only the low 16-bits of
3136 * src1 are used.
3137 *
3138 * If multiplying by an immediate value that fits in 16-bits, do a
3139 * single MUL instruction with that value in the proper location.
3140 */
3141 if (devinfo->gen < 7) {
3142 fs_reg imm(GRF, alloc.allocate(dispatch_width / 8),
3143 inst->dst.type);
3144 ibld.MOV(imm, inst->src[1]);
3145 ibld.MUL(inst->dst, imm, inst->src[0]);
3146 } else {
3147 ibld.MUL(inst->dst, inst->src[0], inst->src[1]);
3148 }
3149 } else {
3150 /* Gen < 8 (and some Gen8+ low-power parts like Cherryview) cannot
3151 * do 32-bit integer multiplication in one instruction, but instead
3152 * must do a sequence (which actually calculates a 64-bit result):
3153 *
3154 * mul(8) acc0<1>D g3<8,8,1>D g4<8,8,1>D
3155 * mach(8) null g3<8,8,1>D g4<8,8,1>D
3156 * mov(8) g2<1>D acc0<8,8,1>D
3157 *
3158 * But on Gen > 6, the ability to use second accumulator register
3159 * (acc1) for non-float data types was removed, preventing a simple
3160 * implementation in SIMD16. A 16-channel result can be calculated by
3161 * executing the three instructions twice in SIMD8, once with quarter
3162 * control of 1Q for the first eight channels and again with 2Q for
3163 * the second eight channels.
3164 *
3165 * Which accumulator register is implicitly accessed (by AccWrEnable
3166 * for instance) is determined by the quarter control. Unfortunately
3167 * Ivybridge (and presumably Baytrail) has a hardware bug in which an
3168 * implicit accumulator access by an instruction with 2Q will access
3169 * acc1 regardless of whether the data type is usable in acc1.
3170 *
3171 * Specifically, the 2Q mach(8) writes acc1 which does not exist for
3172 * integer data types.
3173 *
3174 * Since we only want the low 32-bits of the result, we can do two
3175 * 32-bit x 16-bit multiplies (like the mul and mach are doing), and
3176 * adjust the high result and add them (like the mach is doing):
3177 *
3178 * mul(8) g7<1>D g3<8,8,1>D g4.0<8,8,1>UW
3179 * mul(8) g8<1>D g3<8,8,1>D g4.1<8,8,1>UW
3180 * shl(8) g9<1>D g8<8,8,1>D 16D
3181 * add(8) g2<1>D g7<8,8,1>D g8<8,8,1>D
3182 *
3183 * We avoid the shl instruction by realizing that we only want to add
3184 * the low 16-bits of the "high" result to the high 16-bits of the
3185 * "low" result and using proper regioning on the add:
3186 *
3187 * mul(8) g7<1>D g3<8,8,1>D g4.0<16,8,2>UW
3188 * mul(8) g8<1>D g3<8,8,1>D g4.1<16,8,2>UW
3189 * add(8) g7.1<2>UW g7.1<16,8,2>UW g8<16,8,2>UW
3190 *
3191 * Since it does not use the (single) accumulator register, we can
3192 * schedule multi-component multiplications much better.
3193 */
3194
3195 if (inst->conditional_mod && inst->dst.is_null()) {
3196 inst->dst = fs_reg(GRF, alloc.allocate(dispatch_width / 8),
3197 inst->dst.type);
3198 }
3199 fs_reg low = inst->dst;
3200 fs_reg high(GRF, alloc.allocate(dispatch_width / 8),
3201 inst->dst.type);
3202
3203 if (devinfo->gen >= 7) {
3204 fs_reg src1_0_w = inst->src[1];
3205 fs_reg src1_1_w = inst->src[1];
3206
3207 if (inst->src[1].file == IMM) {
3208 src1_0_w.fixed_hw_reg.dw1.ud &= 0xffff;
3209 src1_1_w.fixed_hw_reg.dw1.ud >>= 16;
3210 } else {
3211 src1_0_w.type = BRW_REGISTER_TYPE_UW;
3212 if (src1_0_w.stride != 0) {
3213 assert(src1_0_w.stride == 1);
3214 src1_0_w.stride = 2;
3215 }
3216
3217 src1_1_w.type = BRW_REGISTER_TYPE_UW;
3218 if (src1_1_w.stride != 0) {
3219 assert(src1_1_w.stride == 1);
3220 src1_1_w.stride = 2;
3221 }
3222 src1_1_w.subreg_offset += type_sz(BRW_REGISTER_TYPE_UW);
3223 }
3224 ibld.MUL(low, inst->src[0], src1_0_w);
3225 ibld.MUL(high, inst->src[0], src1_1_w);
3226 } else {
3227 fs_reg src0_0_w = inst->src[0];
3228 fs_reg src0_1_w = inst->src[0];
3229
3230 src0_0_w.type = BRW_REGISTER_TYPE_UW;
3231 if (src0_0_w.stride != 0) {
3232 assert(src0_0_w.stride == 1);
3233 src0_0_w.stride = 2;
3234 }
3235
3236 src0_1_w.type = BRW_REGISTER_TYPE_UW;
3237 if (src0_1_w.stride != 0) {
3238 assert(src0_1_w.stride == 1);
3239 src0_1_w.stride = 2;
3240 }
3241 src0_1_w.subreg_offset += type_sz(BRW_REGISTER_TYPE_UW);
3242
3243 ibld.MUL(low, src0_0_w, inst->src[1]);
3244 ibld.MUL(high, src0_1_w, inst->src[1]);
3245 }
3246
3247 fs_reg dst = inst->dst;
3248 dst.type = BRW_REGISTER_TYPE_UW;
3249 dst.subreg_offset = 2;
3250 dst.stride = 2;
3251
3252 high.type = BRW_REGISTER_TYPE_UW;
3253 high.stride = 2;
3254
3255 low.type = BRW_REGISTER_TYPE_UW;
3256 low.subreg_offset = 2;
3257 low.stride = 2;
3258
3259 ibld.ADD(dst, low, high);
3260
3261 if (inst->conditional_mod) {
3262 fs_reg null(retype(ibld.null_reg_f(), inst->dst.type));
3263 set_condmod(inst->conditional_mod,
3264 ibld.MOV(null, inst->dst));
3265 }
3266 }
3267
3268 } else if (inst->opcode == SHADER_OPCODE_MULH) {
3269 /* Should have been lowered to 8-wide. */
3270 assert(inst->exec_size <= 8);
3271 const fs_reg acc = retype(brw_acc_reg(inst->exec_size),
3272 inst->dst.type);
3273 fs_inst *mul = ibld.MUL(acc, inst->src[0], inst->src[1]);
3274 fs_inst *mach = ibld.MACH(inst->dst, inst->src[0], inst->src[1]);
3275
3276 if (devinfo->gen >= 8) {
3277 /* Until Gen8, integer multiplies read 32-bits from one source,
3278 * and 16-bits from the other, and relying on the MACH instruction
3279 * to generate the high bits of the result.
3280 *
3281 * On Gen8, the multiply instruction does a full 32x32-bit
3282 * multiply, but in order to do a 64-bit multiply we can simulate
3283 * the previous behavior and then use a MACH instruction.
3284 *
3285 * FINISHME: Don't use source modifiers on src1.
3286 */
3287 assert(mul->src[1].type == BRW_REGISTER_TYPE_D ||
3288 mul->src[1].type == BRW_REGISTER_TYPE_UD);
3289 mul->src[1].type = (type_is_signed(mul->src[1].type) ?
3290 BRW_REGISTER_TYPE_W : BRW_REGISTER_TYPE_UW);
3291 mul->src[1].stride *= 2;
3292
3293 } else if (devinfo->gen == 7 && !devinfo->is_haswell &&
3294 inst->force_sechalf) {
3295 /* Among other things the quarter control bits influence which
3296 * accumulator register is used by the hardware for instructions
3297 * that access the accumulator implicitly (e.g. MACH). A
3298 * second-half instruction would normally map to acc1, which
3299 * doesn't exist on Gen7 and up (the hardware does emulate it for
3300 * floating-point instructions *only* by taking advantage of the
3301 * extra precision of acc0 not normally used for floating point
3302 * arithmetic).
3303 *
3304 * HSW and up are careful enough not to try to access an
3305 * accumulator register that doesn't exist, but on earlier Gen7
3306 * hardware we need to make sure that the quarter control bits are
3307 * zero to avoid non-deterministic behaviour and emit an extra MOV
3308 * to get the result masked correctly according to the current
3309 * channel enables.
3310 */
3311 mach->force_sechalf = false;
3312 mach->force_writemask_all = true;
3313 mach->dst = ibld.vgrf(inst->dst.type);
3314 ibld.MOV(inst->dst, mach->dst);
3315 }
3316 } else {
3317 continue;
3318 }
3319
3320 inst->remove(block);
3321 progress = true;
3322 }
3323
3324 if (progress)
3325 invalidate_live_intervals();
3326
3327 return progress;
3328 }
3329
3330 static void
3331 setup_color_payload(const fs_builder &bld, const brw_wm_prog_key *key,
3332 fs_reg *dst, fs_reg color, unsigned components)
3333 {
3334 if (key->clamp_fragment_color) {
3335 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
3336 assert(color.type == BRW_REGISTER_TYPE_F);
3337
3338 for (unsigned i = 0; i < components; i++)
3339 set_saturate(true,
3340 bld.MOV(offset(tmp, bld, i), offset(color, bld, i)));
3341
3342 color = tmp;
3343 }
3344
3345 for (unsigned i = 0; i < components; i++)
3346 dst[i] = offset(color, bld, i);
3347 }
3348
3349 static void
3350 lower_fb_write_logical_send(const fs_builder &bld, fs_inst *inst,
3351 const brw_wm_prog_data *prog_data,
3352 const brw_wm_prog_key *key,
3353 const fs_visitor::thread_payload &payload)
3354 {
3355 assert(inst->src[6].file == IMM);
3356 const brw_device_info *devinfo = bld.shader->devinfo;
3357 const fs_reg &color0 = inst->src[0];
3358 const fs_reg &color1 = inst->src[1];
3359 const fs_reg &src0_alpha = inst->src[2];
3360 const fs_reg &src_depth = inst->src[3];
3361 const fs_reg &dst_depth = inst->src[4];
3362 fs_reg sample_mask = inst->src[5];
3363 const unsigned components = inst->src[6].fixed_hw_reg.dw1.ud;
3364
3365 /* We can potentially have a message length of up to 15, so we have to set
3366 * base_mrf to either 0 or 1 in order to fit in m0..m15.
3367 */
3368 fs_reg sources[15];
3369 int header_size = 2, payload_header_size;
3370 unsigned length = 0;
3371
3372 /* From the Sandy Bridge PRM, volume 4, page 198:
3373 *
3374 * "Dispatched Pixel Enables. One bit per pixel indicating
3375 * which pixels were originally enabled when the thread was
3376 * dispatched. This field is only required for the end-of-
3377 * thread message and on all dual-source messages."
3378 */
3379 if (devinfo->gen >= 6 &&
3380 (devinfo->is_haswell || devinfo->gen >= 8 || !prog_data->uses_kill) &&
3381 color1.file == BAD_FILE &&
3382 key->nr_color_regions == 1) {
3383 header_size = 0;
3384 }
3385
3386 if (header_size != 0) {
3387 assert(header_size == 2);
3388 /* Allocate 2 registers for a header */
3389 length += 2;
3390 }
3391
3392 if (payload.aa_dest_stencil_reg) {
3393 sources[length] = fs_reg(GRF, bld.shader->alloc.allocate(1));
3394 bld.group(8, 0).exec_all().annotate("FB write stencil/AA alpha")
3395 .MOV(sources[length],
3396 fs_reg(brw_vec8_grf(payload.aa_dest_stencil_reg, 0)));
3397 length++;
3398 }
3399
3400 if (prog_data->uses_omask) {
3401 sources[length] = fs_reg(GRF, bld.shader->alloc.allocate(1),
3402 BRW_REGISTER_TYPE_UD);
3403
3404 /* Hand over gl_SampleMask. Only the lower 16 bits of each channel are
3405 * relevant. Since it's unsigned single words one vgrf is always
3406 * 16-wide, but only the lower or higher 8 channels will be used by the
3407 * hardware when doing a SIMD8 write depending on whether we have
3408 * selected the subspans for the first or second half respectively.
3409 */
3410 assert(sample_mask.file != BAD_FILE && type_sz(sample_mask.type) == 4);
3411 sample_mask.type = BRW_REGISTER_TYPE_UW;
3412 sample_mask.stride *= 2;
3413
3414 bld.exec_all().annotate("FB write oMask")
3415 .MOV(half(retype(sources[length], BRW_REGISTER_TYPE_UW),
3416 inst->force_sechalf),
3417 sample_mask);
3418 length++;
3419 }
3420
3421 payload_header_size = length;
3422
3423 if (src0_alpha.file != BAD_FILE) {
3424 /* FIXME: This is being passed at the wrong location in the payload and
3425 * doesn't work when gl_SampleMask and MRTs are used simultaneously.
3426 * It's supposed to be immediately before oMask but there seems to be no
3427 * reasonable way to pass them in the correct order because LOAD_PAYLOAD
3428 * requires header sources to form a contiguous segment at the beginning
3429 * of the message and src0_alpha has per-channel semantics.
3430 */
3431 setup_color_payload(bld, key, &sources[length], src0_alpha, 1);
3432 length++;
3433 }
3434
3435 setup_color_payload(bld, key, &sources[length], color0, components);
3436 length += 4;
3437
3438 if (color1.file != BAD_FILE) {
3439 setup_color_payload(bld, key, &sources[length], color1, components);
3440 length += 4;
3441 }
3442
3443 if (src_depth.file != BAD_FILE) {
3444 sources[length] = src_depth;
3445 length++;
3446 }
3447
3448 if (dst_depth.file != BAD_FILE) {
3449 sources[length] = dst_depth;
3450 length++;
3451 }
3452
3453 fs_inst *load;
3454 if (devinfo->gen >= 7) {
3455 /* Send from the GRF */
3456 fs_reg payload = fs_reg(GRF, -1, BRW_REGISTER_TYPE_F);
3457 load = bld.LOAD_PAYLOAD(payload, sources, length, payload_header_size);
3458 payload.reg = bld.shader->alloc.allocate(load->regs_written);
3459 load->dst = payload;
3460
3461 inst->src[0] = payload;
3462 inst->resize_sources(1);
3463 inst->base_mrf = -1;
3464 } else {
3465 /* Send from the MRF */
3466 load = bld.LOAD_PAYLOAD(fs_reg(MRF, 1, BRW_REGISTER_TYPE_F),
3467 sources, length, payload_header_size);
3468
3469 /* On pre-SNB, we have to interlace the color values. LOAD_PAYLOAD
3470 * will do this for us if we just give it a COMPR4 destination.
3471 */
3472 if (devinfo->gen < 6 && bld.dispatch_width() == 16)
3473 load->dst.reg |= BRW_MRF_COMPR4;
3474
3475 inst->resize_sources(0);
3476 inst->base_mrf = 1;
3477 }
3478
3479 inst->opcode = FS_OPCODE_FB_WRITE;
3480 inst->mlen = load->regs_written;
3481 inst->header_size = header_size;
3482 }
3483
3484 static void
3485 lower_sampler_logical_send_gen4(const fs_builder &bld, fs_inst *inst, opcode op,
3486 const fs_reg &coordinate,
3487 const fs_reg &shadow_c,
3488 const fs_reg &lod, const fs_reg &lod2,
3489 const fs_reg &sampler,
3490 unsigned coord_components,
3491 unsigned grad_components)
3492 {
3493 const bool has_lod = (op == SHADER_OPCODE_TXL || op == FS_OPCODE_TXB ||
3494 op == SHADER_OPCODE_TXF || op == SHADER_OPCODE_TXS);
3495 fs_reg msg_begin(MRF, 1, BRW_REGISTER_TYPE_F);
3496 fs_reg msg_end = msg_begin;
3497
3498 /* g0 header. */
3499 msg_end = offset(msg_end, bld.group(8, 0), 1);
3500
3501 for (unsigned i = 0; i < coord_components; i++)
3502 bld.MOV(retype(offset(msg_end, bld, i), coordinate.type),
3503 offset(coordinate, bld, i));
3504
3505 msg_end = offset(msg_end, bld, coord_components);
3506
3507 /* Messages other than SAMPLE and RESINFO in SIMD16 and TXD in SIMD8
3508 * require all three components to be present and zero if they are unused.
3509 */
3510 if (coord_components > 0 &&
3511 (has_lod || shadow_c.file != BAD_FILE ||
3512 (op == SHADER_OPCODE_TEX && bld.dispatch_width() == 8))) {
3513 for (unsigned i = coord_components; i < 3; i++)
3514 bld.MOV(offset(msg_end, bld, i), fs_reg(0.0f));
3515
3516 msg_end = offset(msg_end, bld, 3 - coord_components);
3517 }
3518
3519 if (op == SHADER_OPCODE_TXD) {
3520 /* TXD unsupported in SIMD16 mode. */
3521 assert(bld.dispatch_width() == 8);
3522
3523 /* the slots for u and v are always present, but r is optional */
3524 if (coord_components < 2)
3525 msg_end = offset(msg_end, bld, 2 - coord_components);
3526
3527 /* P = u, v, r
3528 * dPdx = dudx, dvdx, drdx
3529 * dPdy = dudy, dvdy, drdy
3530 *
3531 * 1-arg: Does not exist.
3532 *
3533 * 2-arg: dudx dvdx dudy dvdy
3534 * dPdx.x dPdx.y dPdy.x dPdy.y
3535 * m4 m5 m6 m7
3536 *
3537 * 3-arg: dudx dvdx drdx dudy dvdy drdy
3538 * dPdx.x dPdx.y dPdx.z dPdy.x dPdy.y dPdy.z
3539 * m5 m6 m7 m8 m9 m10
3540 */
3541 for (unsigned i = 0; i < grad_components; i++)
3542 bld.MOV(offset(msg_end, bld, i), offset(lod, bld, i));
3543
3544 msg_end = offset(msg_end, bld, MAX2(grad_components, 2));
3545
3546 for (unsigned i = 0; i < grad_components; i++)
3547 bld.MOV(offset(msg_end, bld, i), offset(lod2, bld, i));
3548
3549 msg_end = offset(msg_end, bld, MAX2(grad_components, 2));
3550 }
3551
3552 if (has_lod) {
3553 /* Bias/LOD with shadow comparitor is unsupported in SIMD16 -- *Without*
3554 * shadow comparitor (including RESINFO) it's unsupported in SIMD8 mode.
3555 */
3556 assert(shadow_c.file != BAD_FILE ? bld.dispatch_width() == 8 :
3557 bld.dispatch_width() == 16);
3558
3559 const brw_reg_type type =
3560 (op == SHADER_OPCODE_TXF || op == SHADER_OPCODE_TXS ?
3561 BRW_REGISTER_TYPE_UD : BRW_REGISTER_TYPE_F);
3562 bld.MOV(retype(msg_end, type), lod);
3563 msg_end = offset(msg_end, bld, 1);
3564 }
3565
3566 if (shadow_c.file != BAD_FILE) {
3567 if (op == SHADER_OPCODE_TEX && bld.dispatch_width() == 8) {
3568 /* There's no plain shadow compare message, so we use shadow
3569 * compare with a bias of 0.0.
3570 */
3571 bld.MOV(msg_end, fs_reg(0.0f));
3572 msg_end = offset(msg_end, bld, 1);
3573 }
3574
3575 bld.MOV(msg_end, shadow_c);
3576 msg_end = offset(msg_end, bld, 1);
3577 }
3578
3579 inst->opcode = op;
3580 inst->src[0] = reg_undef;
3581 inst->src[1] = sampler;
3582 inst->resize_sources(2);
3583 inst->base_mrf = msg_begin.reg;
3584 inst->mlen = msg_end.reg - msg_begin.reg;
3585 inst->header_size = 1;
3586 }
3587
3588 static void
3589 lower_sampler_logical_send_gen5(const fs_builder &bld, fs_inst *inst, opcode op,
3590 fs_reg coordinate,
3591 const fs_reg &shadow_c,
3592 fs_reg lod, fs_reg lod2,
3593 const fs_reg &sample_index,
3594 const fs_reg &sampler,
3595 const fs_reg &offset_value,
3596 unsigned coord_components,
3597 unsigned grad_components)
3598 {
3599 fs_reg message(MRF, 2, BRW_REGISTER_TYPE_F);
3600 fs_reg msg_coords = message;
3601 unsigned header_size = 0;
3602
3603 if (offset_value.file != BAD_FILE) {
3604 /* The offsets set up by the visitor are in the m1 header, so we can't
3605 * go headerless.
3606 */
3607 header_size = 1;
3608 message.reg--;
3609 }
3610
3611 for (unsigned i = 0; i < coord_components; i++) {
3612 bld.MOV(retype(offset(msg_coords, bld, i), coordinate.type), coordinate);
3613 coordinate = offset(coordinate, bld, 1);
3614 }
3615 fs_reg msg_end = offset(msg_coords, bld, coord_components);
3616 fs_reg msg_lod = offset(msg_coords, bld, 4);
3617
3618 if (shadow_c.file != BAD_FILE) {
3619 fs_reg msg_shadow = msg_lod;
3620 bld.MOV(msg_shadow, shadow_c);
3621 msg_lod = offset(msg_shadow, bld, 1);
3622 msg_end = msg_lod;
3623 }
3624
3625 switch (op) {
3626 case SHADER_OPCODE_TXL:
3627 case FS_OPCODE_TXB:
3628 bld.MOV(msg_lod, lod);
3629 msg_end = offset(msg_lod, bld, 1);
3630 break;
3631 case SHADER_OPCODE_TXD:
3632 /**
3633 * P = u, v, r
3634 * dPdx = dudx, dvdx, drdx
3635 * dPdy = dudy, dvdy, drdy
3636 *
3637 * Load up these values:
3638 * - dudx dudy dvdx dvdy drdx drdy
3639 * - dPdx.x dPdy.x dPdx.y dPdy.y dPdx.z dPdy.z
3640 */
3641 msg_end = msg_lod;
3642 for (unsigned i = 0; i < grad_components; i++) {
3643 bld.MOV(msg_end, lod);
3644 lod = offset(lod, bld, 1);
3645 msg_end = offset(msg_end, bld, 1);
3646
3647 bld.MOV(msg_end, lod2);
3648 lod2 = offset(lod2, bld, 1);
3649 msg_end = offset(msg_end, bld, 1);
3650 }
3651 break;
3652 case SHADER_OPCODE_TXS:
3653 msg_lod = retype(msg_end, BRW_REGISTER_TYPE_UD);
3654 bld.MOV(msg_lod, lod);
3655 msg_end = offset(msg_lod, bld, 1);
3656 break;
3657 case SHADER_OPCODE_TXF:
3658 msg_lod = offset(msg_coords, bld, 3);
3659 bld.MOV(retype(msg_lod, BRW_REGISTER_TYPE_UD), lod);
3660 msg_end = offset(msg_lod, bld, 1);
3661 break;
3662 case SHADER_OPCODE_TXF_CMS:
3663 msg_lod = offset(msg_coords, bld, 3);
3664 /* lod */
3665 bld.MOV(retype(msg_lod, BRW_REGISTER_TYPE_UD), fs_reg(0u));
3666 /* sample index */
3667 bld.MOV(retype(offset(msg_lod, bld, 1), BRW_REGISTER_TYPE_UD), sample_index);
3668 msg_end = offset(msg_lod, bld, 2);
3669 break;
3670 default:
3671 break;
3672 }
3673
3674 inst->opcode = op;
3675 inst->src[0] = reg_undef;
3676 inst->src[1] = sampler;
3677 inst->resize_sources(2);
3678 inst->base_mrf = message.reg;
3679 inst->mlen = msg_end.reg - message.reg;
3680 inst->header_size = header_size;
3681
3682 /* Message length > MAX_SAMPLER_MESSAGE_SIZE disallowed by hardware. */
3683 assert(inst->mlen <= MAX_SAMPLER_MESSAGE_SIZE);
3684 }
3685
3686 static bool
3687 is_high_sampler(const struct brw_device_info *devinfo, const fs_reg &sampler)
3688 {
3689 if (devinfo->gen < 8 && !devinfo->is_haswell)
3690 return false;
3691
3692 return sampler.file != IMM || sampler.fixed_hw_reg.dw1.ud >= 16;
3693 }
3694
3695 static void
3696 lower_sampler_logical_send_gen7(const fs_builder &bld, fs_inst *inst, opcode op,
3697 fs_reg coordinate,
3698 const fs_reg &shadow_c,
3699 fs_reg lod, fs_reg lod2,
3700 const fs_reg &sample_index,
3701 const fs_reg &mcs, const fs_reg &sampler,
3702 fs_reg offset_value,
3703 unsigned coord_components,
3704 unsigned grad_components)
3705 {
3706 const brw_device_info *devinfo = bld.shader->devinfo;
3707 int reg_width = bld.dispatch_width() / 8;
3708 unsigned header_size = 0, length = 0;
3709 fs_reg sources[MAX_SAMPLER_MESSAGE_SIZE];
3710 for (unsigned i = 0; i < ARRAY_SIZE(sources); i++)
3711 sources[i] = bld.vgrf(BRW_REGISTER_TYPE_F);
3712
3713 if (op == SHADER_OPCODE_TG4 || op == SHADER_OPCODE_TG4_OFFSET ||
3714 offset_value.file != BAD_FILE ||
3715 is_high_sampler(devinfo, sampler)) {
3716 /* For general texture offsets (no txf workaround), we need a header to
3717 * put them in. Note that we're only reserving space for it in the
3718 * message payload as it will be initialized implicitly by the
3719 * generator.
3720 *
3721 * TG4 needs to place its channel select in the header, for interaction
3722 * with ARB_texture_swizzle. The sampler index is only 4-bits, so for
3723 * larger sampler numbers we need to offset the Sampler State Pointer in
3724 * the header.
3725 */
3726 header_size = 1;
3727 sources[0] = fs_reg();
3728 length++;
3729 }
3730
3731 if (shadow_c.file != BAD_FILE) {
3732 bld.MOV(sources[length], shadow_c);
3733 length++;
3734 }
3735
3736 bool coordinate_done = false;
3737
3738 /* The sampler can only meaningfully compute LOD for fragment shader
3739 * messages. For all other stages, we change the opcode to TXL and
3740 * hardcode the LOD to 0.
3741 */
3742 if (bld.shader->stage != MESA_SHADER_FRAGMENT &&
3743 op == SHADER_OPCODE_TEX) {
3744 op = SHADER_OPCODE_TXL;
3745 lod = fs_reg(0.0f);
3746 }
3747
3748 /* Set up the LOD info */
3749 switch (op) {
3750 case FS_OPCODE_TXB:
3751 case SHADER_OPCODE_TXL:
3752 bld.MOV(sources[length], lod);
3753 length++;
3754 break;
3755 case SHADER_OPCODE_TXD:
3756 /* TXD should have been lowered in SIMD16 mode. */
3757 assert(bld.dispatch_width() == 8);
3758
3759 /* Load dPdx and the coordinate together:
3760 * [hdr], [ref], x, dPdx.x, dPdy.x, y, dPdx.y, dPdy.y, z, dPdx.z, dPdy.z
3761 */
3762 for (unsigned i = 0; i < coord_components; i++) {
3763 bld.MOV(sources[length], coordinate);
3764 coordinate = offset(coordinate, bld, 1);
3765 length++;
3766
3767 /* For cube map array, the coordinate is (u,v,r,ai) but there are
3768 * only derivatives for (u, v, r).
3769 */
3770 if (i < grad_components) {
3771 bld.MOV(sources[length], lod);
3772 lod = offset(lod, bld, 1);
3773 length++;
3774
3775 bld.MOV(sources[length], lod2);
3776 lod2 = offset(lod2, bld, 1);
3777 length++;
3778 }
3779 }
3780
3781 coordinate_done = true;
3782 break;
3783 case SHADER_OPCODE_TXS:
3784 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), lod);
3785 length++;
3786 break;
3787 case SHADER_OPCODE_TXF:
3788 /* Unfortunately, the parameters for LD are intermixed: u, lod, v, r.
3789 * On Gen9 they are u, v, lod, r
3790 */
3791 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate);
3792 coordinate = offset(coordinate, bld, 1);
3793 length++;
3794
3795 if (devinfo->gen >= 9) {
3796 if (coord_components >= 2) {
3797 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate);
3798 coordinate = offset(coordinate, bld, 1);
3799 }
3800 length++;
3801 }
3802
3803 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), lod);
3804 length++;
3805
3806 for (unsigned i = devinfo->gen >= 9 ? 2 : 1; i < coord_components; i++) {
3807 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate);
3808 coordinate = offset(coordinate, bld, 1);
3809 length++;
3810 }
3811
3812 coordinate_done = true;
3813 break;
3814 case SHADER_OPCODE_TXF_CMS:
3815 case SHADER_OPCODE_TXF_UMS:
3816 case SHADER_OPCODE_TXF_MCS:
3817 if (op == SHADER_OPCODE_TXF_UMS || op == SHADER_OPCODE_TXF_CMS) {
3818 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), sample_index);
3819 length++;
3820 }
3821
3822 if (op == SHADER_OPCODE_TXF_CMS) {
3823 /* Data from the multisample control surface. */
3824 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), mcs);
3825 length++;
3826 }
3827
3828 /* There is no offsetting for this message; just copy in the integer
3829 * texture coordinates.
3830 */
3831 for (unsigned i = 0; i < coord_components; i++) {
3832 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate);
3833 coordinate = offset(coordinate, bld, 1);
3834 length++;
3835 }
3836
3837 coordinate_done = true;
3838 break;
3839 case SHADER_OPCODE_TG4_OFFSET:
3840 /* gather4_po_c should have been lowered in SIMD16 mode. */
3841 assert(bld.dispatch_width() == 8 || shadow_c.file == BAD_FILE);
3842
3843 /* More crazy intermixing */
3844 for (unsigned i = 0; i < 2; i++) { /* u, v */
3845 bld.MOV(sources[length], coordinate);
3846 coordinate = offset(coordinate, bld, 1);
3847 length++;
3848 }
3849
3850 for (unsigned i = 0; i < 2; i++) { /* offu, offv */
3851 bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), offset_value);
3852 offset_value = offset(offset_value, bld, 1);
3853 length++;
3854 }
3855
3856 if (coord_components == 3) { /* r if present */
3857 bld.MOV(sources[length], coordinate);
3858 coordinate = offset(coordinate, bld, 1);
3859 length++;
3860 }
3861
3862 coordinate_done = true;
3863 break;
3864 default:
3865 break;
3866 }
3867
3868 /* Set up the coordinate (except for cases where it was done above) */
3869 if (!coordinate_done) {
3870 for (unsigned i = 0; i < coord_components; i++) {
3871 bld.MOV(sources[length], coordinate);
3872 coordinate = offset(coordinate, bld, 1);
3873 length++;
3874 }
3875 }
3876
3877 int mlen;
3878 if (reg_width == 2)
3879 mlen = length * reg_width - header_size;
3880 else
3881 mlen = length * reg_width;
3882
3883 const fs_reg src_payload = fs_reg(GRF, bld.shader->alloc.allocate(mlen),
3884 BRW_REGISTER_TYPE_F);
3885 bld.LOAD_PAYLOAD(src_payload, sources, length, header_size);
3886
3887 /* Generate the SEND. */
3888 inst->opcode = op;
3889 inst->src[0] = src_payload;
3890 inst->src[1] = sampler;
3891 inst->resize_sources(2);
3892 inst->base_mrf = -1;
3893 inst->mlen = mlen;
3894 inst->header_size = header_size;
3895
3896 /* Message length > MAX_SAMPLER_MESSAGE_SIZE disallowed by hardware. */
3897 assert(inst->mlen <= MAX_SAMPLER_MESSAGE_SIZE);
3898 }
3899
3900 static void
3901 lower_sampler_logical_send(const fs_builder &bld, fs_inst *inst, opcode op)
3902 {
3903 const brw_device_info *devinfo = bld.shader->devinfo;
3904 const fs_reg &coordinate = inst->src[0];
3905 const fs_reg &shadow_c = inst->src[1];
3906 const fs_reg &lod = inst->src[2];
3907 const fs_reg &lod2 = inst->src[3];
3908 const fs_reg &sample_index = inst->src[4];
3909 const fs_reg &mcs = inst->src[5];
3910 const fs_reg &sampler = inst->src[6];
3911 const fs_reg &offset_value = inst->src[7];
3912 assert(inst->src[8].file == IMM && inst->src[9].file == IMM);
3913 const unsigned coord_components = inst->src[8].fixed_hw_reg.dw1.ud;
3914 const unsigned grad_components = inst->src[9].fixed_hw_reg.dw1.ud;
3915
3916 if (devinfo->gen >= 7) {
3917 lower_sampler_logical_send_gen7(bld, inst, op, coordinate,
3918 shadow_c, lod, lod2, sample_index,
3919 mcs, sampler, offset_value,
3920 coord_components, grad_components);
3921 } else if (devinfo->gen >= 5) {
3922 lower_sampler_logical_send_gen5(bld, inst, op, coordinate,
3923 shadow_c, lod, lod2, sample_index,
3924 sampler, offset_value,
3925 coord_components, grad_components);
3926 } else {
3927 lower_sampler_logical_send_gen4(bld, inst, op, coordinate,
3928 shadow_c, lod, lod2, sampler,
3929 coord_components, grad_components);
3930 }
3931 }
3932
3933 /**
3934 * Initialize the header present in some typed and untyped surface
3935 * messages.
3936 */
3937 static fs_reg
3938 emit_surface_header(const fs_builder &bld, const fs_reg &sample_mask)
3939 {
3940 fs_builder ubld = bld.exec_all().group(8, 0);
3941 const fs_reg dst = ubld.vgrf(BRW_REGISTER_TYPE_UD);
3942 ubld.MOV(dst, fs_reg(0));
3943 ubld.MOV(component(dst, 7), sample_mask);
3944 return dst;
3945 }
3946
3947 static void
3948 lower_surface_logical_send(const fs_builder &bld, fs_inst *inst, opcode op,
3949 const fs_reg &sample_mask)
3950 {
3951 /* Get the logical send arguments. */
3952 const fs_reg &addr = inst->src[0];
3953 const fs_reg &src = inst->src[1];
3954 const fs_reg &surface = inst->src[2];
3955 const UNUSED fs_reg &dims = inst->src[3];
3956 const fs_reg &arg = inst->src[4];
3957
3958 /* Calculate the total number of components of the payload. */
3959 const unsigned addr_sz = inst->components_read(0);
3960 const unsigned src_sz = inst->components_read(1);
3961 const unsigned header_sz = (sample_mask.file == BAD_FILE ? 0 : 1);
3962 const unsigned sz = header_sz + addr_sz + src_sz;
3963
3964 /* Allocate space for the payload. */
3965 fs_reg *const components = new fs_reg[sz];
3966 const fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, sz);
3967 unsigned n = 0;
3968
3969 /* Construct the payload. */
3970 if (header_sz)
3971 components[n++] = emit_surface_header(bld, sample_mask);
3972
3973 for (unsigned i = 0; i < addr_sz; i++)
3974 components[n++] = offset(addr, bld, i);
3975
3976 for (unsigned i = 0; i < src_sz; i++)
3977 components[n++] = offset(src, bld, i);
3978
3979 bld.LOAD_PAYLOAD(payload, components, sz, header_sz);
3980
3981 /* Update the original instruction. */
3982 inst->opcode = op;
3983 inst->mlen = header_sz + (addr_sz + src_sz) * inst->exec_size / 8;
3984 inst->header_size = header_sz;
3985
3986 inst->src[0] = payload;
3987 inst->src[1] = surface;
3988 inst->src[2] = arg;
3989 inst->resize_sources(3);
3990
3991 delete[] components;
3992 }
3993
3994 bool
3995 fs_visitor::lower_logical_sends()
3996 {
3997 bool progress = false;
3998
3999 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
4000 const fs_builder ibld(this, block, inst);
4001
4002 switch (inst->opcode) {
4003 case FS_OPCODE_FB_WRITE_LOGICAL:
4004 assert(stage == MESA_SHADER_FRAGMENT);
4005 lower_fb_write_logical_send(ibld, inst,
4006 (const brw_wm_prog_data *)prog_data,
4007 (const brw_wm_prog_key *)key,
4008 payload);
4009 break;
4010
4011 case SHADER_OPCODE_TEX_LOGICAL:
4012 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TEX);
4013 break;
4014
4015 case SHADER_OPCODE_TXD_LOGICAL:
4016 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXD);
4017 break;
4018
4019 case SHADER_OPCODE_TXF_LOGICAL:
4020 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXF);
4021 break;
4022
4023 case SHADER_OPCODE_TXL_LOGICAL:
4024 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXL);
4025 break;
4026
4027 case SHADER_OPCODE_TXS_LOGICAL:
4028 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXS);
4029 break;
4030
4031 case FS_OPCODE_TXB_LOGICAL:
4032 lower_sampler_logical_send(ibld, inst, FS_OPCODE_TXB);
4033 break;
4034
4035 case SHADER_OPCODE_TXF_CMS_LOGICAL:
4036 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXF_CMS);
4037 break;
4038
4039 case SHADER_OPCODE_TXF_UMS_LOGICAL:
4040 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXF_UMS);
4041 break;
4042
4043 case SHADER_OPCODE_TXF_MCS_LOGICAL:
4044 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TXF_MCS);
4045 break;
4046
4047 case SHADER_OPCODE_LOD_LOGICAL:
4048 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_LOD);
4049 break;
4050
4051 case SHADER_OPCODE_TG4_LOGICAL:
4052 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TG4);
4053 break;
4054
4055 case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
4056 lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TG4_OFFSET);
4057 break;
4058
4059 case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
4060 lower_surface_logical_send(ibld, inst,
4061 SHADER_OPCODE_UNTYPED_SURFACE_READ,
4062 fs_reg(0xffff));
4063 break;
4064
4065 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
4066 lower_surface_logical_send(ibld, inst,
4067 SHADER_OPCODE_UNTYPED_SURFACE_WRITE,
4068 ibld.sample_mask_reg());
4069 break;
4070
4071 case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
4072 lower_surface_logical_send(ibld, inst,
4073 SHADER_OPCODE_UNTYPED_ATOMIC,
4074 ibld.sample_mask_reg());
4075 break;
4076
4077 case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
4078 lower_surface_logical_send(ibld, inst,
4079 SHADER_OPCODE_TYPED_SURFACE_READ,
4080 fs_reg(0xffff));
4081 break;
4082
4083 case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
4084 lower_surface_logical_send(ibld, inst,
4085 SHADER_OPCODE_TYPED_SURFACE_WRITE,
4086 ibld.sample_mask_reg());
4087 break;
4088
4089 case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL:
4090 lower_surface_logical_send(ibld, inst,
4091 SHADER_OPCODE_TYPED_ATOMIC,
4092 ibld.sample_mask_reg());
4093 break;
4094
4095 default:
4096 continue;
4097 }
4098
4099 progress = true;
4100 }
4101
4102 if (progress)
4103 invalidate_live_intervals();
4104
4105 return progress;
4106 }
4107
4108 /**
4109 * Get the closest native SIMD width supported by the hardware for instruction
4110 * \p inst. The instruction will be left untouched by
4111 * fs_visitor::lower_simd_width() if the returned value is equal to the
4112 * original execution size.
4113 */
4114 static unsigned
4115 get_lowered_simd_width(const struct brw_device_info *devinfo,
4116 const fs_inst *inst)
4117 {
4118 switch (inst->opcode) {
4119 case BRW_OPCODE_MOV:
4120 case BRW_OPCODE_SEL:
4121 case BRW_OPCODE_NOT:
4122 case BRW_OPCODE_AND:
4123 case BRW_OPCODE_OR:
4124 case BRW_OPCODE_XOR:
4125 case BRW_OPCODE_SHR:
4126 case BRW_OPCODE_SHL:
4127 case BRW_OPCODE_ASR:
4128 case BRW_OPCODE_CMP:
4129 case BRW_OPCODE_CMPN:
4130 case BRW_OPCODE_CSEL:
4131 case BRW_OPCODE_F32TO16:
4132 case BRW_OPCODE_F16TO32:
4133 case BRW_OPCODE_BFREV:
4134 case BRW_OPCODE_BFE:
4135 case BRW_OPCODE_BFI1:
4136 case BRW_OPCODE_BFI2:
4137 case BRW_OPCODE_ADD:
4138 case BRW_OPCODE_MUL:
4139 case BRW_OPCODE_AVG:
4140 case BRW_OPCODE_FRC:
4141 case BRW_OPCODE_RNDU:
4142 case BRW_OPCODE_RNDD:
4143 case BRW_OPCODE_RNDE:
4144 case BRW_OPCODE_RNDZ:
4145 case BRW_OPCODE_LZD:
4146 case BRW_OPCODE_FBH:
4147 case BRW_OPCODE_FBL:
4148 case BRW_OPCODE_CBIT:
4149 case BRW_OPCODE_SAD2:
4150 case BRW_OPCODE_MAD:
4151 case BRW_OPCODE_LRP:
4152 case SHADER_OPCODE_RCP:
4153 case SHADER_OPCODE_RSQ:
4154 case SHADER_OPCODE_SQRT:
4155 case SHADER_OPCODE_EXP2:
4156 case SHADER_OPCODE_LOG2:
4157 case SHADER_OPCODE_POW:
4158 case SHADER_OPCODE_INT_QUOTIENT:
4159 case SHADER_OPCODE_INT_REMAINDER:
4160 case SHADER_OPCODE_SIN:
4161 case SHADER_OPCODE_COS: {
4162 /* According to the PRMs:
4163 * "A. In Direct Addressing mode, a source cannot span more than 2
4164 * adjacent GRF registers.
4165 * B. A destination cannot span more than 2 adjacent GRF registers."
4166 *
4167 * Look for the source or destination with the largest register region
4168 * which is the one that is going to limit the overal execution size of
4169 * the instruction due to this rule.
4170 */
4171 unsigned reg_count = inst->regs_written;
4172
4173 for (unsigned i = 0; i < inst->sources; i++)
4174 reg_count = MAX2(reg_count, (unsigned)inst->regs_read(i));
4175
4176 /* Calculate the maximum execution size of the instruction based on the
4177 * factor by which it goes over the hardware limit of 2 GRFs.
4178 */
4179 return inst->exec_size / DIV_ROUND_UP(reg_count, 2);
4180 }
4181 case SHADER_OPCODE_MULH:
4182 /* MULH is lowered to the MUL/MACH sequence using the accumulator, which
4183 * is 8-wide on Gen7+.
4184 */
4185 return (devinfo->gen >= 7 ? 8 : inst->exec_size);
4186
4187 case FS_OPCODE_FB_WRITE_LOGICAL:
4188 /* Gen6 doesn't support SIMD16 depth writes but we cannot handle them
4189 * here.
4190 */
4191 assert(devinfo->gen != 6 || inst->src[3].file == BAD_FILE ||
4192 inst->exec_size == 8);
4193 /* Dual-source FB writes are unsupported in SIMD16 mode. */
4194 return (inst->src[1].file != BAD_FILE ? 8 : inst->exec_size);
4195
4196 case SHADER_OPCODE_TXD_LOGICAL:
4197 /* TXD is unsupported in SIMD16 mode. */
4198 return 8;
4199
4200 case SHADER_OPCODE_TG4_OFFSET_LOGICAL: {
4201 /* gather4_po_c is unsupported in SIMD16 mode. */
4202 const fs_reg &shadow_c = inst->src[1];
4203 return (shadow_c.file != BAD_FILE ? 8 : inst->exec_size);
4204 }
4205 case SHADER_OPCODE_TXL_LOGICAL:
4206 case FS_OPCODE_TXB_LOGICAL: {
4207 /* Gen4 doesn't have SIMD8 non-shadow-compare bias/LOD instructions, and
4208 * Gen4-6 can't support TXL and TXB with shadow comparison in SIMD16
4209 * mode because the message exceeds the maximum length of 11.
4210 */
4211 const fs_reg &shadow_c = inst->src[1];
4212 if (devinfo->gen == 4 && shadow_c.file == BAD_FILE)
4213 return 16;
4214 else if (devinfo->gen < 7 && shadow_c.file != BAD_FILE)
4215 return 8;
4216 else
4217 return inst->exec_size;
4218 }
4219 case SHADER_OPCODE_TXF_LOGICAL:
4220 case SHADER_OPCODE_TXS_LOGICAL:
4221 /* Gen4 doesn't have SIMD8 variants for the RESINFO and LD-with-LOD
4222 * messages. Use SIMD16 instead.
4223 */
4224 if (devinfo->gen == 4)
4225 return 16;
4226 else
4227 return inst->exec_size;
4228
4229 case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL:
4230 case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
4231 case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
4232 return 8;
4233
4234 default:
4235 return inst->exec_size;
4236 }
4237 }
4238
4239 /**
4240 * The \p rows array of registers represents a \p num_rows by \p num_columns
4241 * matrix in row-major order, write it in column-major order into the register
4242 * passed as destination. \p stride gives the separation between matrix
4243 * elements in the input in fs_builder::dispatch_width() units.
4244 */
4245 static void
4246 emit_transpose(const fs_builder &bld,
4247 const fs_reg &dst, const fs_reg *rows,
4248 unsigned num_rows, unsigned num_columns, unsigned stride)
4249 {
4250 fs_reg *const components = new fs_reg[num_rows * num_columns];
4251
4252 for (unsigned i = 0; i < num_columns; ++i) {
4253 for (unsigned j = 0; j < num_rows; ++j)
4254 components[num_rows * i + j] = offset(rows[j], bld, stride * i);
4255 }
4256
4257 bld.LOAD_PAYLOAD(dst, components, num_rows * num_columns, 0);
4258
4259 delete[] components;
4260 }
4261
4262 bool
4263 fs_visitor::lower_simd_width()
4264 {
4265 bool progress = false;
4266
4267 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
4268 const unsigned lower_width = get_lowered_simd_width(devinfo, inst);
4269
4270 if (lower_width != inst->exec_size) {
4271 /* Builder matching the original instruction. We may also need to
4272 * emit an instruction of width larger than the original, set the
4273 * execution size of the builder to the highest of both for now so
4274 * we're sure that both cases can be handled.
4275 */
4276 const fs_builder ibld = bld.at(block, inst)
4277 .exec_all(inst->force_writemask_all)
4278 .group(MAX2(inst->exec_size, lower_width),
4279 inst->force_sechalf);
4280
4281 /* Split the copies in chunks of the execution width of either the
4282 * original or the lowered instruction, whichever is lower.
4283 */
4284 const unsigned copy_width = MIN2(lower_width, inst->exec_size);
4285 const unsigned n = inst->exec_size / copy_width;
4286 const unsigned dst_size = inst->regs_written * REG_SIZE /
4287 inst->dst.component_size(inst->exec_size);
4288 fs_reg dsts[4];
4289
4290 assert(n > 0 && n <= ARRAY_SIZE(dsts) &&
4291 !inst->writes_accumulator && !inst->mlen);
4292
4293 for (unsigned i = 0; i < n; i++) {
4294 /* Emit a copy of the original instruction with the lowered width.
4295 * If the EOT flag was set throw it away except for the last
4296 * instruction to avoid killing the thread prematurely.
4297 */
4298 fs_inst split_inst = *inst;
4299 split_inst.exec_size = lower_width;
4300 split_inst.eot = inst->eot && i == n - 1;
4301
4302 /* Select the correct channel enables for the i-th group, then
4303 * transform the sources and destination and emit the lowered
4304 * instruction.
4305 */
4306 const fs_builder lbld = ibld.group(lower_width, i);
4307
4308 for (unsigned j = 0; j < inst->sources; j++) {
4309 if (inst->src[j].file != BAD_FILE &&
4310 !is_uniform(inst->src[j])) {
4311 /* Get the i-th copy_width-wide chunk of the source. */
4312 const fs_reg src = horiz_offset(inst->src[j], copy_width * i);
4313 const unsigned src_size = inst->components_read(j);
4314
4315 /* Use a trivial transposition to copy one every n
4316 * copy_width-wide components of the register into a
4317 * temporary passed as source to the lowered instruction.
4318 */
4319 split_inst.src[j] = lbld.vgrf(inst->src[j].type, src_size);
4320 emit_transpose(lbld.group(copy_width, 0),
4321 split_inst.src[j], &src, 1, src_size, n);
4322 }
4323 }
4324
4325 if (inst->regs_written) {
4326 /* Allocate enough space to hold the result of the lowered
4327 * instruction and fix up the number of registers written.
4328 */
4329 split_inst.dst = dsts[i] =
4330 lbld.vgrf(inst->dst.type, dst_size);
4331 split_inst.regs_written =
4332 DIV_ROUND_UP(inst->regs_written * lower_width,
4333 inst->exec_size);
4334 }
4335
4336 lbld.emit(split_inst);
4337 }
4338
4339 if (inst->regs_written) {
4340 /* Distance between useful channels in the temporaries, skipping
4341 * garbage if the lowered instruction is wider than the original.
4342 */
4343 const unsigned m = lower_width / copy_width;
4344
4345 /* Interleave the components of the result from the lowered
4346 * instructions. We need to set exec_all() when copying more than
4347 * one half per component, because LOAD_PAYLOAD (in terms of which
4348 * emit_transpose is implemented) can only use the same channel
4349 * enable signals for all of its non-header sources.
4350 */
4351 emit_transpose(ibld.exec_all(inst->exec_size > copy_width)
4352 .group(copy_width, 0),
4353 inst->dst, dsts, n, dst_size, m);
4354 }
4355
4356 inst->remove(block);
4357 progress = true;
4358 }
4359 }
4360
4361 if (progress)
4362 invalidate_live_intervals();
4363
4364 return progress;
4365 }
4366
4367 void
4368 fs_visitor::dump_instructions()
4369 {
4370 dump_instructions(NULL);
4371 }
4372
4373 void
4374 fs_visitor::dump_instructions(const char *name)
4375 {
4376 FILE *file = stderr;
4377 if (name && geteuid() != 0) {
4378 file = fopen(name, "w");
4379 if (!file)
4380 file = stderr;
4381 }
4382
4383 if (cfg) {
4384 calculate_register_pressure();
4385 int ip = 0, max_pressure = 0;
4386 foreach_block_and_inst(block, backend_instruction, inst, cfg) {
4387 max_pressure = MAX2(max_pressure, regs_live_at_ip[ip]);
4388 fprintf(file, "{%3d} %4d: ", regs_live_at_ip[ip], ip);
4389 dump_instruction(inst, file);
4390 ip++;
4391 }
4392 fprintf(file, "Maximum %3d registers live at once.\n", max_pressure);
4393 } else {
4394 int ip = 0;
4395 foreach_in_list(backend_instruction, inst, &instructions) {
4396 fprintf(file, "%4d: ", ip++);
4397 dump_instruction(inst, file);
4398 }
4399 }
4400
4401 if (file != stderr) {
4402 fclose(file);
4403 }
4404 }
4405
4406 void
4407 fs_visitor::dump_instruction(backend_instruction *be_inst)
4408 {
4409 dump_instruction(be_inst, stderr);
4410 }
4411
4412 void
4413 fs_visitor::dump_instruction(backend_instruction *be_inst, FILE *file)
4414 {
4415 fs_inst *inst = (fs_inst *)be_inst;
4416
4417 if (inst->predicate) {
4418 fprintf(file, "(%cf0.%d) ",
4419 inst->predicate_inverse ? '-' : '+',
4420 inst->flag_subreg);
4421 }
4422
4423 fprintf(file, "%s", brw_instruction_name(inst->opcode));
4424 if (inst->saturate)
4425 fprintf(file, ".sat");
4426 if (inst->conditional_mod) {
4427 fprintf(file, "%s", conditional_modifier[inst->conditional_mod]);
4428 if (!inst->predicate &&
4429 (devinfo->gen < 5 || (inst->opcode != BRW_OPCODE_SEL &&
4430 inst->opcode != BRW_OPCODE_IF &&
4431 inst->opcode != BRW_OPCODE_WHILE))) {
4432 fprintf(file, ".f0.%d", inst->flag_subreg);
4433 }
4434 }
4435 fprintf(file, "(%d) ", inst->exec_size);
4436
4437 if (inst->mlen) {
4438 fprintf(file, "(mlen: %d) ", inst->mlen);
4439 }
4440
4441 switch (inst->dst.file) {
4442 case GRF:
4443 fprintf(file, "vgrf%d", inst->dst.reg);
4444 if (alloc.sizes[inst->dst.reg] != inst->regs_written ||
4445 inst->dst.subreg_offset)
4446 fprintf(file, "+%d.%d",
4447 inst->dst.reg_offset, inst->dst.subreg_offset);
4448 break;
4449 case MRF:
4450 fprintf(file, "m%d", inst->dst.reg);
4451 break;
4452 case BAD_FILE:
4453 fprintf(file, "(null)");
4454 break;
4455 case UNIFORM:
4456 fprintf(file, "***u%d***", inst->dst.reg + inst->dst.reg_offset);
4457 break;
4458 case ATTR:
4459 fprintf(file, "***attr%d***", inst->dst.reg + inst->dst.reg_offset);
4460 break;
4461 case HW_REG:
4462 if (inst->dst.fixed_hw_reg.file == BRW_ARCHITECTURE_REGISTER_FILE) {
4463 switch (inst->dst.fixed_hw_reg.nr) {
4464 case BRW_ARF_NULL:
4465 fprintf(file, "null");
4466 break;
4467 case BRW_ARF_ADDRESS:
4468 fprintf(file, "a0.%d", inst->dst.fixed_hw_reg.subnr);
4469 break;
4470 case BRW_ARF_ACCUMULATOR:
4471 fprintf(file, "acc%d", inst->dst.fixed_hw_reg.subnr);
4472 break;
4473 case BRW_ARF_FLAG:
4474 fprintf(file, "f%d.%d", inst->dst.fixed_hw_reg.nr & 0xf,
4475 inst->dst.fixed_hw_reg.subnr);
4476 break;
4477 default:
4478 fprintf(file, "arf%d.%d", inst->dst.fixed_hw_reg.nr & 0xf,
4479 inst->dst.fixed_hw_reg.subnr);
4480 break;
4481 }
4482 } else {
4483 fprintf(file, "hw_reg%d", inst->dst.fixed_hw_reg.nr);
4484 }
4485 if (inst->dst.fixed_hw_reg.subnr)
4486 fprintf(file, "+%d", inst->dst.fixed_hw_reg.subnr);
4487 break;
4488 default:
4489 fprintf(file, "???");
4490 break;
4491 }
4492 fprintf(file, ":%s, ", brw_reg_type_letters(inst->dst.type));
4493
4494 for (int i = 0; i < inst->sources; i++) {
4495 if (inst->src[i].negate)
4496 fprintf(file, "-");
4497 if (inst->src[i].abs)
4498 fprintf(file, "|");
4499 switch (inst->src[i].file) {
4500 case GRF:
4501 fprintf(file, "vgrf%d", inst->src[i].reg);
4502 if (alloc.sizes[inst->src[i].reg] != (unsigned)inst->regs_read(i) ||
4503 inst->src[i].subreg_offset)
4504 fprintf(file, "+%d.%d", inst->src[i].reg_offset,
4505 inst->src[i].subreg_offset);
4506 break;
4507 case MRF:
4508 fprintf(file, "***m%d***", inst->src[i].reg);
4509 break;
4510 case ATTR:
4511 fprintf(file, "attr%d", inst->src[i].reg + inst->src[i].reg_offset);
4512 break;
4513 case UNIFORM:
4514 fprintf(file, "u%d", inst->src[i].reg + inst->src[i].reg_offset);
4515 if (inst->src[i].reladdr) {
4516 fprintf(file, "+reladdr");
4517 } else if (inst->src[i].subreg_offset) {
4518 fprintf(file, "+%d.%d", inst->src[i].reg_offset,
4519 inst->src[i].subreg_offset);
4520 }
4521 break;
4522 case BAD_FILE:
4523 fprintf(file, "(null)");
4524 break;
4525 case IMM:
4526 switch (inst->src[i].type) {
4527 case BRW_REGISTER_TYPE_F:
4528 fprintf(file, "%ff", inst->src[i].fixed_hw_reg.dw1.f);
4529 break;
4530 case BRW_REGISTER_TYPE_W:
4531 case BRW_REGISTER_TYPE_D:
4532 fprintf(file, "%dd", inst->src[i].fixed_hw_reg.dw1.d);
4533 break;
4534 case BRW_REGISTER_TYPE_UW:
4535 case BRW_REGISTER_TYPE_UD:
4536 fprintf(file, "%uu", inst->src[i].fixed_hw_reg.dw1.ud);
4537 break;
4538 case BRW_REGISTER_TYPE_VF:
4539 fprintf(file, "[%-gF, %-gF, %-gF, %-gF]",
4540 brw_vf_to_float((inst->src[i].fixed_hw_reg.dw1.ud >> 0) & 0xff),
4541 brw_vf_to_float((inst->src[i].fixed_hw_reg.dw1.ud >> 8) & 0xff),
4542 brw_vf_to_float((inst->src[i].fixed_hw_reg.dw1.ud >> 16) & 0xff),
4543 brw_vf_to_float((inst->src[i].fixed_hw_reg.dw1.ud >> 24) & 0xff));
4544 break;
4545 default:
4546 fprintf(file, "???");
4547 break;
4548 }
4549 break;
4550 case HW_REG:
4551 if (inst->src[i].fixed_hw_reg.negate)
4552 fprintf(file, "-");
4553 if (inst->src[i].fixed_hw_reg.abs)
4554 fprintf(file, "|");
4555 if (inst->src[i].fixed_hw_reg.file == BRW_ARCHITECTURE_REGISTER_FILE) {
4556 switch (inst->src[i].fixed_hw_reg.nr) {
4557 case BRW_ARF_NULL:
4558 fprintf(file, "null");
4559 break;
4560 case BRW_ARF_ADDRESS:
4561 fprintf(file, "a0.%d", inst->src[i].fixed_hw_reg.subnr);
4562 break;
4563 case BRW_ARF_ACCUMULATOR:
4564 fprintf(file, "acc%d", inst->src[i].fixed_hw_reg.subnr);
4565 break;
4566 case BRW_ARF_FLAG:
4567 fprintf(file, "f%d.%d", inst->src[i].fixed_hw_reg.nr & 0xf,
4568 inst->src[i].fixed_hw_reg.subnr);
4569 break;
4570 default:
4571 fprintf(file, "arf%d.%d", inst->src[i].fixed_hw_reg.nr & 0xf,
4572 inst->src[i].fixed_hw_reg.subnr);
4573 break;
4574 }
4575 } else {
4576 fprintf(file, "hw_reg%d", inst->src[i].fixed_hw_reg.nr);
4577 }
4578 if (inst->src[i].fixed_hw_reg.subnr)
4579 fprintf(file, "+%d", inst->src[i].fixed_hw_reg.subnr);
4580 if (inst->src[i].fixed_hw_reg.abs)
4581 fprintf(file, "|");
4582 break;
4583 default:
4584 fprintf(file, "???");
4585 break;
4586 }
4587 if (inst->src[i].abs)
4588 fprintf(file, "|");
4589
4590 if (inst->src[i].file != IMM) {
4591 fprintf(file, ":%s", brw_reg_type_letters(inst->src[i].type));
4592 }
4593
4594 if (i < inst->sources - 1 && inst->src[i + 1].file != BAD_FILE)
4595 fprintf(file, ", ");
4596 }
4597
4598 fprintf(file, " ");
4599
4600 if (dispatch_width == 16 && inst->exec_size == 8) {
4601 if (inst->force_sechalf)
4602 fprintf(file, "2ndhalf ");
4603 else
4604 fprintf(file, "1sthalf ");
4605 }
4606
4607 fprintf(file, "\n");
4608 }
4609
4610 /**
4611 * Possibly returns an instruction that set up @param reg.
4612 *
4613 * Sometimes we want to take the result of some expression/variable
4614 * dereference tree and rewrite the instruction generating the result
4615 * of the tree. When processing the tree, we know that the
4616 * instructions generated are all writing temporaries that are dead
4617 * outside of this tree. So, if we have some instructions that write
4618 * a temporary, we're free to point that temp write somewhere else.
4619 *
4620 * Note that this doesn't guarantee that the instruction generated
4621 * only reg -- it might be the size=4 destination of a texture instruction.
4622 */
4623 fs_inst *
4624 fs_visitor::get_instruction_generating_reg(fs_inst *start,
4625 fs_inst *end,
4626 const fs_reg &reg)
4627 {
4628 if (end == start ||
4629 end->is_partial_write() ||
4630 reg.reladdr ||
4631 !reg.equals(end->dst)) {
4632 return NULL;
4633 } else {
4634 return end;
4635 }
4636 }
4637
4638 void
4639 fs_visitor::setup_payload_gen6()
4640 {
4641 bool uses_depth =
4642 (prog->InputsRead & (1 << VARYING_SLOT_POS)) != 0;
4643 unsigned barycentric_interp_modes =
4644 (stage == MESA_SHADER_FRAGMENT) ?
4645 ((brw_wm_prog_data*) this->prog_data)->barycentric_interp_modes : 0;
4646
4647 assert(devinfo->gen >= 6);
4648
4649 /* R0-1: masks, pixel X/Y coordinates. */
4650 payload.num_regs = 2;
4651 /* R2: only for 32-pixel dispatch.*/
4652
4653 /* R3-26: barycentric interpolation coordinates. These appear in the
4654 * same order that they appear in the brw_wm_barycentric_interp_mode
4655 * enum. Each set of coordinates occupies 2 registers if dispatch width
4656 * == 8 and 4 registers if dispatch width == 16. Coordinates only
4657 * appear if they were enabled using the "Barycentric Interpolation
4658 * Mode" bits in WM_STATE.
4659 */
4660 for (int i = 0; i < BRW_WM_BARYCENTRIC_INTERP_MODE_COUNT; ++i) {
4661 if (barycentric_interp_modes & (1 << i)) {
4662 payload.barycentric_coord_reg[i] = payload.num_regs;
4663 payload.num_regs += 2;
4664 if (dispatch_width == 16) {
4665 payload.num_regs += 2;
4666 }
4667 }
4668 }
4669
4670 /* R27: interpolated depth if uses source depth */
4671 if (uses_depth) {
4672 payload.source_depth_reg = payload.num_regs;
4673 payload.num_regs++;
4674 if (dispatch_width == 16) {
4675 /* R28: interpolated depth if not SIMD8. */
4676 payload.num_regs++;
4677 }
4678 }
4679 /* R29: interpolated W set if GEN6_WM_USES_SOURCE_W. */
4680 if (uses_depth) {
4681 payload.source_w_reg = payload.num_regs;
4682 payload.num_regs++;
4683 if (dispatch_width == 16) {
4684 /* R30: interpolated W if not SIMD8. */
4685 payload.num_regs++;
4686 }
4687 }
4688
4689 if (stage == MESA_SHADER_FRAGMENT) {
4690 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
4691 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
4692 prog_data->uses_pos_offset = key->compute_pos_offset;
4693 /* R31: MSAA position offsets. */
4694 if (prog_data->uses_pos_offset) {
4695 payload.sample_pos_reg = payload.num_regs;
4696 payload.num_regs++;
4697 }
4698 }
4699
4700 /* R32: MSAA input coverage mask */
4701 if (prog->SystemValuesRead & SYSTEM_BIT_SAMPLE_MASK_IN) {
4702 assert(devinfo->gen >= 7);
4703 payload.sample_mask_in_reg = payload.num_regs;
4704 payload.num_regs++;
4705 if (dispatch_width == 16) {
4706 /* R33: input coverage mask if not SIMD8. */
4707 payload.num_regs++;
4708 }
4709 }
4710
4711 /* R34-: bary for 32-pixel. */
4712 /* R58-59: interp W for 32-pixel. */
4713
4714 if (prog->OutputsWritten & BITFIELD64_BIT(FRAG_RESULT_DEPTH)) {
4715 source_depth_to_render_target = true;
4716 }
4717 }
4718
4719 void
4720 fs_visitor::setup_vs_payload()
4721 {
4722 /* R0: thread header, R1: urb handles */
4723 payload.num_regs = 2;
4724 }
4725
4726 void
4727 fs_visitor::setup_cs_payload()
4728 {
4729 assert(devinfo->gen >= 7);
4730
4731 payload.num_regs = 1;
4732 }
4733
4734 void
4735 fs_visitor::assign_binding_table_offsets()
4736 {
4737 assert(stage == MESA_SHADER_FRAGMENT);
4738 brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
4739 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
4740 uint32_t next_binding_table_offset = 0;
4741
4742 /* If there are no color regions, we still perform an FB write to a null
4743 * renderbuffer, which we place at surface index 0.
4744 */
4745 prog_data->binding_table.render_target_start = next_binding_table_offset;
4746 next_binding_table_offset += MAX2(key->nr_color_regions, 1);
4747
4748 assign_common_binding_table_offsets(next_binding_table_offset);
4749 }
4750
4751 void
4752 fs_visitor::calculate_register_pressure()
4753 {
4754 invalidate_live_intervals();
4755 calculate_live_intervals();
4756
4757 unsigned num_instructions = 0;
4758 foreach_block(block, cfg)
4759 num_instructions += block->instructions.length();
4760
4761 regs_live_at_ip = rzalloc_array(mem_ctx, int, num_instructions);
4762
4763 for (unsigned reg = 0; reg < alloc.count; reg++) {
4764 for (int ip = virtual_grf_start[reg]; ip <= virtual_grf_end[reg]; ip++)
4765 regs_live_at_ip[ip] += alloc.sizes[reg];
4766 }
4767 }
4768
4769 void
4770 fs_visitor::optimize()
4771 {
4772 /* bld is the common builder object pointing at the end of the program we
4773 * used to translate it into i965 IR. For the optimization and lowering
4774 * passes coming next, any code added after the end of the program without
4775 * having explicitly called fs_builder::at() clearly points at a mistake.
4776 * Ideally optimization passes wouldn't be part of the visitor so they
4777 * wouldn't have access to bld at all, but they do, so just in case some
4778 * pass forgets to ask for a location explicitly set it to NULL here to
4779 * make it trip. The dispatch width is initialized to a bogus value to
4780 * make sure that optimizations set the execution controls explicitly to
4781 * match the code they are manipulating instead of relying on the defaults.
4782 */
4783 bld = fs_builder(this, 64);
4784
4785 assign_constant_locations();
4786 demote_pull_constants();
4787
4788 split_virtual_grfs();
4789
4790 #define OPT(pass, args...) ({ \
4791 pass_num++; \
4792 bool this_progress = pass(args); \
4793 \
4794 if (unlikely(INTEL_DEBUG & DEBUG_OPTIMIZER) && this_progress) { \
4795 char filename[64]; \
4796 snprintf(filename, 64, "%s%d-%04d-%02d-%02d-" #pass, \
4797 stage_abbrev, dispatch_width, shader_prog ? shader_prog->Name : 0, iteration, pass_num); \
4798 \
4799 backend_shader::dump_instructions(filename); \
4800 } \
4801 \
4802 progress = progress || this_progress; \
4803 this_progress; \
4804 })
4805
4806 if (unlikely(INTEL_DEBUG & DEBUG_OPTIMIZER)) {
4807 char filename[64];
4808 snprintf(filename, 64, "%s%d-%04d-00-start",
4809 stage_abbrev, dispatch_width,
4810 shader_prog ? shader_prog->Name : 0);
4811
4812 backend_shader::dump_instructions(filename);
4813 }
4814
4815 bool progress = false;
4816 int iteration = 0;
4817 int pass_num = 0;
4818
4819 OPT(lower_simd_width);
4820 OPT(lower_logical_sends);
4821
4822 do {
4823 progress = false;
4824 pass_num = 0;
4825 iteration++;
4826
4827 OPT(remove_duplicate_mrf_writes);
4828
4829 OPT(opt_algebraic);
4830 OPT(opt_cse);
4831 OPT(opt_copy_propagate);
4832 OPT(opt_peephole_predicated_break);
4833 OPT(opt_cmod_propagation);
4834 OPT(dead_code_eliminate);
4835 OPT(opt_peephole_sel);
4836 OPT(dead_control_flow_eliminate, this);
4837 OPT(opt_register_renaming);
4838 OPT(opt_redundant_discard_jumps);
4839 OPT(opt_saturate_propagation);
4840 OPT(opt_zero_samples);
4841 OPT(register_coalesce);
4842 OPT(compute_to_mrf);
4843 OPT(eliminate_find_live_channel);
4844
4845 OPT(compact_virtual_grfs);
4846 } while (progress);
4847
4848 pass_num = 0;
4849
4850 OPT(opt_sampler_eot);
4851
4852 if (OPT(lower_load_payload)) {
4853 split_virtual_grfs();
4854 OPT(register_coalesce);
4855 OPT(compute_to_mrf);
4856 OPT(dead_code_eliminate);
4857 }
4858
4859 OPT(opt_combine_constants);
4860 OPT(lower_integer_multiplication);
4861
4862 lower_uniform_pull_constant_loads();
4863 }
4864
4865 /**
4866 * Three source instruction must have a GRF/MRF destination register.
4867 * ARF NULL is not allowed. Fix that up by allocating a temporary GRF.
4868 */
4869 void
4870 fs_visitor::fixup_3src_null_dest()
4871 {
4872 foreach_block_and_inst_safe (block, fs_inst, inst, cfg) {
4873 if (inst->is_3src() && inst->dst.is_null()) {
4874 inst->dst = fs_reg(GRF, alloc.allocate(dispatch_width / 8),
4875 inst->dst.type);
4876 }
4877 }
4878 }
4879
4880 void
4881 fs_visitor::allocate_registers()
4882 {
4883 bool allocated_without_spills;
4884
4885 static const enum instruction_scheduler_mode pre_modes[] = {
4886 SCHEDULE_PRE,
4887 SCHEDULE_PRE_NON_LIFO,
4888 SCHEDULE_PRE_LIFO,
4889 };
4890
4891 /* Try each scheduling heuristic to see if it can successfully register
4892 * allocate without spilling. They should be ordered by decreasing
4893 * performance but increasing likelihood of allocating.
4894 */
4895 for (unsigned i = 0; i < ARRAY_SIZE(pre_modes); i++) {
4896 schedule_instructions(pre_modes[i]);
4897
4898 if (0) {
4899 assign_regs_trivial();
4900 allocated_without_spills = true;
4901 } else {
4902 allocated_without_spills = assign_regs(false);
4903 }
4904 if (allocated_without_spills)
4905 break;
4906 }
4907
4908 if (!allocated_without_spills) {
4909 /* We assume that any spilling is worse than just dropping back to
4910 * SIMD8. There's probably actually some intermediate point where
4911 * SIMD16 with a couple of spills is still better.
4912 */
4913 if (dispatch_width == 16) {
4914 fail("Failure to register allocate. Reduce number of "
4915 "live scalar values to avoid this.");
4916 } else {
4917 compiler->shader_perf_log(log_data,
4918 "%s shader triggered register spilling. "
4919 "Try reducing the number of live scalar "
4920 "values to improve performance.\n",
4921 stage_name);
4922 }
4923
4924 /* Since we're out of heuristics, just go spill registers until we
4925 * get an allocation.
4926 */
4927 while (!assign_regs(true)) {
4928 if (failed)
4929 break;
4930 }
4931 }
4932
4933 /* This must come after all optimization and register allocation, since
4934 * it inserts dead code that happens to have side effects, and it does
4935 * so based on the actual physical registers in use.
4936 */
4937 insert_gen4_send_dependency_workarounds();
4938
4939 if (failed)
4940 return;
4941
4942 if (!allocated_without_spills)
4943 schedule_instructions(SCHEDULE_POST);
4944
4945 if (last_scratch > 0)
4946 prog_data->total_scratch = brw_get_scratch_size(last_scratch);
4947 }
4948
4949 bool
4950 fs_visitor::run_vs(gl_clip_plane *clip_planes)
4951 {
4952 assert(stage == MESA_SHADER_VERTEX);
4953
4954 assign_common_binding_table_offsets(0);
4955 setup_vs_payload();
4956
4957 if (shader_time_index >= 0)
4958 emit_shader_time_begin();
4959
4960 emit_nir_code();
4961
4962 if (failed)
4963 return false;
4964
4965 compute_clip_distance(clip_planes);
4966
4967 emit_urb_writes();
4968
4969 if (shader_time_index >= 0)
4970 emit_shader_time_end();
4971
4972 calculate_cfg();
4973
4974 optimize();
4975
4976 assign_curb_setup();
4977 assign_vs_urb_setup();
4978
4979 fixup_3src_null_dest();
4980 allocate_registers();
4981
4982 return !failed;
4983 }
4984
4985 bool
4986 fs_visitor::run_fs(bool do_rep_send)
4987 {
4988 brw_wm_prog_data *wm_prog_data = (brw_wm_prog_data *) this->prog_data;
4989 brw_wm_prog_key *wm_key = (brw_wm_prog_key *) this->key;
4990
4991 assert(stage == MESA_SHADER_FRAGMENT);
4992
4993 sanity_param_count = prog->Parameters->NumParameters;
4994
4995 assign_binding_table_offsets();
4996
4997 if (devinfo->gen >= 6)
4998 setup_payload_gen6();
4999 else
5000 setup_payload_gen4();
5001
5002 if (0) {
5003 emit_dummy_fs();
5004 } else if (do_rep_send) {
5005 assert(dispatch_width == 16);
5006 emit_repclear_shader();
5007 } else {
5008 if (shader_time_index >= 0)
5009 emit_shader_time_begin();
5010
5011 calculate_urb_setup();
5012 if (prog->InputsRead > 0) {
5013 if (devinfo->gen < 6)
5014 emit_interpolation_setup_gen4();
5015 else
5016 emit_interpolation_setup_gen6();
5017 }
5018
5019 /* We handle discards by keeping track of the still-live pixels in f0.1.
5020 * Initialize it with the dispatched pixels.
5021 */
5022 if (wm_prog_data->uses_kill) {
5023 fs_inst *discard_init = bld.emit(FS_OPCODE_MOV_DISPATCH_TO_FLAGS);
5024 discard_init->flag_subreg = 1;
5025 }
5026
5027 /* Generate FS IR for main(). (the visitor only descends into
5028 * functions called "main").
5029 */
5030 emit_nir_code();
5031
5032 if (failed)
5033 return false;
5034
5035 if (wm_prog_data->uses_kill)
5036 bld.emit(FS_OPCODE_PLACEHOLDER_HALT);
5037
5038 if (wm_key->alpha_test_func)
5039 emit_alpha_test();
5040
5041 emit_fb_writes();
5042
5043 if (shader_time_index >= 0)
5044 emit_shader_time_end();
5045
5046 calculate_cfg();
5047
5048 optimize();
5049
5050 assign_curb_setup();
5051 assign_urb_setup();
5052
5053 fixup_3src_null_dest();
5054 allocate_registers();
5055
5056 if (failed)
5057 return false;
5058 }
5059
5060 if (dispatch_width == 8)
5061 wm_prog_data->reg_blocks = brw_register_blocks(grf_used);
5062 else
5063 wm_prog_data->reg_blocks_16 = brw_register_blocks(grf_used);
5064
5065 /* If any state parameters were appended, then ParameterValues could have
5066 * been realloced, in which case the driver uniform storage set up by
5067 * _mesa_associate_uniform_storage() would point to freed memory. Make
5068 * sure that didn't happen.
5069 */
5070 assert(sanity_param_count == prog->Parameters->NumParameters);
5071
5072 return !failed;
5073 }
5074
5075 bool
5076 fs_visitor::run_cs()
5077 {
5078 assert(stage == MESA_SHADER_COMPUTE);
5079 assert(shader);
5080
5081 sanity_param_count = prog->Parameters->NumParameters;
5082
5083 assign_common_binding_table_offsets(0);
5084
5085 setup_cs_payload();
5086
5087 if (shader_time_index >= 0)
5088 emit_shader_time_begin();
5089
5090 emit_nir_code();
5091
5092 if (failed)
5093 return false;
5094
5095 emit_cs_terminate();
5096
5097 if (shader_time_index >= 0)
5098 emit_shader_time_end();
5099
5100 calculate_cfg();
5101
5102 optimize();
5103
5104 assign_curb_setup();
5105
5106 fixup_3src_null_dest();
5107 allocate_registers();
5108
5109 if (failed)
5110 return false;
5111
5112 /* If any state parameters were appended, then ParameterValues could have
5113 * been realloced, in which case the driver uniform storage set up by
5114 * _mesa_associate_uniform_storage() would point to freed memory. Make
5115 * sure that didn't happen.
5116 */
5117 assert(sanity_param_count == prog->Parameters->NumParameters);
5118
5119 return !failed;
5120 }
5121
5122 const unsigned *
5123 brw_wm_fs_emit(struct brw_context *brw,
5124 void *mem_ctx,
5125 const struct brw_wm_prog_key *key,
5126 struct brw_wm_prog_data *prog_data,
5127 struct gl_fragment_program *fp,
5128 struct gl_shader_program *prog,
5129 unsigned *final_assembly_size)
5130 {
5131 bool start_busy = false;
5132 double start_time = 0;
5133
5134 if (unlikely(brw->perf_debug)) {
5135 start_busy = (brw->batch.last_bo &&
5136 drm_intel_bo_busy(brw->batch.last_bo));
5137 start_time = get_time();
5138 }
5139
5140 struct brw_shader *shader = NULL;
5141 if (prog)
5142 shader = (brw_shader *) prog->_LinkedShaders[MESA_SHADER_FRAGMENT];
5143
5144 if (unlikely(INTEL_DEBUG & DEBUG_WM))
5145 brw_dump_ir("fragment", prog, &shader->base, &fp->Base);
5146
5147 int st_index8 = -1, st_index16 = -1;
5148 if (INTEL_DEBUG & DEBUG_SHADER_TIME) {
5149 st_index8 = brw_get_shader_time_index(brw, prog, &fp->Base, ST_FS8);
5150 st_index16 = brw_get_shader_time_index(brw, prog, &fp->Base, ST_FS16);
5151 }
5152
5153 /* Now the main event: Visit the shader IR and generate our FS IR for it.
5154 */
5155 fs_visitor v(brw->intelScreen->compiler, brw,
5156 mem_ctx, MESA_SHADER_FRAGMENT, key, &prog_data->base,
5157 prog, &fp->Base, 8, st_index8);
5158 if (!v.run_fs(false /* do_rep_send */)) {
5159 if (prog) {
5160 prog->LinkStatus = false;
5161 ralloc_strcat(&prog->InfoLog, v.fail_msg);
5162 }
5163
5164 _mesa_problem(NULL, "Failed to compile fragment shader: %s\n",
5165 v.fail_msg);
5166
5167 return NULL;
5168 }
5169
5170 cfg_t *simd16_cfg = NULL;
5171 fs_visitor v2(brw->intelScreen->compiler, brw,
5172 mem_ctx, MESA_SHADER_FRAGMENT, key, &prog_data->base,
5173 prog, &fp->Base, 16, st_index16);
5174 if (likely(!(INTEL_DEBUG & DEBUG_NO16) || brw->use_rep_send)) {
5175 if (!v.simd16_unsupported) {
5176 /* Try a SIMD16 compile */
5177 v2.import_uniforms(&v);
5178 if (!v2.run_fs(brw->use_rep_send)) {
5179 perf_debug("SIMD16 shader failed to compile: %s", v2.fail_msg);
5180 } else {
5181 simd16_cfg = v2.cfg;
5182 }
5183 }
5184 }
5185
5186 cfg_t *simd8_cfg;
5187 int no_simd8 = (INTEL_DEBUG & DEBUG_NO8) || brw->no_simd8;
5188 if ((no_simd8 || brw->gen < 5) && simd16_cfg) {
5189 simd8_cfg = NULL;
5190 prog_data->no_8 = true;
5191 } else {
5192 simd8_cfg = v.cfg;
5193 prog_data->no_8 = false;
5194 }
5195
5196 fs_generator g(brw->intelScreen->compiler, brw,
5197 mem_ctx, (void *) key, &prog_data->base,
5198 &fp->Base, v.promoted_constants, v.runtime_check_aads_emit, "FS");
5199
5200 if (unlikely(INTEL_DEBUG & DEBUG_WM)) {
5201 char *name;
5202 if (prog)
5203 name = ralloc_asprintf(mem_ctx, "%s fragment shader %d",
5204 prog->Label ? prog->Label : "unnamed",
5205 prog->Name);
5206 else
5207 name = ralloc_asprintf(mem_ctx, "fragment program %d", fp->Base.Id);
5208
5209 g.enable_debug(name);
5210 }
5211
5212 if (simd8_cfg)
5213 g.generate_code(simd8_cfg, 8);
5214 if (simd16_cfg)
5215 prog_data->prog_offset_16 = g.generate_code(simd16_cfg, 16);
5216
5217 if (unlikely(brw->perf_debug) && shader) {
5218 if (shader->compiled_once)
5219 brw_wm_debug_recompile(brw, prog, key);
5220 shader->compiled_once = true;
5221
5222 if (start_busy && !drm_intel_bo_busy(brw->batch.last_bo)) {
5223 perf_debug("FS compile took %.03f ms and stalled the GPU\n",
5224 (get_time() - start_time) * 1000);
5225 }
5226 }
5227
5228 return g.get_assembly(final_assembly_size);
5229 }
5230
5231 extern "C" bool
5232 brw_fs_precompile(struct gl_context *ctx,
5233 struct gl_shader_program *shader_prog,
5234 struct gl_program *prog)
5235 {
5236 struct brw_context *brw = brw_context(ctx);
5237 struct brw_wm_prog_key key;
5238
5239 struct gl_fragment_program *fp = (struct gl_fragment_program *) prog;
5240 struct brw_fragment_program *bfp = brw_fragment_program(fp);
5241 bool program_uses_dfdy = fp->UsesDFdy;
5242
5243 memset(&key, 0, sizeof(key));
5244
5245 if (brw->gen < 6) {
5246 if (fp->UsesKill)
5247 key.iz_lookup |= IZ_PS_KILL_ALPHATEST_BIT;
5248
5249 if (fp->Base.OutputsWritten & BITFIELD64_BIT(FRAG_RESULT_DEPTH))
5250 key.iz_lookup |= IZ_PS_COMPUTES_DEPTH_BIT;
5251
5252 /* Just assume depth testing. */
5253 key.iz_lookup |= IZ_DEPTH_TEST_ENABLE_BIT;
5254 key.iz_lookup |= IZ_DEPTH_WRITE_ENABLE_BIT;
5255 }
5256
5257 if (brw->gen < 6 || _mesa_bitcount_64(fp->Base.InputsRead &
5258 BRW_FS_VARYING_INPUT_MASK) > 16)
5259 key.input_slots_valid = fp->Base.InputsRead | VARYING_BIT_POS;
5260
5261 brw_setup_tex_for_precompile(brw, &key.tex, &fp->Base);
5262
5263 if (fp->Base.InputsRead & VARYING_BIT_POS) {
5264 key.drawable_height = ctx->DrawBuffer->Height;
5265 }
5266
5267 key.nr_color_regions = _mesa_bitcount_64(fp->Base.OutputsWritten &
5268 ~(BITFIELD64_BIT(FRAG_RESULT_DEPTH) |
5269 BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK)));
5270
5271 if ((fp->Base.InputsRead & VARYING_BIT_POS) || program_uses_dfdy) {
5272 key.render_to_fbo = _mesa_is_user_fbo(ctx->DrawBuffer) ||
5273 key.nr_color_regions > 1;
5274 }
5275
5276 key.program_string_id = bfp->id;
5277
5278 uint32_t old_prog_offset = brw->wm.base.prog_offset;
5279 struct brw_wm_prog_data *old_prog_data = brw->wm.prog_data;
5280
5281 bool success = brw_codegen_wm_prog(brw, shader_prog, bfp, &key);
5282
5283 brw->wm.base.prog_offset = old_prog_offset;
5284 brw->wm.prog_data = old_prog_data;
5285
5286 return success;
5287 }
5288
5289 void
5290 brw_setup_tex_for_precompile(struct brw_context *brw,
5291 struct brw_sampler_prog_key_data *tex,
5292 struct gl_program *prog)
5293 {
5294 const bool has_shader_channel_select = brw->is_haswell || brw->gen >= 8;
5295 unsigned sampler_count = _mesa_fls(prog->SamplersUsed);
5296 for (unsigned i = 0; i < sampler_count; i++) {
5297 if (!has_shader_channel_select && (prog->ShadowSamplers & (1 << i))) {
5298 /* Assume DEPTH_TEXTURE_MODE is the default: X, X, X, 1 */
5299 tex->swizzles[i] =
5300 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_ONE);
5301 } else {
5302 /* Color sampler: assume no swizzling. */
5303 tex->swizzles[i] = SWIZZLE_XYZW;
5304 }
5305 }
5306 }