8182bb307590700791bba9a8e8a7fc575b047060
[mesa.git] / src / intel / compiler / brw_fs.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file brw_fs.cpp
25 *
26 * This file drives the GLSL IR -> LIR translation, contains the
27 * optimizations on the LIR, and drives the generation of native code
28 * from the LIR.
29 */
30
31 #include "main/macros.h"
32 #include "brw_eu.h"
33 #include "brw_fs.h"
34 #include "brw_fs_live_variables.h"
35 #include "brw_nir.h"
36 #include "brw_vec4_gs_visitor.h"
37 #include "brw_cfg.h"
38 #include "brw_dead_control_flow.h"
39 #include "dev/gen_debug.h"
40 #include "compiler/glsl_types.h"
41 #include "compiler/nir/nir_builder.h"
42 #include "program/prog_parameter.h"
43 #include "util/u_math.h"
44
45 using namespace brw;
46
47 static unsigned get_lowered_simd_width(const struct gen_device_info *devinfo,
48 const fs_inst *inst);
49
50 void
51 fs_inst::init(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
52 const fs_reg *src, unsigned sources)
53 {
54 memset((void*)this, 0, sizeof(*this));
55
56 this->src = new fs_reg[MAX2(sources, 3)];
57 for (unsigned i = 0; i < sources; i++)
58 this->src[i] = src[i];
59
60 this->opcode = opcode;
61 this->dst = dst;
62 this->sources = sources;
63 this->exec_size = exec_size;
64 this->base_mrf = -1;
65
66 assert(dst.file != IMM && dst.file != UNIFORM);
67
68 assert(this->exec_size != 0);
69
70 this->conditional_mod = BRW_CONDITIONAL_NONE;
71
72 /* This will be the case for almost all instructions. */
73 switch (dst.file) {
74 case VGRF:
75 case ARF:
76 case FIXED_GRF:
77 case MRF:
78 case ATTR:
79 this->size_written = dst.component_size(exec_size);
80 break;
81 case BAD_FILE:
82 this->size_written = 0;
83 break;
84 case IMM:
85 case UNIFORM:
86 unreachable("Invalid destination register file");
87 }
88
89 this->writes_accumulator = false;
90 }
91
92 fs_inst::fs_inst()
93 {
94 init(BRW_OPCODE_NOP, 8, dst, NULL, 0);
95 }
96
97 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size)
98 {
99 init(opcode, exec_size, reg_undef, NULL, 0);
100 }
101
102 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst)
103 {
104 init(opcode, exec_size, dst, NULL, 0);
105 }
106
107 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
108 const fs_reg &src0)
109 {
110 const fs_reg src[1] = { src0 };
111 init(opcode, exec_size, dst, src, 1);
112 }
113
114 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
115 const fs_reg &src0, const fs_reg &src1)
116 {
117 const fs_reg src[2] = { src0, src1 };
118 init(opcode, exec_size, dst, src, 2);
119 }
120
121 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_size, const fs_reg &dst,
122 const fs_reg &src0, const fs_reg &src1, const fs_reg &src2)
123 {
124 const fs_reg src[3] = { src0, src1, src2 };
125 init(opcode, exec_size, dst, src, 3);
126 }
127
128 fs_inst::fs_inst(enum opcode opcode, uint8_t exec_width, const fs_reg &dst,
129 const fs_reg src[], unsigned sources)
130 {
131 init(opcode, exec_width, dst, src, sources);
132 }
133
134 fs_inst::fs_inst(const fs_inst &that)
135 {
136 memcpy((void*)this, &that, sizeof(that));
137
138 this->src = new fs_reg[MAX2(that.sources, 3)];
139
140 for (unsigned i = 0; i < that.sources; i++)
141 this->src[i] = that.src[i];
142 }
143
144 fs_inst::~fs_inst()
145 {
146 delete[] this->src;
147 }
148
149 void
150 fs_inst::resize_sources(uint8_t num_sources)
151 {
152 if (this->sources != num_sources) {
153 fs_reg *src = new fs_reg[MAX2(num_sources, 3)];
154
155 for (unsigned i = 0; i < MIN2(this->sources, num_sources); ++i)
156 src[i] = this->src[i];
157
158 delete[] this->src;
159 this->src = src;
160 this->sources = num_sources;
161 }
162 }
163
164 void
165 fs_visitor::VARYING_PULL_CONSTANT_LOAD(const fs_builder &bld,
166 const fs_reg &dst,
167 const fs_reg &surf_index,
168 const fs_reg &varying_offset,
169 uint32_t const_offset)
170 {
171 /* We have our constant surface use a pitch of 4 bytes, so our index can
172 * be any component of a vector, and then we load 4 contiguous
173 * components starting from that.
174 *
175 * We break down the const_offset to a portion added to the variable offset
176 * and a portion done using fs_reg::offset, which means that if you have
177 * GLSL using something like "uniform vec4 a[20]; gl_FragColor = a[i]",
178 * we'll temporarily generate 4 vec4 loads from offset i * 4, and CSE can
179 * later notice that those loads are all the same and eliminate the
180 * redundant ones.
181 */
182 fs_reg vec4_offset = vgrf(glsl_type::uint_type);
183 bld.ADD(vec4_offset, varying_offset, brw_imm_ud(const_offset & ~0xf));
184
185 /* The pull load message will load a vec4 (16 bytes). If we are loading
186 * a double this means we are only loading 2 elements worth of data.
187 * We also want to use a 32-bit data type for the dst of the load operation
188 * so other parts of the driver don't get confused about the size of the
189 * result.
190 */
191 fs_reg vec4_result = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
192 fs_inst *inst = bld.emit(FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL,
193 vec4_result, surf_index, vec4_offset);
194 inst->size_written = 4 * vec4_result.component_size(inst->exec_size);
195
196 shuffle_from_32bit_read(bld, dst, vec4_result,
197 (const_offset & 0xf) / type_sz(dst.type), 1);
198 }
199
200 /**
201 * A helper for MOV generation for fixing up broken hardware SEND dependency
202 * handling.
203 */
204 void
205 fs_visitor::DEP_RESOLVE_MOV(const fs_builder &bld, int grf)
206 {
207 /* The caller always wants uncompressed to emit the minimal extra
208 * dependencies, and to avoid having to deal with aligning its regs to 2.
209 */
210 const fs_builder ubld = bld.annotate("send dependency resolve")
211 .quarter(0);
212
213 ubld.MOV(ubld.null_reg_f(), fs_reg(VGRF, grf, BRW_REGISTER_TYPE_F));
214 }
215
216 bool
217 fs_inst::is_send_from_grf() const
218 {
219 switch (opcode) {
220 case SHADER_OPCODE_SEND:
221 case SHADER_OPCODE_SHADER_TIME_ADD:
222 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
223 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
224 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
225 case SHADER_OPCODE_URB_WRITE_SIMD8:
226 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
227 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
228 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
229 case SHADER_OPCODE_URB_READ_SIMD8:
230 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
231 case SHADER_OPCODE_INTERLOCK:
232 case SHADER_OPCODE_MEMORY_FENCE:
233 case SHADER_OPCODE_BARRIER:
234 return true;
235 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
236 return src[1].file == VGRF;
237 case FS_OPCODE_FB_WRITE:
238 case FS_OPCODE_FB_READ:
239 return src[0].file == VGRF;
240 default:
241 if (is_tex())
242 return src[0].file == VGRF;
243
244 return false;
245 }
246 }
247
248 bool
249 fs_inst::is_control_source(unsigned arg) const
250 {
251 switch (opcode) {
252 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
253 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
254 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
255 return arg == 0;
256
257 case SHADER_OPCODE_BROADCAST:
258 case SHADER_OPCODE_SHUFFLE:
259 case SHADER_OPCODE_QUAD_SWIZZLE:
260 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
261 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
262 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
263 case SHADER_OPCODE_GET_BUFFER_SIZE:
264 return arg == 1;
265
266 case SHADER_OPCODE_MOV_INDIRECT:
267 case SHADER_OPCODE_CLUSTER_BROADCAST:
268 case SHADER_OPCODE_TEX:
269 case FS_OPCODE_TXB:
270 case SHADER_OPCODE_TXD:
271 case SHADER_OPCODE_TXF:
272 case SHADER_OPCODE_TXF_LZ:
273 case SHADER_OPCODE_TXF_CMS:
274 case SHADER_OPCODE_TXF_CMS_W:
275 case SHADER_OPCODE_TXF_UMS:
276 case SHADER_OPCODE_TXF_MCS:
277 case SHADER_OPCODE_TXL:
278 case SHADER_OPCODE_TXL_LZ:
279 case SHADER_OPCODE_TXS:
280 case SHADER_OPCODE_LOD:
281 case SHADER_OPCODE_TG4:
282 case SHADER_OPCODE_TG4_OFFSET:
283 case SHADER_OPCODE_SAMPLEINFO:
284 return arg == 1 || arg == 2;
285
286 case SHADER_OPCODE_SEND:
287 return arg == 0 || arg == 1;
288
289 default:
290 return false;
291 }
292 }
293
294 bool
295 fs_inst::is_payload(unsigned arg) const
296 {
297 switch (opcode) {
298 case FS_OPCODE_FB_WRITE:
299 case FS_OPCODE_FB_READ:
300 case SHADER_OPCODE_URB_WRITE_SIMD8:
301 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
302 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
303 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
304 case SHADER_OPCODE_URB_READ_SIMD8:
305 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
306 case VEC4_OPCODE_UNTYPED_ATOMIC:
307 case VEC4_OPCODE_UNTYPED_SURFACE_READ:
308 case VEC4_OPCODE_UNTYPED_SURFACE_WRITE:
309 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
310 case SHADER_OPCODE_SHADER_TIME_ADD:
311 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
312 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
313 case SHADER_OPCODE_INTERLOCK:
314 case SHADER_OPCODE_MEMORY_FENCE:
315 case SHADER_OPCODE_BARRIER:
316 return arg == 0;
317
318 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
319 return arg == 1;
320
321 case SHADER_OPCODE_SEND:
322 return arg == 2 || arg == 3;
323
324 default:
325 if (is_tex())
326 return arg == 0;
327 else
328 return false;
329 }
330 }
331
332 /**
333 * Returns true if this instruction's sources and destinations cannot
334 * safely be the same register.
335 *
336 * In most cases, a register can be written over safely by the same
337 * instruction that is its last use. For a single instruction, the
338 * sources are dereferenced before writing of the destination starts
339 * (naturally).
340 *
341 * However, there are a few cases where this can be problematic:
342 *
343 * - Virtual opcodes that translate to multiple instructions in the
344 * code generator: if src == dst and one instruction writes the
345 * destination before a later instruction reads the source, then
346 * src will have been clobbered.
347 *
348 * - SIMD16 compressed instructions with certain regioning (see below).
349 *
350 * The register allocator uses this information to set up conflicts between
351 * GRF sources and the destination.
352 */
353 bool
354 fs_inst::has_source_and_destination_hazard() const
355 {
356 switch (opcode) {
357 case FS_OPCODE_PACK_HALF_2x16_SPLIT:
358 /* Multiple partial writes to the destination */
359 return true;
360 case SHADER_OPCODE_SHUFFLE:
361 /* This instruction returns an arbitrary channel from the source and
362 * gets split into smaller instructions in the generator. It's possible
363 * that one of the instructions will read from a channel corresponding
364 * to an earlier instruction.
365 */
366 case SHADER_OPCODE_SEL_EXEC:
367 /* This is implemented as
368 *
369 * mov(16) g4<1>D 0D { align1 WE_all 1H };
370 * mov(16) g4<1>D g5<8,8,1>D { align1 1H }
371 *
372 * Because the source is only read in the second instruction, the first
373 * may stomp all over it.
374 */
375 return true;
376 case SHADER_OPCODE_QUAD_SWIZZLE:
377 switch (src[1].ud) {
378 case BRW_SWIZZLE_XXXX:
379 case BRW_SWIZZLE_YYYY:
380 case BRW_SWIZZLE_ZZZZ:
381 case BRW_SWIZZLE_WWWW:
382 case BRW_SWIZZLE_XXZZ:
383 case BRW_SWIZZLE_YYWW:
384 case BRW_SWIZZLE_XYXY:
385 case BRW_SWIZZLE_ZWZW:
386 /* These can be implemented as a single Align1 region on all
387 * platforms, so there's never a hazard between source and
388 * destination. C.f. fs_generator::generate_quad_swizzle().
389 */
390 return false;
391 default:
392 return !is_uniform(src[0]);
393 }
394 default:
395 /* The SIMD16 compressed instruction
396 *
397 * add(16) g4<1>F g4<8,8,1>F g6<8,8,1>F
398 *
399 * is actually decoded in hardware as:
400 *
401 * add(8) g4<1>F g4<8,8,1>F g6<8,8,1>F
402 * add(8) g5<1>F g5<8,8,1>F g7<8,8,1>F
403 *
404 * Which is safe. However, if we have uniform accesses
405 * happening, we get into trouble:
406 *
407 * add(8) g4<1>F g4<0,1,0>F g6<8,8,1>F
408 * add(8) g5<1>F g4<0,1,0>F g7<8,8,1>F
409 *
410 * Now our destination for the first instruction overwrote the
411 * second instruction's src0, and we get garbage for those 8
412 * pixels. There's a similar issue for the pre-gen6
413 * pixel_x/pixel_y, which are registers of 16-bit values and thus
414 * would get stomped by the first decode as well.
415 */
416 if (exec_size == 16) {
417 for (int i = 0; i < sources; i++) {
418 if (src[i].file == VGRF && (src[i].stride == 0 ||
419 src[i].type == BRW_REGISTER_TYPE_UW ||
420 src[i].type == BRW_REGISTER_TYPE_W ||
421 src[i].type == BRW_REGISTER_TYPE_UB ||
422 src[i].type == BRW_REGISTER_TYPE_B)) {
423 return true;
424 }
425 }
426 }
427 return false;
428 }
429 }
430
431 bool
432 fs_inst::can_do_source_mods(const struct gen_device_info *devinfo) const
433 {
434 if (devinfo->gen == 6 && is_math())
435 return false;
436
437 if (is_send_from_grf())
438 return false;
439
440 /* From GEN:BUG:1604601757:
441 *
442 * "When multiplying a DW and any lower precision integer, source modifier
443 * is not supported."
444 */
445 if (devinfo->gen >= 12 && (opcode == BRW_OPCODE_MUL ||
446 opcode == BRW_OPCODE_MAD)) {
447 const brw_reg_type exec_type = get_exec_type(this);
448 const unsigned min_type_sz = opcode == BRW_OPCODE_MAD ?
449 MIN2(type_sz(src[1].type), type_sz(src[2].type)) :
450 MIN2(type_sz(src[0].type), type_sz(src[1].type));
451
452 if (brw_reg_type_is_integer(exec_type) &&
453 type_sz(exec_type) >= 4 &&
454 type_sz(exec_type) != min_type_sz)
455 return false;
456 }
457
458 if (!backend_instruction::can_do_source_mods())
459 return false;
460
461 return true;
462 }
463
464 bool
465 fs_inst::can_do_cmod()
466 {
467 if (!backend_instruction::can_do_cmod())
468 return false;
469
470 /* The accumulator result appears to get used for the conditional modifier
471 * generation. When negating a UD value, there is a 33rd bit generated for
472 * the sign in the accumulator value, so now you can't check, for example,
473 * equality with a 32-bit value. See piglit fs-op-neg-uvec4.
474 */
475 for (unsigned i = 0; i < sources; i++) {
476 if (type_is_unsigned_int(src[i].type) && src[i].negate)
477 return false;
478 }
479
480 return true;
481 }
482
483 bool
484 fs_inst::can_change_types() const
485 {
486 return dst.type == src[0].type &&
487 !src[0].abs && !src[0].negate && !saturate &&
488 (opcode == BRW_OPCODE_MOV ||
489 (opcode == BRW_OPCODE_SEL &&
490 dst.type == src[1].type &&
491 predicate != BRW_PREDICATE_NONE &&
492 !src[1].abs && !src[1].negate));
493 }
494
495 void
496 fs_reg::init()
497 {
498 memset((void*)this, 0, sizeof(*this));
499 type = BRW_REGISTER_TYPE_UD;
500 stride = 1;
501 }
502
503 /** Generic unset register constructor. */
504 fs_reg::fs_reg()
505 {
506 init();
507 this->file = BAD_FILE;
508 }
509
510 fs_reg::fs_reg(struct ::brw_reg reg) :
511 backend_reg(reg)
512 {
513 this->offset = 0;
514 this->stride = 1;
515 if (this->file == IMM &&
516 (this->type != BRW_REGISTER_TYPE_V &&
517 this->type != BRW_REGISTER_TYPE_UV &&
518 this->type != BRW_REGISTER_TYPE_VF)) {
519 this->stride = 0;
520 }
521 }
522
523 bool
524 fs_reg::equals(const fs_reg &r) const
525 {
526 return (this->backend_reg::equals(r) &&
527 stride == r.stride);
528 }
529
530 bool
531 fs_reg::negative_equals(const fs_reg &r) const
532 {
533 return (this->backend_reg::negative_equals(r) &&
534 stride == r.stride);
535 }
536
537 bool
538 fs_reg::is_contiguous() const
539 {
540 switch (file) {
541 case ARF:
542 case FIXED_GRF:
543 return hstride == BRW_HORIZONTAL_STRIDE_1 &&
544 vstride == width + hstride;
545 case MRF:
546 case VGRF:
547 case ATTR:
548 return stride == 1;
549 case UNIFORM:
550 case IMM:
551 case BAD_FILE:
552 return true;
553 }
554
555 unreachable("Invalid register file");
556 }
557
558 unsigned
559 fs_reg::component_size(unsigned width) const
560 {
561 const unsigned stride = ((file != ARF && file != FIXED_GRF) ? this->stride :
562 hstride == 0 ? 0 :
563 1 << (hstride - 1));
564 return MAX2(width * stride, 1) * type_sz(type);
565 }
566
567 /**
568 * Create a MOV to read the timestamp register.
569 */
570 fs_reg
571 fs_visitor::get_timestamp(const fs_builder &bld)
572 {
573 assert(devinfo->gen >= 7);
574
575 fs_reg ts = fs_reg(retype(brw_vec4_reg(BRW_ARCHITECTURE_REGISTER_FILE,
576 BRW_ARF_TIMESTAMP,
577 0),
578 BRW_REGISTER_TYPE_UD));
579
580 fs_reg dst = fs_reg(VGRF, alloc.allocate(1), BRW_REGISTER_TYPE_UD);
581
582 /* We want to read the 3 fields we care about even if it's not enabled in
583 * the dispatch.
584 */
585 bld.group(4, 0).exec_all().MOV(dst, ts);
586
587 return dst;
588 }
589
590 void
591 fs_visitor::emit_shader_time_begin()
592 {
593 /* We want only the low 32 bits of the timestamp. Since it's running
594 * at the GPU clock rate of ~1.2ghz, it will roll over every ~3 seconds,
595 * which is plenty of time for our purposes. It is identical across the
596 * EUs, but since it's tracking GPU core speed it will increment at a
597 * varying rate as render P-states change.
598 */
599 shader_start_time = component(
600 get_timestamp(bld.annotate("shader time start")), 0);
601 }
602
603 void
604 fs_visitor::emit_shader_time_end()
605 {
606 /* Insert our code just before the final SEND with EOT. */
607 exec_node *end = this->instructions.get_tail();
608 assert(end && ((fs_inst *) end)->eot);
609 const fs_builder ibld = bld.annotate("shader time end")
610 .exec_all().at(NULL, end);
611 const fs_reg timestamp = get_timestamp(ibld);
612
613 /* We only use the low 32 bits of the timestamp - see
614 * emit_shader_time_begin()).
615 *
616 * We could also check if render P-states have changed (or anything
617 * else that might disrupt timing) by setting smear to 2 and checking if
618 * that field is != 0.
619 */
620 const fs_reg shader_end_time = component(timestamp, 0);
621
622 /* Check that there weren't any timestamp reset events (assuming these
623 * were the only two timestamp reads that happened).
624 */
625 const fs_reg reset = component(timestamp, 2);
626 set_condmod(BRW_CONDITIONAL_Z,
627 ibld.AND(ibld.null_reg_ud(), reset, brw_imm_ud(1u)));
628 ibld.IF(BRW_PREDICATE_NORMAL);
629
630 fs_reg start = shader_start_time;
631 start.negate = true;
632 const fs_reg diff = component(fs_reg(VGRF, alloc.allocate(1),
633 BRW_REGISTER_TYPE_UD),
634 0);
635 const fs_builder cbld = ibld.group(1, 0);
636 cbld.group(1, 0).ADD(diff, start, shader_end_time);
637
638 /* If there were no instructions between the two timestamp gets, the diff
639 * is 2 cycles. Remove that overhead, so I can forget about that when
640 * trying to determine the time taken for single instructions.
641 */
642 cbld.ADD(diff, diff, brw_imm_ud(-2u));
643 SHADER_TIME_ADD(cbld, 0, diff);
644 SHADER_TIME_ADD(cbld, 1, brw_imm_ud(1u));
645 ibld.emit(BRW_OPCODE_ELSE);
646 SHADER_TIME_ADD(cbld, 2, brw_imm_ud(1u));
647 ibld.emit(BRW_OPCODE_ENDIF);
648 }
649
650 void
651 fs_visitor::SHADER_TIME_ADD(const fs_builder &bld,
652 int shader_time_subindex,
653 fs_reg value)
654 {
655 int index = shader_time_index * 3 + shader_time_subindex;
656 struct brw_reg offset = brw_imm_d(index * BRW_SHADER_TIME_STRIDE);
657
658 fs_reg payload;
659 if (dispatch_width == 8)
660 payload = vgrf(glsl_type::uvec2_type);
661 else
662 payload = vgrf(glsl_type::uint_type);
663
664 bld.emit(SHADER_OPCODE_SHADER_TIME_ADD, fs_reg(), payload, offset, value);
665 }
666
667 void
668 fs_visitor::vfail(const char *format, va_list va)
669 {
670 char *msg;
671
672 if (failed)
673 return;
674
675 failed = true;
676
677 msg = ralloc_vasprintf(mem_ctx, format, va);
678 msg = ralloc_asprintf(mem_ctx, "%s compile failed: %s\n", stage_abbrev, msg);
679
680 this->fail_msg = msg;
681
682 if (debug_enabled) {
683 fprintf(stderr, "%s", msg);
684 }
685 }
686
687 void
688 fs_visitor::fail(const char *format, ...)
689 {
690 va_list va;
691
692 va_start(va, format);
693 vfail(format, va);
694 va_end(va);
695 }
696
697 /**
698 * Mark this program as impossible to compile with dispatch width greater
699 * than n.
700 *
701 * During the SIMD8 compile (which happens first), we can detect and flag
702 * things that are unsupported in SIMD16+ mode, so the compiler can skip the
703 * SIMD16+ compile altogether.
704 *
705 * During a compile of dispatch width greater than n (if one happens anyway),
706 * this just calls fail().
707 */
708 void
709 fs_visitor::limit_dispatch_width(unsigned n, const char *msg)
710 {
711 if (dispatch_width > n) {
712 fail("%s", msg);
713 } else {
714 max_dispatch_width = n;
715 compiler->shader_perf_log(log_data,
716 "Shader dispatch width limited to SIMD%d: %s",
717 n, msg);
718 }
719 }
720
721 /**
722 * Returns true if the instruction has a flag that means it won't
723 * update an entire destination register.
724 *
725 * For example, dead code elimination and live variable analysis want to know
726 * when a write to a variable screens off any preceding values that were in
727 * it.
728 */
729 bool
730 fs_inst::is_partial_write() const
731 {
732 return ((this->predicate && this->opcode != BRW_OPCODE_SEL) ||
733 (this->exec_size * type_sz(this->dst.type)) < 32 ||
734 !this->dst.is_contiguous() ||
735 this->dst.offset % REG_SIZE != 0);
736 }
737
738 unsigned
739 fs_inst::components_read(unsigned i) const
740 {
741 /* Return zero if the source is not present. */
742 if (src[i].file == BAD_FILE)
743 return 0;
744
745 switch (opcode) {
746 case FS_OPCODE_LINTERP:
747 if (i == 0)
748 return 2;
749 else
750 return 1;
751
752 case FS_OPCODE_PIXEL_X:
753 case FS_OPCODE_PIXEL_Y:
754 assert(i == 0);
755 return 2;
756
757 case FS_OPCODE_FB_WRITE_LOGICAL:
758 assert(src[FB_WRITE_LOGICAL_SRC_COMPONENTS].file == IMM);
759 /* First/second FB write color. */
760 if (i < 2)
761 return src[FB_WRITE_LOGICAL_SRC_COMPONENTS].ud;
762 else
763 return 1;
764
765 case SHADER_OPCODE_TEX_LOGICAL:
766 case SHADER_OPCODE_TXD_LOGICAL:
767 case SHADER_OPCODE_TXF_LOGICAL:
768 case SHADER_OPCODE_TXL_LOGICAL:
769 case SHADER_OPCODE_TXS_LOGICAL:
770 case SHADER_OPCODE_IMAGE_SIZE_LOGICAL:
771 case FS_OPCODE_TXB_LOGICAL:
772 case SHADER_OPCODE_TXF_CMS_LOGICAL:
773 case SHADER_OPCODE_TXF_CMS_W_LOGICAL:
774 case SHADER_OPCODE_TXF_UMS_LOGICAL:
775 case SHADER_OPCODE_TXF_MCS_LOGICAL:
776 case SHADER_OPCODE_LOD_LOGICAL:
777 case SHADER_OPCODE_TG4_LOGICAL:
778 case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
779 case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
780 assert(src[TEX_LOGICAL_SRC_COORD_COMPONENTS].file == IMM &&
781 src[TEX_LOGICAL_SRC_GRAD_COMPONENTS].file == IMM);
782 /* Texture coordinates. */
783 if (i == TEX_LOGICAL_SRC_COORDINATE)
784 return src[TEX_LOGICAL_SRC_COORD_COMPONENTS].ud;
785 /* Texture derivatives. */
786 else if ((i == TEX_LOGICAL_SRC_LOD || i == TEX_LOGICAL_SRC_LOD2) &&
787 opcode == SHADER_OPCODE_TXD_LOGICAL)
788 return src[TEX_LOGICAL_SRC_GRAD_COMPONENTS].ud;
789 /* Texture offset. */
790 else if (i == TEX_LOGICAL_SRC_TG4_OFFSET)
791 return 2;
792 /* MCS */
793 else if (i == TEX_LOGICAL_SRC_MCS && opcode == SHADER_OPCODE_TXF_CMS_W_LOGICAL)
794 return 2;
795 else
796 return 1;
797
798 case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
799 case SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL:
800 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM);
801 /* Surface coordinates. */
802 if (i == SURFACE_LOGICAL_SRC_ADDRESS)
803 return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud;
804 /* Surface operation source (ignored for reads). */
805 else if (i == SURFACE_LOGICAL_SRC_DATA)
806 return 0;
807 else
808 return 1;
809
810 case SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL:
811 case SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL:
812 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM &&
813 src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM);
814 /* Surface coordinates. */
815 if (i == SURFACE_LOGICAL_SRC_ADDRESS)
816 return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud;
817 /* Surface operation source. */
818 else if (i == SURFACE_LOGICAL_SRC_DATA)
819 return src[SURFACE_LOGICAL_SRC_IMM_ARG].ud;
820 else
821 return 1;
822
823 case SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL:
824 assert(src[2].file == IMM);
825 return 1;
826
827 case SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL:
828 assert(src[2].file == IMM);
829 return i == 1 ? src[2].ud : 1;
830
831 case SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL:
832 case SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL:
833 assert(src[2].file == IMM);
834 if (i == 1) {
835 /* Data source */
836 const unsigned op = src[2].ud;
837 switch (op) {
838 case BRW_AOP_INC:
839 case BRW_AOP_DEC:
840 case BRW_AOP_PREDEC:
841 return 0;
842 case BRW_AOP_CMPWR:
843 return 2;
844 default:
845 return 1;
846 }
847 } else {
848 return 1;
849 }
850
851 case SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT_LOGICAL:
852 assert(src[2].file == IMM);
853 if (i == 1) {
854 /* Data source */
855 const unsigned op = src[2].ud;
856 return op == BRW_AOP_FCMPWR ? 2 : 1;
857 } else {
858 return 1;
859 }
860
861 case SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL:
862 case SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL:
863 /* Scattered logical opcodes use the following params:
864 * src[0] Surface coordinates
865 * src[1] Surface operation source (ignored for reads)
866 * src[2] Surface
867 * src[3] IMM with always 1 dimension.
868 * src[4] IMM with arg bitsize for scattered read/write 8, 16, 32
869 */
870 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM &&
871 src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM);
872 return i == SURFACE_LOGICAL_SRC_DATA ? 0 : 1;
873
874 case SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL:
875 case SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL:
876 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM &&
877 src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM);
878 return 1;
879
880 case SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL:
881 case SHADER_OPCODE_TYPED_ATOMIC_LOGICAL: {
882 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM &&
883 src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM);
884 const unsigned op = src[SURFACE_LOGICAL_SRC_IMM_ARG].ud;
885 /* Surface coordinates. */
886 if (i == SURFACE_LOGICAL_SRC_ADDRESS)
887 return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud;
888 /* Surface operation source. */
889 else if (i == SURFACE_LOGICAL_SRC_DATA && op == BRW_AOP_CMPWR)
890 return 2;
891 else if (i == SURFACE_LOGICAL_SRC_DATA &&
892 (op == BRW_AOP_INC || op == BRW_AOP_DEC || op == BRW_AOP_PREDEC))
893 return 0;
894 else
895 return 1;
896 }
897 case FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET:
898 return (i == 0 ? 2 : 1);
899
900 case SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL: {
901 assert(src[SURFACE_LOGICAL_SRC_IMM_DIMS].file == IMM &&
902 src[SURFACE_LOGICAL_SRC_IMM_ARG].file == IMM);
903 const unsigned op = src[SURFACE_LOGICAL_SRC_IMM_ARG].ud;
904 /* Surface coordinates. */
905 if (i == SURFACE_LOGICAL_SRC_ADDRESS)
906 return src[SURFACE_LOGICAL_SRC_IMM_DIMS].ud;
907 /* Surface operation source. */
908 else if (i == SURFACE_LOGICAL_SRC_DATA && op == BRW_AOP_FCMPWR)
909 return 2;
910 else
911 return 1;
912 }
913
914 default:
915 return 1;
916 }
917 }
918
919 unsigned
920 fs_inst::size_read(int arg) const
921 {
922 switch (opcode) {
923 case SHADER_OPCODE_SEND:
924 if (arg == 2) {
925 return mlen * REG_SIZE;
926 } else if (arg == 3) {
927 return ex_mlen * REG_SIZE;
928 }
929 break;
930
931 case FS_OPCODE_FB_WRITE:
932 case FS_OPCODE_REP_FB_WRITE:
933 if (arg == 0) {
934 if (base_mrf >= 0)
935 return src[0].file == BAD_FILE ? 0 : 2 * REG_SIZE;
936 else
937 return mlen * REG_SIZE;
938 }
939 break;
940
941 case FS_OPCODE_FB_READ:
942 case SHADER_OPCODE_URB_WRITE_SIMD8:
943 case SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT:
944 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED:
945 case SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT:
946 case SHADER_OPCODE_URB_READ_SIMD8:
947 case SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT:
948 case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
949 case FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET:
950 if (arg == 0)
951 return mlen * REG_SIZE;
952 break;
953
954 case FS_OPCODE_SET_SAMPLE_ID:
955 if (arg == 1)
956 return 1;
957 break;
958
959 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD_GEN7:
960 /* The payload is actually stored in src1 */
961 if (arg == 1)
962 return mlen * REG_SIZE;
963 break;
964
965 case FS_OPCODE_LINTERP:
966 if (arg == 1)
967 return 16;
968 break;
969
970 case SHADER_OPCODE_LOAD_PAYLOAD:
971 if (arg < this->header_size)
972 return REG_SIZE;
973 break;
974
975 case CS_OPCODE_CS_TERMINATE:
976 case SHADER_OPCODE_BARRIER:
977 return REG_SIZE;
978
979 case SHADER_OPCODE_MOV_INDIRECT:
980 if (arg == 0) {
981 assert(src[2].file == IMM);
982 return src[2].ud;
983 }
984 break;
985
986 default:
987 if (is_tex() && arg == 0 && src[0].file == VGRF)
988 return mlen * REG_SIZE;
989 break;
990 }
991
992 switch (src[arg].file) {
993 case UNIFORM:
994 case IMM:
995 return components_read(arg) * type_sz(src[arg].type);
996 case BAD_FILE:
997 case ARF:
998 case FIXED_GRF:
999 case VGRF:
1000 case ATTR:
1001 return components_read(arg) * src[arg].component_size(exec_size);
1002 case MRF:
1003 unreachable("MRF registers are not allowed as sources");
1004 }
1005 return 0;
1006 }
1007
1008 namespace {
1009 unsigned
1010 predicate_width(brw_predicate predicate)
1011 {
1012 switch (predicate) {
1013 case BRW_PREDICATE_NONE: return 1;
1014 case BRW_PREDICATE_NORMAL: return 1;
1015 case BRW_PREDICATE_ALIGN1_ANY2H: return 2;
1016 case BRW_PREDICATE_ALIGN1_ALL2H: return 2;
1017 case BRW_PREDICATE_ALIGN1_ANY4H: return 4;
1018 case BRW_PREDICATE_ALIGN1_ALL4H: return 4;
1019 case BRW_PREDICATE_ALIGN1_ANY8H: return 8;
1020 case BRW_PREDICATE_ALIGN1_ALL8H: return 8;
1021 case BRW_PREDICATE_ALIGN1_ANY16H: return 16;
1022 case BRW_PREDICATE_ALIGN1_ALL16H: return 16;
1023 case BRW_PREDICATE_ALIGN1_ANY32H: return 32;
1024 case BRW_PREDICATE_ALIGN1_ALL32H: return 32;
1025 default: unreachable("Unsupported predicate");
1026 }
1027 }
1028
1029 /* Return the subset of flag registers that an instruction could
1030 * potentially read or write based on the execution controls and flag
1031 * subregister number of the instruction.
1032 */
1033 unsigned
1034 flag_mask(const fs_inst *inst, unsigned width)
1035 {
1036 assert(util_is_power_of_two_nonzero(width));
1037 const unsigned start = (inst->flag_subreg * 16 + inst->group) &
1038 ~(width - 1);
1039 const unsigned end = start + ALIGN(inst->exec_size, width);
1040 return ((1 << DIV_ROUND_UP(end, 8)) - 1) & ~((1 << (start / 8)) - 1);
1041 }
1042
1043 unsigned
1044 bit_mask(unsigned n)
1045 {
1046 return (n >= CHAR_BIT * sizeof(bit_mask(n)) ? ~0u : (1u << n) - 1);
1047 }
1048
1049 unsigned
1050 flag_mask(const fs_reg &r, unsigned sz)
1051 {
1052 if (r.file == ARF) {
1053 const unsigned start = (r.nr - BRW_ARF_FLAG) * 4 + r.subnr;
1054 const unsigned end = start + sz;
1055 return bit_mask(end) & ~bit_mask(start);
1056 } else {
1057 return 0;
1058 }
1059 }
1060 }
1061
1062 unsigned
1063 fs_inst::flags_read(const gen_device_info *devinfo) const
1064 {
1065 if (predicate == BRW_PREDICATE_ALIGN1_ANYV ||
1066 predicate == BRW_PREDICATE_ALIGN1_ALLV) {
1067 /* The vertical predication modes combine corresponding bits from
1068 * f0.0 and f1.0 on Gen7+, and f0.0 and f0.1 on older hardware.
1069 */
1070 const unsigned shift = devinfo->gen >= 7 ? 4 : 2;
1071 return flag_mask(this, 1) << shift | flag_mask(this, 1);
1072 } else if (predicate) {
1073 return flag_mask(this, predicate_width(predicate));
1074 } else {
1075 unsigned mask = 0;
1076 for (int i = 0; i < sources; i++) {
1077 mask |= flag_mask(src[i], size_read(i));
1078 }
1079 return mask;
1080 }
1081 }
1082
1083 unsigned
1084 fs_inst::flags_written() const
1085 {
1086 if ((conditional_mod && (opcode != BRW_OPCODE_SEL &&
1087 opcode != BRW_OPCODE_CSEL &&
1088 opcode != BRW_OPCODE_IF &&
1089 opcode != BRW_OPCODE_WHILE)) ||
1090 opcode == FS_OPCODE_FB_WRITE) {
1091 return flag_mask(this, 1);
1092 } else if (opcode == SHADER_OPCODE_FIND_LIVE_CHANNEL ||
1093 opcode == FS_OPCODE_LOAD_LIVE_CHANNELS) {
1094 return flag_mask(this, 32);
1095 } else {
1096 return flag_mask(dst, size_written);
1097 }
1098 }
1099
1100 /**
1101 * Returns how many MRFs an FS opcode will write over.
1102 *
1103 * Note that this is not the 0 or 1 implied writes in an actual gen
1104 * instruction -- the FS opcodes often generate MOVs in addition.
1105 */
1106 unsigned
1107 fs_inst::implied_mrf_writes() const
1108 {
1109 if (mlen == 0)
1110 return 0;
1111
1112 if (base_mrf == -1)
1113 return 0;
1114
1115 switch (opcode) {
1116 case SHADER_OPCODE_RCP:
1117 case SHADER_OPCODE_RSQ:
1118 case SHADER_OPCODE_SQRT:
1119 case SHADER_OPCODE_EXP2:
1120 case SHADER_OPCODE_LOG2:
1121 case SHADER_OPCODE_SIN:
1122 case SHADER_OPCODE_COS:
1123 return 1 * exec_size / 8;
1124 case SHADER_OPCODE_POW:
1125 case SHADER_OPCODE_INT_QUOTIENT:
1126 case SHADER_OPCODE_INT_REMAINDER:
1127 return 2 * exec_size / 8;
1128 case SHADER_OPCODE_TEX:
1129 case FS_OPCODE_TXB:
1130 case SHADER_OPCODE_TXD:
1131 case SHADER_OPCODE_TXF:
1132 case SHADER_OPCODE_TXF_CMS:
1133 case SHADER_OPCODE_TXF_MCS:
1134 case SHADER_OPCODE_TG4:
1135 case SHADER_OPCODE_TG4_OFFSET:
1136 case SHADER_OPCODE_TXL:
1137 case SHADER_OPCODE_TXS:
1138 case SHADER_OPCODE_LOD:
1139 case SHADER_OPCODE_SAMPLEINFO:
1140 return 1;
1141 case FS_OPCODE_FB_WRITE:
1142 case FS_OPCODE_REP_FB_WRITE:
1143 return src[0].file == BAD_FILE ? 0 : 2;
1144 case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
1145 case SHADER_OPCODE_GEN4_SCRATCH_READ:
1146 return 1;
1147 case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
1148 return mlen;
1149 case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
1150 return mlen;
1151 default:
1152 unreachable("not reached");
1153 }
1154 }
1155
1156 fs_reg
1157 fs_visitor::vgrf(const glsl_type *const type)
1158 {
1159 int reg_width = dispatch_width / 8;
1160 return fs_reg(VGRF,
1161 alloc.allocate(glsl_count_dword_slots(type, false) * reg_width),
1162 brw_type_for_base_type(type));
1163 }
1164
1165 fs_reg::fs_reg(enum brw_reg_file file, int nr)
1166 {
1167 init();
1168 this->file = file;
1169 this->nr = nr;
1170 this->type = BRW_REGISTER_TYPE_F;
1171 this->stride = (file == UNIFORM ? 0 : 1);
1172 }
1173
1174 fs_reg::fs_reg(enum brw_reg_file file, int nr, enum brw_reg_type type)
1175 {
1176 init();
1177 this->file = file;
1178 this->nr = nr;
1179 this->type = type;
1180 this->stride = (file == UNIFORM ? 0 : 1);
1181 }
1182
1183 /* For SIMD16, we need to follow from the uniform setup of SIMD8 dispatch.
1184 * This brings in those uniform definitions
1185 */
1186 void
1187 fs_visitor::import_uniforms(fs_visitor *v)
1188 {
1189 this->push_constant_loc = v->push_constant_loc;
1190 this->pull_constant_loc = v->pull_constant_loc;
1191 this->uniforms = v->uniforms;
1192 this->subgroup_id = v->subgroup_id;
1193 for (unsigned i = 0; i < ARRAY_SIZE(this->group_size); i++)
1194 this->group_size[i] = v->group_size[i];
1195 }
1196
1197 void
1198 fs_visitor::emit_fragcoord_interpolation(fs_reg wpos)
1199 {
1200 assert(stage == MESA_SHADER_FRAGMENT);
1201
1202 /* gl_FragCoord.x */
1203 bld.MOV(wpos, this->pixel_x);
1204 wpos = offset(wpos, bld, 1);
1205
1206 /* gl_FragCoord.y */
1207 bld.MOV(wpos, this->pixel_y);
1208 wpos = offset(wpos, bld, 1);
1209
1210 /* gl_FragCoord.z */
1211 if (devinfo->gen >= 6) {
1212 bld.MOV(wpos, fetch_payload_reg(bld, payload.source_depth_reg));
1213 } else {
1214 bld.emit(FS_OPCODE_LINTERP, wpos,
1215 this->delta_xy[BRW_BARYCENTRIC_PERSPECTIVE_PIXEL],
1216 component(interp_reg(VARYING_SLOT_POS, 2), 0));
1217 }
1218 wpos = offset(wpos, bld, 1);
1219
1220 /* gl_FragCoord.w: Already set up in emit_interpolation */
1221 bld.MOV(wpos, this->wpos_w);
1222 }
1223
1224 enum brw_barycentric_mode
1225 brw_barycentric_mode(enum glsl_interp_mode mode, nir_intrinsic_op op)
1226 {
1227 /* Barycentric modes don't make sense for flat inputs. */
1228 assert(mode != INTERP_MODE_FLAT);
1229
1230 unsigned bary;
1231 switch (op) {
1232 case nir_intrinsic_load_barycentric_pixel:
1233 case nir_intrinsic_load_barycentric_at_offset:
1234 bary = BRW_BARYCENTRIC_PERSPECTIVE_PIXEL;
1235 break;
1236 case nir_intrinsic_load_barycentric_centroid:
1237 bary = BRW_BARYCENTRIC_PERSPECTIVE_CENTROID;
1238 break;
1239 case nir_intrinsic_load_barycentric_sample:
1240 case nir_intrinsic_load_barycentric_at_sample:
1241 bary = BRW_BARYCENTRIC_PERSPECTIVE_SAMPLE;
1242 break;
1243 default:
1244 unreachable("invalid intrinsic");
1245 }
1246
1247 if (mode == INTERP_MODE_NOPERSPECTIVE)
1248 bary += 3;
1249
1250 return (enum brw_barycentric_mode) bary;
1251 }
1252
1253 /**
1254 * Turn one of the two CENTROID barycentric modes into PIXEL mode.
1255 */
1256 static enum brw_barycentric_mode
1257 centroid_to_pixel(enum brw_barycentric_mode bary)
1258 {
1259 assert(bary == BRW_BARYCENTRIC_PERSPECTIVE_CENTROID ||
1260 bary == BRW_BARYCENTRIC_NONPERSPECTIVE_CENTROID);
1261 return (enum brw_barycentric_mode) ((unsigned) bary - 1);
1262 }
1263
1264 fs_reg *
1265 fs_visitor::emit_frontfacing_interpolation()
1266 {
1267 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::bool_type));
1268
1269 if (devinfo->gen >= 12) {
1270 fs_reg g1 = fs_reg(retype(brw_vec1_grf(1, 1), BRW_REGISTER_TYPE_W));
1271
1272 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_W);
1273 bld.ASR(tmp, g1, brw_imm_d(15));
1274 bld.NOT(*reg, tmp);
1275 } else if (devinfo->gen >= 6) {
1276 /* Bit 15 of g0.0 is 0 if the polygon is front facing. We want to create
1277 * a boolean result from this (~0/true or 0/false).
1278 *
1279 * We can use the fact that bit 15 is the MSB of g0.0:W to accomplish
1280 * this task in only one instruction:
1281 * - a negation source modifier will flip the bit; and
1282 * - a W -> D type conversion will sign extend the bit into the high
1283 * word of the destination.
1284 *
1285 * An ASR 15 fills the low word of the destination.
1286 */
1287 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
1288 g0.negate = true;
1289
1290 bld.ASR(*reg, g0, brw_imm_d(15));
1291 } else {
1292 /* Bit 31 of g1.6 is 0 if the polygon is front facing. We want to create
1293 * a boolean result from this (1/true or 0/false).
1294 *
1295 * Like in the above case, since the bit is the MSB of g1.6:UD we can use
1296 * the negation source modifier to flip it. Unfortunately the SHR
1297 * instruction only operates on UD (or D with an abs source modifier)
1298 * sources without negation.
1299 *
1300 * Instead, use ASR (which will give ~0/true or 0/false).
1301 */
1302 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
1303 g1_6.negate = true;
1304
1305 bld.ASR(*reg, g1_6, brw_imm_d(31));
1306 }
1307
1308 return reg;
1309 }
1310
1311 void
1312 fs_visitor::compute_sample_position(fs_reg dst, fs_reg int_sample_pos)
1313 {
1314 assert(stage == MESA_SHADER_FRAGMENT);
1315 struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(this->prog_data);
1316 assert(dst.type == BRW_REGISTER_TYPE_F);
1317
1318 if (wm_prog_data->persample_dispatch) {
1319 /* Convert int_sample_pos to floating point */
1320 bld.MOV(dst, int_sample_pos);
1321 /* Scale to the range [0, 1] */
1322 bld.MUL(dst, dst, brw_imm_f(1 / 16.0f));
1323 }
1324 else {
1325 /* From ARB_sample_shading specification:
1326 * "When rendering to a non-multisample buffer, or if multisample
1327 * rasterization is disabled, gl_SamplePosition will always be
1328 * (0.5, 0.5).
1329 */
1330 bld.MOV(dst, brw_imm_f(0.5f));
1331 }
1332 }
1333
1334 fs_reg *
1335 fs_visitor::emit_samplepos_setup()
1336 {
1337 assert(devinfo->gen >= 6);
1338
1339 const fs_builder abld = bld.annotate("compute sample position");
1340 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::vec2_type));
1341 fs_reg pos = *reg;
1342 fs_reg int_sample_x = vgrf(glsl_type::int_type);
1343 fs_reg int_sample_y = vgrf(glsl_type::int_type);
1344
1345 /* WM will be run in MSDISPMODE_PERSAMPLE. So, only one of SIMD8 or SIMD16
1346 * mode will be enabled.
1347 *
1348 * From the Ivy Bridge PRM, volume 2 part 1, page 344:
1349 * R31.1:0 Position Offset X/Y for Slot[3:0]
1350 * R31.3:2 Position Offset X/Y for Slot[7:4]
1351 * .....
1352 *
1353 * The X, Y sample positions come in as bytes in thread payload. So, read
1354 * the positions using vstride=16, width=8, hstride=2.
1355 */
1356 const fs_reg sample_pos_reg =
1357 fetch_payload_reg(abld, payload.sample_pos_reg, BRW_REGISTER_TYPE_W);
1358
1359 /* Compute gl_SamplePosition.x */
1360 abld.MOV(int_sample_x, subscript(sample_pos_reg, BRW_REGISTER_TYPE_B, 0));
1361 compute_sample_position(offset(pos, abld, 0), int_sample_x);
1362
1363 /* Compute gl_SamplePosition.y */
1364 abld.MOV(int_sample_y, subscript(sample_pos_reg, BRW_REGISTER_TYPE_B, 1));
1365 compute_sample_position(offset(pos, abld, 1), int_sample_y);
1366 return reg;
1367 }
1368
1369 fs_reg *
1370 fs_visitor::emit_sampleid_setup()
1371 {
1372 assert(stage == MESA_SHADER_FRAGMENT);
1373 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
1374 assert(devinfo->gen >= 6);
1375
1376 const fs_builder abld = bld.annotate("compute sample id");
1377 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::uint_type));
1378
1379 if (!key->multisample_fbo) {
1380 /* As per GL_ARB_sample_shading specification:
1381 * "When rendering to a non-multisample buffer, or if multisample
1382 * rasterization is disabled, gl_SampleID will always be zero."
1383 */
1384 abld.MOV(*reg, brw_imm_d(0));
1385 } else if (devinfo->gen >= 8) {
1386 /* Sample ID comes in as 4-bit numbers in g1.0:
1387 *
1388 * 15:12 Slot 3 SampleID (only used in SIMD16)
1389 * 11:8 Slot 2 SampleID (only used in SIMD16)
1390 * 7:4 Slot 1 SampleID
1391 * 3:0 Slot 0 SampleID
1392 *
1393 * Each slot corresponds to four channels, so we want to replicate each
1394 * half-byte value to 4 channels in a row:
1395 *
1396 * dst+0: .7 .6 .5 .4 .3 .2 .1 .0
1397 * 7:4 7:4 7:4 7:4 3:0 3:0 3:0 3:0
1398 *
1399 * dst+1: .7 .6 .5 .4 .3 .2 .1 .0 (if SIMD16)
1400 * 15:12 15:12 15:12 15:12 11:8 11:8 11:8 11:8
1401 *
1402 * First, we read g1.0 with a <1,8,0>UB region, causing the first 8
1403 * channels to read the first byte (7:0), and the second group of 8
1404 * channels to read the second byte (15:8). Then, we shift right by
1405 * a vector immediate of <4, 4, 4, 4, 0, 0, 0, 0>, moving the slot 1 / 3
1406 * values into place. Finally, we AND with 0xf to keep the low nibble.
1407 *
1408 * shr(16) tmp<1>W g1.0<1,8,0>B 0x44440000:V
1409 * and(16) dst<1>D tmp<8,8,1>W 0xf:W
1410 *
1411 * TODO: These payload bits exist on Gen7 too, but they appear to always
1412 * be zero, so this code fails to work. We should find out why.
1413 */
1414 const fs_reg tmp = abld.vgrf(BRW_REGISTER_TYPE_UW);
1415
1416 for (unsigned i = 0; i < DIV_ROUND_UP(dispatch_width, 16); i++) {
1417 const fs_builder hbld = abld.group(MIN2(16, dispatch_width), i);
1418 hbld.SHR(offset(tmp, hbld, i),
1419 stride(retype(brw_vec1_grf(1 + i, 0), BRW_REGISTER_TYPE_UB),
1420 1, 8, 0),
1421 brw_imm_v(0x44440000));
1422 }
1423
1424 abld.AND(*reg, tmp, brw_imm_w(0xf));
1425 } else {
1426 const fs_reg t1 = component(abld.vgrf(BRW_REGISTER_TYPE_UD), 0);
1427 const fs_reg t2 = abld.vgrf(BRW_REGISTER_TYPE_UW);
1428
1429 /* The PS will be run in MSDISPMODE_PERSAMPLE. For example with
1430 * 8x multisampling, subspan 0 will represent sample N (where N
1431 * is 0, 2, 4 or 6), subspan 1 will represent sample 1, 3, 5 or
1432 * 7. We can find the value of N by looking at R0.0 bits 7:6
1433 * ("Starting Sample Pair Index (SSPI)") and multiplying by two
1434 * (since samples are always delivered in pairs). That is, we
1435 * compute 2*((R0.0 & 0xc0) >> 6) == (R0.0 & 0xc0) >> 5. Then
1436 * we need to add N to the sequence (0, 0, 0, 0, 1, 1, 1, 1) in
1437 * case of SIMD8 and sequence (0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2,
1438 * 2, 3, 3, 3, 3) in case of SIMD16. We compute this sequence by
1439 * populating a temporary variable with the sequence (0, 1, 2, 3),
1440 * and then reading from it using vstride=1, width=4, hstride=0.
1441 * These computations hold good for 4x multisampling as well.
1442 *
1443 * For 2x MSAA and SIMD16, we want to use the sequence (0, 1, 0, 1):
1444 * the first four slots are sample 0 of subspan 0; the next four
1445 * are sample 1 of subspan 0; the third group is sample 0 of
1446 * subspan 1, and finally sample 1 of subspan 1.
1447 */
1448
1449 /* SKL+ has an extra bit for the Starting Sample Pair Index to
1450 * accomodate 16x MSAA.
1451 */
1452 abld.exec_all().group(1, 0)
1453 .AND(t1, fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)),
1454 brw_imm_ud(0xc0));
1455 abld.exec_all().group(1, 0).SHR(t1, t1, brw_imm_d(5));
1456
1457 /* This works for SIMD8-SIMD16. It also works for SIMD32 but only if we
1458 * can assume 4x MSAA. Disallow it on IVB+
1459 *
1460 * FINISHME: One day, we could come up with a way to do this that
1461 * actually works on gen7.
1462 */
1463 if (devinfo->gen >= 7)
1464 limit_dispatch_width(16, "gl_SampleId is unsupported in SIMD32 on gen7");
1465 abld.exec_all().group(8, 0).MOV(t2, brw_imm_v(0x32103210));
1466
1467 /* This special instruction takes care of setting vstride=1,
1468 * width=4, hstride=0 of t2 during an ADD instruction.
1469 */
1470 abld.emit(FS_OPCODE_SET_SAMPLE_ID, *reg, t1, t2);
1471 }
1472
1473 return reg;
1474 }
1475
1476 fs_reg *
1477 fs_visitor::emit_samplemaskin_setup()
1478 {
1479 assert(stage == MESA_SHADER_FRAGMENT);
1480 struct brw_wm_prog_data *wm_prog_data = brw_wm_prog_data(this->prog_data);
1481 assert(devinfo->gen >= 6);
1482
1483 fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::int_type));
1484
1485 fs_reg coverage_mask =
1486 fetch_payload_reg(bld, payload.sample_mask_in_reg, BRW_REGISTER_TYPE_D);
1487
1488 if (wm_prog_data->persample_dispatch) {
1489 /* gl_SampleMaskIn[] comes from two sources: the input coverage mask,
1490 * and a mask representing which sample is being processed by the
1491 * current shader invocation.
1492 *
1493 * From the OES_sample_variables specification:
1494 * "When per-sample shading is active due to the use of a fragment input
1495 * qualified by "sample" or due to the use of the gl_SampleID or
1496 * gl_SamplePosition variables, only the bit for the current sample is
1497 * set in gl_SampleMaskIn."
1498 */
1499 const fs_builder abld = bld.annotate("compute gl_SampleMaskIn");
1500
1501 if (nir_system_values[SYSTEM_VALUE_SAMPLE_ID].file == BAD_FILE)
1502 nir_system_values[SYSTEM_VALUE_SAMPLE_ID] = *emit_sampleid_setup();
1503
1504 fs_reg one = vgrf(glsl_type::int_type);
1505 fs_reg enabled_mask = vgrf(glsl_type::int_type);
1506 abld.MOV(one, brw_imm_d(1));
1507 abld.SHL(enabled_mask, one, nir_system_values[SYSTEM_VALUE_SAMPLE_ID]);
1508 abld.AND(*reg, enabled_mask, coverage_mask);
1509 } else {
1510 /* In per-pixel mode, the coverage mask is sufficient. */
1511 *reg = coverage_mask;
1512 }
1513 return reg;
1514 }
1515
1516 fs_reg
1517 fs_visitor::resolve_source_modifiers(const fs_reg &src)
1518 {
1519 if (!src.abs && !src.negate)
1520 return src;
1521
1522 fs_reg temp = bld.vgrf(src.type);
1523 bld.MOV(temp, src);
1524
1525 return temp;
1526 }
1527
1528 void
1529 fs_visitor::emit_discard_jump()
1530 {
1531 assert(brw_wm_prog_data(this->prog_data)->uses_kill);
1532
1533 /* For performance, after a discard, jump to the end of the
1534 * shader if all relevant channels have been discarded.
1535 */
1536 fs_inst *discard_jump = bld.emit(FS_OPCODE_DISCARD_JUMP);
1537 discard_jump->flag_subreg = sample_mask_flag_subreg(this);
1538
1539 discard_jump->predicate = BRW_PREDICATE_ALIGN1_ANY4H;
1540 discard_jump->predicate_inverse = true;
1541 }
1542
1543 void
1544 fs_visitor::emit_gs_thread_end()
1545 {
1546 assert(stage == MESA_SHADER_GEOMETRY);
1547
1548 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
1549
1550 if (gs_compile->control_data_header_size_bits > 0) {
1551 emit_gs_control_data_bits(this->final_gs_vertex_count);
1552 }
1553
1554 const fs_builder abld = bld.annotate("thread end");
1555 fs_inst *inst;
1556
1557 if (gs_prog_data->static_vertex_count != -1) {
1558 foreach_in_list_reverse(fs_inst, prev, &this->instructions) {
1559 if (prev->opcode == SHADER_OPCODE_URB_WRITE_SIMD8 ||
1560 prev->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED ||
1561 prev->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT ||
1562 prev->opcode == SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT) {
1563 prev->eot = true;
1564
1565 /* Delete now dead instructions. */
1566 foreach_in_list_reverse_safe(exec_node, dead, &this->instructions) {
1567 if (dead == prev)
1568 break;
1569 dead->remove();
1570 }
1571 return;
1572 } else if (prev->is_control_flow() || prev->has_side_effects()) {
1573 break;
1574 }
1575 }
1576 fs_reg hdr = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1577 abld.MOV(hdr, fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD)));
1578 inst = abld.emit(SHADER_OPCODE_URB_WRITE_SIMD8, reg_undef, hdr);
1579 inst->mlen = 1;
1580 } else {
1581 fs_reg payload = abld.vgrf(BRW_REGISTER_TYPE_UD, 2);
1582 fs_reg *sources = ralloc_array(mem_ctx, fs_reg, 2);
1583 sources[0] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
1584 sources[1] = this->final_gs_vertex_count;
1585 abld.LOAD_PAYLOAD(payload, sources, 2, 2);
1586 inst = abld.emit(SHADER_OPCODE_URB_WRITE_SIMD8, reg_undef, payload);
1587 inst->mlen = 2;
1588 }
1589 inst->eot = true;
1590 inst->offset = 0;
1591 }
1592
1593 void
1594 fs_visitor::assign_curb_setup()
1595 {
1596 unsigned uniform_push_length = DIV_ROUND_UP(stage_prog_data->nr_params, 8);
1597
1598 unsigned ubo_push_length = 0;
1599 unsigned ubo_push_start[4];
1600 for (int i = 0; i < 4; i++) {
1601 ubo_push_start[i] = 8 * (ubo_push_length + uniform_push_length);
1602 ubo_push_length += stage_prog_data->ubo_ranges[i].length;
1603 }
1604
1605 prog_data->curb_read_length = uniform_push_length + ubo_push_length;
1606
1607 uint64_t used = 0;
1608
1609 /* Map the offsets in the UNIFORM file to fixed HW regs. */
1610 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1611 for (unsigned int i = 0; i < inst->sources; i++) {
1612 if (inst->src[i].file == UNIFORM) {
1613 int uniform_nr = inst->src[i].nr + inst->src[i].offset / 4;
1614 int constant_nr;
1615 if (inst->src[i].nr >= UBO_START) {
1616 /* constant_nr is in 32-bit units, the rest are in bytes */
1617 constant_nr = ubo_push_start[inst->src[i].nr - UBO_START] +
1618 inst->src[i].offset / 4;
1619 } else if (uniform_nr >= 0 && uniform_nr < (int) uniforms) {
1620 constant_nr = push_constant_loc[uniform_nr];
1621 } else {
1622 /* Section 5.11 of the OpenGL 4.1 spec says:
1623 * "Out-of-bounds reads return undefined values, which include
1624 * values from other variables of the active program or zero."
1625 * Just return the first push constant.
1626 */
1627 constant_nr = 0;
1628 }
1629
1630 assert(constant_nr / 8 < 64);
1631 used |= BITFIELD64_BIT(constant_nr / 8);
1632
1633 struct brw_reg brw_reg = brw_vec1_grf(payload.num_regs +
1634 constant_nr / 8,
1635 constant_nr % 8);
1636 brw_reg.abs = inst->src[i].abs;
1637 brw_reg.negate = inst->src[i].negate;
1638
1639 assert(inst->src[i].stride == 0);
1640 inst->src[i] = byte_offset(
1641 retype(brw_reg, inst->src[i].type),
1642 inst->src[i].offset % 4);
1643 }
1644 }
1645 }
1646
1647 uint64_t want_zero = used & stage_prog_data->zero_push_reg;
1648 if (want_zero) {
1649 assert(!compiler->compact_params);
1650 fs_builder ubld = bld.exec_all().group(8, 0).at(
1651 cfg->first_block(), cfg->first_block()->start());
1652
1653 /* push_reg_mask_param is in 32-bit units */
1654 unsigned mask_param = stage_prog_data->push_reg_mask_param;
1655 struct brw_reg mask = brw_vec1_grf(payload.num_regs + mask_param / 8,
1656 mask_param % 8);
1657
1658 fs_reg b32;
1659 for (unsigned i = 0; i < 64; i++) {
1660 if (i % 16 == 0 && (want_zero & BITFIELD64_RANGE(i, 16))) {
1661 fs_reg shifted = ubld.vgrf(BRW_REGISTER_TYPE_W, 2);
1662 ubld.SHL(horiz_offset(shifted, 8),
1663 byte_offset(retype(mask, BRW_REGISTER_TYPE_W), i / 8),
1664 brw_imm_v(0x01234567));
1665 ubld.SHL(shifted, horiz_offset(shifted, 8), brw_imm_w(8));
1666
1667 fs_builder ubld16 = ubld.group(16, 0);
1668 b32 = ubld16.vgrf(BRW_REGISTER_TYPE_D);
1669 ubld16.group(16, 0).ASR(b32, shifted, brw_imm_w(15));
1670 }
1671
1672 if (want_zero & BITFIELD64_BIT(i)) {
1673 assert(i < prog_data->curb_read_length);
1674 struct brw_reg push_reg =
1675 retype(brw_vec8_grf(payload.num_regs + i, 0),
1676 BRW_REGISTER_TYPE_D);
1677
1678 ubld.AND(push_reg, push_reg, component(b32, i % 16));
1679 }
1680 }
1681
1682 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
1683 }
1684
1685 /* This may be updated in assign_urb_setup or assign_vs_urb_setup. */
1686 this->first_non_payload_grf = payload.num_regs + prog_data->curb_read_length;
1687 }
1688
1689 /*
1690 * Build up an array of indices into the urb_setup array that
1691 * references the active entries of the urb_setup array.
1692 * Used to accelerate walking the active entries of the urb_setup array
1693 * on each upload.
1694 */
1695 void
1696 brw_compute_urb_setup_index(struct brw_wm_prog_data *wm_prog_data)
1697 {
1698 /* Make sure uint8_t is sufficient */
1699 STATIC_ASSERT(VARYING_SLOT_MAX <= 0xff);
1700 uint8_t index = 0;
1701 for (uint8_t attr = 0; attr < VARYING_SLOT_MAX; attr++) {
1702 if (wm_prog_data->urb_setup[attr] >= 0) {
1703 wm_prog_data->urb_setup_attribs[index++] = attr;
1704 }
1705 }
1706 wm_prog_data->urb_setup_attribs_count = index;
1707 }
1708
1709 static void
1710 calculate_urb_setup(const struct gen_device_info *devinfo,
1711 const struct brw_wm_prog_key *key,
1712 struct brw_wm_prog_data *prog_data,
1713 const nir_shader *nir)
1714 {
1715 memset(prog_data->urb_setup, -1,
1716 sizeof(prog_data->urb_setup[0]) * VARYING_SLOT_MAX);
1717
1718 int urb_next = 0;
1719 /* Figure out where each of the incoming setup attributes lands. */
1720 if (devinfo->gen >= 6) {
1721 if (util_bitcount64(nir->info.inputs_read &
1722 BRW_FS_VARYING_INPUT_MASK) <= 16) {
1723 /* The SF/SBE pipeline stage can do arbitrary rearrangement of the
1724 * first 16 varying inputs, so we can put them wherever we want.
1725 * Just put them in order.
1726 *
1727 * This is useful because it means that (a) inputs not used by the
1728 * fragment shader won't take up valuable register space, and (b) we
1729 * won't have to recompile the fragment shader if it gets paired with
1730 * a different vertex (or geometry) shader.
1731 */
1732 for (unsigned int i = 0; i < VARYING_SLOT_MAX; i++) {
1733 if (nir->info.inputs_read & BRW_FS_VARYING_INPUT_MASK &
1734 BITFIELD64_BIT(i)) {
1735 prog_data->urb_setup[i] = urb_next++;
1736 }
1737 }
1738 } else {
1739 /* We have enough input varyings that the SF/SBE pipeline stage can't
1740 * arbitrarily rearrange them to suit our whim; we have to put them
1741 * in an order that matches the output of the previous pipeline stage
1742 * (geometry or vertex shader).
1743 */
1744 struct brw_vue_map prev_stage_vue_map;
1745 brw_compute_vue_map(devinfo, &prev_stage_vue_map,
1746 key->input_slots_valid,
1747 nir->info.separate_shader, 1);
1748
1749 int first_slot =
1750 brw_compute_first_urb_slot_required(nir->info.inputs_read,
1751 &prev_stage_vue_map);
1752
1753 assert(prev_stage_vue_map.num_slots <= first_slot + 32);
1754 for (int slot = first_slot; slot < prev_stage_vue_map.num_slots;
1755 slot++) {
1756 int varying = prev_stage_vue_map.slot_to_varying[slot];
1757 if (varying != BRW_VARYING_SLOT_PAD &&
1758 (nir->info.inputs_read & BRW_FS_VARYING_INPUT_MASK &
1759 BITFIELD64_BIT(varying))) {
1760 prog_data->urb_setup[varying] = slot - first_slot;
1761 }
1762 }
1763 urb_next = prev_stage_vue_map.num_slots - first_slot;
1764 }
1765 } else {
1766 /* FINISHME: The sf doesn't map VS->FS inputs for us very well. */
1767 for (unsigned int i = 0; i < VARYING_SLOT_MAX; i++) {
1768 /* Point size is packed into the header, not as a general attribute */
1769 if (i == VARYING_SLOT_PSIZ)
1770 continue;
1771
1772 if (key->input_slots_valid & BITFIELD64_BIT(i)) {
1773 /* The back color slot is skipped when the front color is
1774 * also written to. In addition, some slots can be
1775 * written in the vertex shader and not read in the
1776 * fragment shader. So the register number must always be
1777 * incremented, mapped or not.
1778 */
1779 if (_mesa_varying_slot_in_fs((gl_varying_slot) i))
1780 prog_data->urb_setup[i] = urb_next;
1781 urb_next++;
1782 }
1783 }
1784
1785 /*
1786 * It's a FS only attribute, and we did interpolation for this attribute
1787 * in SF thread. So, count it here, too.
1788 *
1789 * See compile_sf_prog() for more info.
1790 */
1791 if (nir->info.inputs_read & BITFIELD64_BIT(VARYING_SLOT_PNTC))
1792 prog_data->urb_setup[VARYING_SLOT_PNTC] = urb_next++;
1793 }
1794
1795 prog_data->num_varying_inputs = urb_next;
1796 prog_data->inputs = nir->info.inputs_read;
1797
1798 brw_compute_urb_setup_index(prog_data);
1799 }
1800
1801 void
1802 fs_visitor::assign_urb_setup()
1803 {
1804 assert(stage == MESA_SHADER_FRAGMENT);
1805 struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
1806
1807 int urb_start = payload.num_regs + prog_data->base.curb_read_length;
1808
1809 /* Offset all the urb_setup[] index by the actual position of the
1810 * setup regs, now that the location of the constants has been chosen.
1811 */
1812 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1813 for (int i = 0; i < inst->sources; i++) {
1814 if (inst->src[i].file == ATTR) {
1815 /* ATTR regs in the FS are in units of logical scalar inputs each
1816 * of which consumes half of a GRF register.
1817 */
1818 assert(inst->src[i].offset < REG_SIZE / 2);
1819 const unsigned grf = urb_start + inst->src[i].nr / 2;
1820 const unsigned offset = (inst->src[i].nr % 2) * (REG_SIZE / 2) +
1821 inst->src[i].offset;
1822 const unsigned width = inst->src[i].stride == 0 ?
1823 1 : MIN2(inst->exec_size, 8);
1824 struct brw_reg reg = stride(
1825 byte_offset(retype(brw_vec8_grf(grf, 0), inst->src[i].type),
1826 offset),
1827 width * inst->src[i].stride,
1828 width, inst->src[i].stride);
1829 reg.abs = inst->src[i].abs;
1830 reg.negate = inst->src[i].negate;
1831 inst->src[i] = reg;
1832 }
1833 }
1834 }
1835
1836 /* Each attribute is 4 setup channels, each of which is half a reg. */
1837 this->first_non_payload_grf += prog_data->num_varying_inputs * 2;
1838 }
1839
1840 void
1841 fs_visitor::convert_attr_sources_to_hw_regs(fs_inst *inst)
1842 {
1843 for (int i = 0; i < inst->sources; i++) {
1844 if (inst->src[i].file == ATTR) {
1845 int grf = payload.num_regs +
1846 prog_data->curb_read_length +
1847 inst->src[i].nr +
1848 inst->src[i].offset / REG_SIZE;
1849
1850 /* As explained at brw_reg_from_fs_reg, From the Haswell PRM:
1851 *
1852 * VertStride must be used to cross GRF register boundaries. This
1853 * rule implies that elements within a 'Width' cannot cross GRF
1854 * boundaries.
1855 *
1856 * So, for registers that are large enough, we have to split the exec
1857 * size in two and trust the compression state to sort it out.
1858 */
1859 unsigned total_size = inst->exec_size *
1860 inst->src[i].stride *
1861 type_sz(inst->src[i].type);
1862
1863 assert(total_size <= 2 * REG_SIZE);
1864 const unsigned exec_size =
1865 (total_size <= REG_SIZE) ? inst->exec_size : inst->exec_size / 2;
1866
1867 unsigned width = inst->src[i].stride == 0 ? 1 : exec_size;
1868 struct brw_reg reg =
1869 stride(byte_offset(retype(brw_vec8_grf(grf, 0), inst->src[i].type),
1870 inst->src[i].offset % REG_SIZE),
1871 exec_size * inst->src[i].stride,
1872 width, inst->src[i].stride);
1873 reg.abs = inst->src[i].abs;
1874 reg.negate = inst->src[i].negate;
1875
1876 inst->src[i] = reg;
1877 }
1878 }
1879 }
1880
1881 void
1882 fs_visitor::assign_vs_urb_setup()
1883 {
1884 struct brw_vs_prog_data *vs_prog_data = brw_vs_prog_data(prog_data);
1885
1886 assert(stage == MESA_SHADER_VERTEX);
1887
1888 /* Each attribute is 4 regs. */
1889 this->first_non_payload_grf += 4 * vs_prog_data->nr_attribute_slots;
1890
1891 assert(vs_prog_data->base.urb_read_length <= 15);
1892
1893 /* Rewrite all ATTR file references to the hw grf that they land in. */
1894 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1895 convert_attr_sources_to_hw_regs(inst);
1896 }
1897 }
1898
1899 void
1900 fs_visitor::assign_tcs_urb_setup()
1901 {
1902 assert(stage == MESA_SHADER_TESS_CTRL);
1903
1904 /* Rewrite all ATTR file references to HW_REGs. */
1905 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1906 convert_attr_sources_to_hw_regs(inst);
1907 }
1908 }
1909
1910 void
1911 fs_visitor::assign_tes_urb_setup()
1912 {
1913 assert(stage == MESA_SHADER_TESS_EVAL);
1914
1915 struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data);
1916
1917 first_non_payload_grf += 8 * vue_prog_data->urb_read_length;
1918
1919 /* Rewrite all ATTR file references to HW_REGs. */
1920 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1921 convert_attr_sources_to_hw_regs(inst);
1922 }
1923 }
1924
1925 void
1926 fs_visitor::assign_gs_urb_setup()
1927 {
1928 assert(stage == MESA_SHADER_GEOMETRY);
1929
1930 struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data);
1931
1932 first_non_payload_grf +=
1933 8 * vue_prog_data->urb_read_length * nir->info.gs.vertices_in;
1934
1935 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1936 /* Rewrite all ATTR file references to GRFs. */
1937 convert_attr_sources_to_hw_regs(inst);
1938 }
1939 }
1940
1941
1942 /**
1943 * Split large virtual GRFs into separate components if we can.
1944 *
1945 * This is mostly duplicated with what brw_fs_vector_splitting does,
1946 * but that's really conservative because it's afraid of doing
1947 * splitting that doesn't result in real progress after the rest of
1948 * the optimization phases, which would cause infinite looping in
1949 * optimization. We can do it once here, safely. This also has the
1950 * opportunity to split interpolated values, or maybe even uniforms,
1951 * which we don't have at the IR level.
1952 *
1953 * We want to split, because virtual GRFs are what we register
1954 * allocate and spill (due to contiguousness requirements for some
1955 * instructions), and they're what we naturally generate in the
1956 * codegen process, but most virtual GRFs don't actually need to be
1957 * contiguous sets of GRFs. If we split, we'll end up with reduced
1958 * live intervals and better dead code elimination and coalescing.
1959 */
1960 void
1961 fs_visitor::split_virtual_grfs()
1962 {
1963 /* Compact the register file so we eliminate dead vgrfs. This
1964 * only defines split points for live registers, so if we have
1965 * too large dead registers they will hit assertions later.
1966 */
1967 compact_virtual_grfs();
1968
1969 int num_vars = this->alloc.count;
1970
1971 /* Count the total number of registers */
1972 int reg_count = 0;
1973 int vgrf_to_reg[num_vars];
1974 for (int i = 0; i < num_vars; i++) {
1975 vgrf_to_reg[i] = reg_count;
1976 reg_count += alloc.sizes[i];
1977 }
1978
1979 /* An array of "split points". For each register slot, this indicates
1980 * if this slot can be separated from the previous slot. Every time an
1981 * instruction uses multiple elements of a register (as a source or
1982 * destination), we mark the used slots as inseparable. Then we go
1983 * through and split the registers into the smallest pieces we can.
1984 */
1985 bool *split_points = new bool[reg_count];
1986 memset(split_points, 0, reg_count * sizeof(*split_points));
1987
1988 /* Mark all used registers as fully splittable */
1989 foreach_block_and_inst(block, fs_inst, inst, cfg) {
1990 if (inst->dst.file == VGRF) {
1991 int reg = vgrf_to_reg[inst->dst.nr];
1992 for (unsigned j = 1; j < this->alloc.sizes[inst->dst.nr]; j++)
1993 split_points[reg + j] = true;
1994 }
1995
1996 for (int i = 0; i < inst->sources; i++) {
1997 if (inst->src[i].file == VGRF) {
1998 int reg = vgrf_to_reg[inst->src[i].nr];
1999 for (unsigned j = 1; j < this->alloc.sizes[inst->src[i].nr]; j++)
2000 split_points[reg + j] = true;
2001 }
2002 }
2003 }
2004
2005 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2006 /* We fix up undef instructions later */
2007 if (inst->opcode == SHADER_OPCODE_UNDEF) {
2008 /* UNDEF instructions are currently only used to undef entire
2009 * registers. We need this invariant later when we split them.
2010 */
2011 assert(inst->dst.file == VGRF);
2012 assert(inst->dst.offset == 0);
2013 assert(inst->size_written == alloc.sizes[inst->dst.nr] * REG_SIZE);
2014 continue;
2015 }
2016
2017 if (inst->dst.file == VGRF) {
2018 int reg = vgrf_to_reg[inst->dst.nr] + inst->dst.offset / REG_SIZE;
2019 for (unsigned j = 1; j < regs_written(inst); j++)
2020 split_points[reg + j] = false;
2021 }
2022 for (int i = 0; i < inst->sources; i++) {
2023 if (inst->src[i].file == VGRF) {
2024 int reg = vgrf_to_reg[inst->src[i].nr] + inst->src[i].offset / REG_SIZE;
2025 for (unsigned j = 1; j < regs_read(inst, i); j++)
2026 split_points[reg + j] = false;
2027 }
2028 }
2029 }
2030
2031 int *new_virtual_grf = new int[reg_count];
2032 int *new_reg_offset = new int[reg_count];
2033
2034 int reg = 0;
2035 for (int i = 0; i < num_vars; i++) {
2036 /* The first one should always be 0 as a quick sanity check. */
2037 assert(split_points[reg] == false);
2038
2039 /* j = 0 case */
2040 new_reg_offset[reg] = 0;
2041 reg++;
2042 int offset = 1;
2043
2044 /* j > 0 case */
2045 for (unsigned j = 1; j < alloc.sizes[i]; j++) {
2046 /* If this is a split point, reset the offset to 0 and allocate a
2047 * new virtual GRF for the previous offset many registers
2048 */
2049 if (split_points[reg]) {
2050 assert(offset <= MAX_VGRF_SIZE);
2051 int grf = alloc.allocate(offset);
2052 for (int k = reg - offset; k < reg; k++)
2053 new_virtual_grf[k] = grf;
2054 offset = 0;
2055 }
2056 new_reg_offset[reg] = offset;
2057 offset++;
2058 reg++;
2059 }
2060
2061 /* The last one gets the original register number */
2062 assert(offset <= MAX_VGRF_SIZE);
2063 alloc.sizes[i] = offset;
2064 for (int k = reg - offset; k < reg; k++)
2065 new_virtual_grf[k] = i;
2066 }
2067 assert(reg == reg_count);
2068
2069 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
2070 if (inst->opcode == SHADER_OPCODE_UNDEF) {
2071 const fs_builder ibld(this, block, inst);
2072 assert(inst->size_written % REG_SIZE == 0);
2073 unsigned reg_offset = 0;
2074 while (reg_offset < inst->size_written / REG_SIZE) {
2075 reg = vgrf_to_reg[inst->dst.nr] + reg_offset;
2076 ibld.UNDEF(fs_reg(VGRF, new_virtual_grf[reg], inst->dst.type));
2077 reg_offset += alloc.sizes[new_virtual_grf[reg]];
2078 }
2079 inst->remove(block);
2080 continue;
2081 }
2082
2083 if (inst->dst.file == VGRF) {
2084 reg = vgrf_to_reg[inst->dst.nr] + inst->dst.offset / REG_SIZE;
2085 inst->dst.nr = new_virtual_grf[reg];
2086 inst->dst.offset = new_reg_offset[reg] * REG_SIZE +
2087 inst->dst.offset % REG_SIZE;
2088 assert((unsigned)new_reg_offset[reg] < alloc.sizes[new_virtual_grf[reg]]);
2089 }
2090 for (int i = 0; i < inst->sources; i++) {
2091 if (inst->src[i].file == VGRF) {
2092 reg = vgrf_to_reg[inst->src[i].nr] + inst->src[i].offset / REG_SIZE;
2093 inst->src[i].nr = new_virtual_grf[reg];
2094 inst->src[i].offset = new_reg_offset[reg] * REG_SIZE +
2095 inst->src[i].offset % REG_SIZE;
2096 assert((unsigned)new_reg_offset[reg] < alloc.sizes[new_virtual_grf[reg]]);
2097 }
2098 }
2099 }
2100 invalidate_analysis(DEPENDENCY_INSTRUCTION_DETAIL | DEPENDENCY_VARIABLES);
2101
2102 delete[] split_points;
2103 delete[] new_virtual_grf;
2104 delete[] new_reg_offset;
2105 }
2106
2107 /**
2108 * Remove unused virtual GRFs and compact the vgrf_* arrays.
2109 *
2110 * During code generation, we create tons of temporary variables, many of
2111 * which get immediately killed and are never used again. Yet, in later
2112 * optimization and analysis passes, such as compute_live_intervals, we need
2113 * to loop over all the virtual GRFs. Compacting them can save a lot of
2114 * overhead.
2115 */
2116 bool
2117 fs_visitor::compact_virtual_grfs()
2118 {
2119 bool progress = false;
2120 int *remap_table = new int[this->alloc.count];
2121 memset(remap_table, -1, this->alloc.count * sizeof(int));
2122
2123 /* Mark which virtual GRFs are used. */
2124 foreach_block_and_inst(block, const fs_inst, inst, cfg) {
2125 if (inst->dst.file == VGRF)
2126 remap_table[inst->dst.nr] = 0;
2127
2128 for (int i = 0; i < inst->sources; i++) {
2129 if (inst->src[i].file == VGRF)
2130 remap_table[inst->src[i].nr] = 0;
2131 }
2132 }
2133
2134 /* Compact the GRF arrays. */
2135 int new_index = 0;
2136 for (unsigned i = 0; i < this->alloc.count; i++) {
2137 if (remap_table[i] == -1) {
2138 /* We just found an unused register. This means that we are
2139 * actually going to compact something.
2140 */
2141 progress = true;
2142 } else {
2143 remap_table[i] = new_index;
2144 alloc.sizes[new_index] = alloc.sizes[i];
2145 invalidate_analysis(DEPENDENCY_INSTRUCTION_DETAIL | DEPENDENCY_VARIABLES);
2146 ++new_index;
2147 }
2148 }
2149
2150 this->alloc.count = new_index;
2151
2152 /* Patch all the instructions to use the newly renumbered registers */
2153 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2154 if (inst->dst.file == VGRF)
2155 inst->dst.nr = remap_table[inst->dst.nr];
2156
2157 for (int i = 0; i < inst->sources; i++) {
2158 if (inst->src[i].file == VGRF)
2159 inst->src[i].nr = remap_table[inst->src[i].nr];
2160 }
2161 }
2162
2163 /* Patch all the references to delta_xy, since they're used in register
2164 * allocation. If they're unused, switch them to BAD_FILE so we don't
2165 * think some random VGRF is delta_xy.
2166 */
2167 for (unsigned i = 0; i < ARRAY_SIZE(delta_xy); i++) {
2168 if (delta_xy[i].file == VGRF) {
2169 if (remap_table[delta_xy[i].nr] != -1) {
2170 delta_xy[i].nr = remap_table[delta_xy[i].nr];
2171 } else {
2172 delta_xy[i].file = BAD_FILE;
2173 }
2174 }
2175 }
2176
2177 delete[] remap_table;
2178
2179 return progress;
2180 }
2181
2182 static int
2183 get_subgroup_id_param_index(const brw_stage_prog_data *prog_data)
2184 {
2185 if (prog_data->nr_params == 0)
2186 return -1;
2187
2188 /* The local thread id is always the last parameter in the list */
2189 uint32_t last_param = prog_data->param[prog_data->nr_params - 1];
2190 if (last_param == BRW_PARAM_BUILTIN_SUBGROUP_ID)
2191 return prog_data->nr_params - 1;
2192
2193 return -1;
2194 }
2195
2196 /**
2197 * Struct for handling complex alignments.
2198 *
2199 * A complex alignment is stored as multiplier and an offset. A value is
2200 * considered to be aligned if it is {offset} larger than a multiple of {mul}.
2201 * For instance, with an alignment of {8, 2}, cplx_align_apply would do the
2202 * following:
2203 *
2204 * N | cplx_align_apply({8, 2}, N)
2205 * ----+-----------------------------
2206 * 4 | 6
2207 * 6 | 6
2208 * 8 | 14
2209 * 10 | 14
2210 * 12 | 14
2211 * 14 | 14
2212 * 16 | 22
2213 */
2214 struct cplx_align {
2215 unsigned mul:4;
2216 unsigned offset:4;
2217 };
2218
2219 #define CPLX_ALIGN_MAX_MUL 8
2220
2221 static void
2222 cplx_align_assert_sane(struct cplx_align a)
2223 {
2224 assert(a.mul > 0 && util_is_power_of_two_nonzero(a.mul));
2225 assert(a.offset < a.mul);
2226 }
2227
2228 /**
2229 * Combines two alignments to produce a least multiple of sorts.
2230 *
2231 * The returned alignment is the smallest (in terms of multiplier) such that
2232 * anything aligned to both a and b will be aligned to the new alignment.
2233 * This function will assert-fail if a and b are not compatible, i.e. if the
2234 * offset parameters are such that no common alignment is possible.
2235 */
2236 static struct cplx_align
2237 cplx_align_combine(struct cplx_align a, struct cplx_align b)
2238 {
2239 cplx_align_assert_sane(a);
2240 cplx_align_assert_sane(b);
2241
2242 /* Assert that the alignments agree. */
2243 assert((a.offset & (b.mul - 1)) == (b.offset & (a.mul - 1)));
2244
2245 return a.mul > b.mul ? a : b;
2246 }
2247
2248 /**
2249 * Apply a complex alignment
2250 *
2251 * This function will return the smallest number greater than or equal to
2252 * offset that is aligned to align.
2253 */
2254 static unsigned
2255 cplx_align_apply(struct cplx_align align, unsigned offset)
2256 {
2257 return ALIGN(offset - align.offset, align.mul) + align.offset;
2258 }
2259
2260 #define UNIFORM_SLOT_SIZE 4
2261
2262 struct uniform_slot_info {
2263 /** True if the given uniform slot is live */
2264 unsigned is_live:1;
2265
2266 /** True if this slot and the next slot must remain contiguous */
2267 unsigned contiguous:1;
2268
2269 struct cplx_align align;
2270 };
2271
2272 static void
2273 mark_uniform_slots_read(struct uniform_slot_info *slots,
2274 unsigned num_slots, unsigned alignment)
2275 {
2276 assert(alignment > 0 && util_is_power_of_two_nonzero(alignment));
2277 assert(alignment <= CPLX_ALIGN_MAX_MUL);
2278
2279 /* We can't align a slot to anything less than the slot size */
2280 alignment = MAX2(alignment, UNIFORM_SLOT_SIZE);
2281
2282 struct cplx_align align = {alignment, 0};
2283 cplx_align_assert_sane(align);
2284
2285 for (unsigned i = 0; i < num_slots; i++) {
2286 slots[i].is_live = true;
2287 if (i < num_slots - 1)
2288 slots[i].contiguous = true;
2289
2290 align.offset = (i * UNIFORM_SLOT_SIZE) & (align.mul - 1);
2291 if (slots[i].align.mul == 0) {
2292 slots[i].align = align;
2293 } else {
2294 slots[i].align = cplx_align_combine(slots[i].align, align);
2295 }
2296 }
2297 }
2298
2299 /**
2300 * Assign UNIFORM file registers to either push constants or pull constants.
2301 *
2302 * We allow a fragment shader to have more than the specified minimum
2303 * maximum number of fragment shader uniform components (64). If
2304 * there are too many of these, they'd fill up all of register space.
2305 * So, this will push some of them out to the pull constant buffer and
2306 * update the program to load them.
2307 */
2308 void
2309 fs_visitor::assign_constant_locations()
2310 {
2311 /* Only the first compile gets to decide on locations. */
2312 if (push_constant_loc) {
2313 assert(pull_constant_loc);
2314 return;
2315 }
2316
2317 if (compiler->compact_params) {
2318 struct uniform_slot_info slots[uniforms + 1];
2319 memset(slots, 0, sizeof(slots));
2320
2321 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
2322 for (int i = 0 ; i < inst->sources; i++) {
2323 if (inst->src[i].file != UNIFORM)
2324 continue;
2325
2326 /* NIR tightly packs things so the uniform number might not be
2327 * aligned (if we have a double right after a float, for
2328 * instance). This is fine because the process of re-arranging
2329 * them will ensure that things are properly aligned. The offset
2330 * into that uniform, however, must be aligned.
2331 *
2332 * In Vulkan, we have explicit offsets but everything is crammed
2333 * into a single "variable" so inst->src[i].nr will always be 0.
2334 * Everything will be properly aligned relative to that one base.
2335 */
2336 assert(inst->src[i].offset % type_sz(inst->src[i].type) == 0);
2337
2338 unsigned u = inst->src[i].nr +
2339 inst->src[i].offset / UNIFORM_SLOT_SIZE;
2340
2341 if (u >= uniforms)
2342 continue;
2343
2344 unsigned slots_read;
2345 if (inst->opcode == SHADER_OPCODE_MOV_INDIRECT && i == 0) {
2346 slots_read = DIV_ROUND_UP(inst->src[2].ud, UNIFORM_SLOT_SIZE);
2347 } else {
2348 unsigned bytes_read = inst->components_read(i) *
2349 type_sz(inst->src[i].type);
2350 slots_read = DIV_ROUND_UP(bytes_read, UNIFORM_SLOT_SIZE);
2351 }
2352
2353 assert(u + slots_read <= uniforms);
2354 mark_uniform_slots_read(&slots[u], slots_read,
2355 type_sz(inst->src[i].type));
2356 }
2357 }
2358
2359 int subgroup_id_index = get_subgroup_id_param_index(stage_prog_data);
2360
2361 /* Only allow 16 registers (128 uniform components) as push constants.
2362 *
2363 * Just demote the end of the list. We could probably do better
2364 * here, demoting things that are rarely used in the program first.
2365 *
2366 * If changing this value, note the limitation about total_regs in
2367 * brw_curbe.c.
2368 */
2369 unsigned int max_push_components = 16 * 8;
2370 if (subgroup_id_index >= 0)
2371 max_push_components--; /* Save a slot for the thread ID */
2372
2373 /* We push small arrays, but no bigger than 16 floats. This is big
2374 * enough for a vec4 but hopefully not large enough to push out other
2375 * stuff. We should probably use a better heuristic at some point.
2376 */
2377 const unsigned int max_chunk_size = 16;
2378
2379 unsigned int num_push_constants = 0;
2380 unsigned int num_pull_constants = 0;
2381
2382 push_constant_loc = ralloc_array(mem_ctx, int, uniforms);
2383 pull_constant_loc = ralloc_array(mem_ctx, int, uniforms);
2384
2385 /* Default to -1 meaning no location */
2386 memset(push_constant_loc, -1, uniforms * sizeof(*push_constant_loc));
2387 memset(pull_constant_loc, -1, uniforms * sizeof(*pull_constant_loc));
2388
2389 int chunk_start = -1;
2390 struct cplx_align align;
2391 for (unsigned u = 0; u < uniforms; u++) {
2392 if (!slots[u].is_live) {
2393 assert(chunk_start == -1);
2394 continue;
2395 }
2396
2397 /* Skip subgroup_id_index to put it in the last push register. */
2398 if (subgroup_id_index == (int)u)
2399 continue;
2400
2401 if (chunk_start == -1) {
2402 chunk_start = u;
2403 align = slots[u].align;
2404 } else {
2405 /* Offset into the chunk */
2406 unsigned chunk_offset = (u - chunk_start) * UNIFORM_SLOT_SIZE;
2407
2408 /* Shift the slot alignment down by the chunk offset so it is
2409 * comparable with the base chunk alignment.
2410 */
2411 struct cplx_align slot_align = slots[u].align;
2412 slot_align.offset =
2413 (slot_align.offset - chunk_offset) & (align.mul - 1);
2414
2415 align = cplx_align_combine(align, slot_align);
2416 }
2417
2418 /* Sanity check the alignment */
2419 cplx_align_assert_sane(align);
2420
2421 if (slots[u].contiguous)
2422 continue;
2423
2424 /* Adjust the alignment to be in terms of slots, not bytes */
2425 assert((align.mul & (UNIFORM_SLOT_SIZE - 1)) == 0);
2426 assert((align.offset & (UNIFORM_SLOT_SIZE - 1)) == 0);
2427 align.mul /= UNIFORM_SLOT_SIZE;
2428 align.offset /= UNIFORM_SLOT_SIZE;
2429
2430 unsigned push_start_align = cplx_align_apply(align, num_push_constants);
2431 unsigned chunk_size = u - chunk_start + 1;
2432 if ((!compiler->supports_pull_constants && u < UBO_START) ||
2433 (chunk_size < max_chunk_size &&
2434 push_start_align + chunk_size <= max_push_components)) {
2435 /* Align up the number of push constants */
2436 num_push_constants = push_start_align;
2437 for (unsigned i = 0; i < chunk_size; i++)
2438 push_constant_loc[chunk_start + i] = num_push_constants++;
2439 } else {
2440 /* We need to pull this one */
2441 num_pull_constants = cplx_align_apply(align, num_pull_constants);
2442 for (unsigned i = 0; i < chunk_size; i++)
2443 pull_constant_loc[chunk_start + i] = num_pull_constants++;
2444 }
2445
2446 /* Reset the chunk and start again */
2447 chunk_start = -1;
2448 }
2449
2450 /* Add the CS local thread ID uniform at the end of the push constants */
2451 if (subgroup_id_index >= 0)
2452 push_constant_loc[subgroup_id_index] = num_push_constants++;
2453
2454 /* As the uniforms are going to be reordered, stash the old array and
2455 * create two new arrays for push/pull params.
2456 */
2457 uint32_t *param = stage_prog_data->param;
2458 stage_prog_data->nr_params = num_push_constants;
2459 if (num_push_constants) {
2460 stage_prog_data->param = rzalloc_array(mem_ctx, uint32_t,
2461 num_push_constants);
2462 } else {
2463 stage_prog_data->param = NULL;
2464 }
2465 assert(stage_prog_data->nr_pull_params == 0);
2466 assert(stage_prog_data->pull_param == NULL);
2467 if (num_pull_constants > 0) {
2468 stage_prog_data->nr_pull_params = num_pull_constants;
2469 stage_prog_data->pull_param = rzalloc_array(mem_ctx, uint32_t,
2470 num_pull_constants);
2471 }
2472
2473 /* Up until now, the param[] array has been indexed by reg + offset
2474 * of UNIFORM registers. Move pull constants into pull_param[] and
2475 * condense param[] to only contain the uniforms we chose to push.
2476 *
2477 * NOTE: Because we are condensing the params[] array, we know that
2478 * push_constant_loc[i] <= i and we can do it in one smooth loop without
2479 * having to make a copy.
2480 */
2481 for (unsigned int i = 0; i < uniforms; i++) {
2482 uint32_t value = param[i];
2483 if (pull_constant_loc[i] != -1) {
2484 stage_prog_data->pull_param[pull_constant_loc[i]] = value;
2485 } else if (push_constant_loc[i] != -1) {
2486 stage_prog_data->param[push_constant_loc[i]] = value;
2487 }
2488 }
2489 ralloc_free(param);
2490 } else {
2491 /* If we don't want to compact anything, just set up dummy push/pull
2492 * arrays. All the rest of the compiler cares about are these arrays.
2493 */
2494 push_constant_loc = ralloc_array(mem_ctx, int, uniforms);
2495 pull_constant_loc = ralloc_array(mem_ctx, int, uniforms);
2496
2497 for (unsigned u = 0; u < uniforms; u++)
2498 push_constant_loc[u] = u;
2499
2500 memset(pull_constant_loc, -1, uniforms * sizeof(*pull_constant_loc));
2501 }
2502
2503 /* Now that we know how many regular uniforms we'll push, reduce the
2504 * UBO push ranges so we don't exceed the 3DSTATE_CONSTANT limits.
2505 */
2506 unsigned push_length = DIV_ROUND_UP(stage_prog_data->nr_params, 8);
2507 for (int i = 0; i < 4; i++) {
2508 struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
2509
2510 if (push_length + range->length > 64)
2511 range->length = 64 - push_length;
2512
2513 push_length += range->length;
2514 }
2515 assert(push_length <= 64);
2516 }
2517
2518 bool
2519 fs_visitor::get_pull_locs(const fs_reg &src,
2520 unsigned *out_surf_index,
2521 unsigned *out_pull_index)
2522 {
2523 assert(src.file == UNIFORM);
2524
2525 if (src.nr >= UBO_START) {
2526 const struct brw_ubo_range *range =
2527 &prog_data->ubo_ranges[src.nr - UBO_START];
2528
2529 /* If this access is in our (reduced) range, use the push data. */
2530 if (src.offset / 32 < range->length)
2531 return false;
2532
2533 *out_surf_index = prog_data->binding_table.ubo_start + range->block;
2534 *out_pull_index = (32 * range->start + src.offset) / 4;
2535
2536 prog_data->has_ubo_pull = true;
2537 return true;
2538 }
2539
2540 const unsigned location = src.nr + src.offset / 4;
2541
2542 if (location < uniforms && pull_constant_loc[location] != -1) {
2543 /* A regular uniform push constant */
2544 *out_surf_index = stage_prog_data->binding_table.pull_constants_start;
2545 *out_pull_index = pull_constant_loc[location];
2546
2547 prog_data->has_ubo_pull = true;
2548 return true;
2549 }
2550
2551 return false;
2552 }
2553
2554 /**
2555 * Replace UNIFORM register file access with either UNIFORM_PULL_CONSTANT_LOAD
2556 * or VARYING_PULL_CONSTANT_LOAD instructions which load values into VGRFs.
2557 */
2558 void
2559 fs_visitor::lower_constant_loads()
2560 {
2561 unsigned index, pull_index;
2562
2563 foreach_block_and_inst_safe (block, fs_inst, inst, cfg) {
2564 /* Set up the annotation tracking for new generated instructions. */
2565 const fs_builder ibld(this, block, inst);
2566
2567 for (int i = 0; i < inst->sources; i++) {
2568 if (inst->src[i].file != UNIFORM)
2569 continue;
2570
2571 /* We'll handle this case later */
2572 if (inst->opcode == SHADER_OPCODE_MOV_INDIRECT && i == 0)
2573 continue;
2574
2575 if (!get_pull_locs(inst->src[i], &index, &pull_index))
2576 continue;
2577
2578 assert(inst->src[i].stride == 0);
2579
2580 const unsigned block_sz = 64; /* Fetch one cacheline at a time. */
2581 const fs_builder ubld = ibld.exec_all().group(block_sz / 4, 0);
2582 const fs_reg dst = ubld.vgrf(BRW_REGISTER_TYPE_UD);
2583 const unsigned base = pull_index * 4;
2584
2585 ubld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
2586 dst, brw_imm_ud(index), brw_imm_ud(base & ~(block_sz - 1)));
2587
2588 /* Rewrite the instruction to use the temporary VGRF. */
2589 inst->src[i].file = VGRF;
2590 inst->src[i].nr = dst.nr;
2591 inst->src[i].offset = (base & (block_sz - 1)) +
2592 inst->src[i].offset % 4;
2593 }
2594
2595 if (inst->opcode == SHADER_OPCODE_MOV_INDIRECT &&
2596 inst->src[0].file == UNIFORM) {
2597
2598 if (!get_pull_locs(inst->src[0], &index, &pull_index))
2599 continue;
2600
2601 VARYING_PULL_CONSTANT_LOAD(ibld, inst->dst,
2602 brw_imm_ud(index),
2603 inst->src[1],
2604 pull_index * 4);
2605 inst->remove(block);
2606 }
2607 }
2608 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
2609 }
2610
2611 bool
2612 fs_visitor::opt_algebraic()
2613 {
2614 bool progress = false;
2615
2616 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
2617 switch (inst->opcode) {
2618 case BRW_OPCODE_MOV:
2619 if (!devinfo->has_64bit_float &&
2620 !devinfo->has_64bit_int &&
2621 (inst->dst.type == BRW_REGISTER_TYPE_DF ||
2622 inst->dst.type == BRW_REGISTER_TYPE_UQ ||
2623 inst->dst.type == BRW_REGISTER_TYPE_Q)) {
2624 assert(inst->dst.type == inst->src[0].type);
2625 assert(!inst->saturate);
2626 assert(!inst->src[0].abs);
2627 assert(!inst->src[0].negate);
2628 const brw::fs_builder ibld(this, block, inst);
2629
2630 if (inst->src[0].file == IMM) {
2631 ibld.MOV(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 1),
2632 brw_imm_ud(inst->src[0].u64 >> 32));
2633 ibld.MOV(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 0),
2634 brw_imm_ud(inst->src[0].u64));
2635 } else {
2636 ibld.MOV(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 1),
2637 subscript(inst->src[0], BRW_REGISTER_TYPE_UD, 1));
2638 ibld.MOV(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 0),
2639 subscript(inst->src[0], BRW_REGISTER_TYPE_UD, 0));
2640 }
2641
2642 inst->remove(block);
2643 progress = true;
2644 }
2645
2646 if ((inst->conditional_mod == BRW_CONDITIONAL_Z ||
2647 inst->conditional_mod == BRW_CONDITIONAL_NZ) &&
2648 inst->dst.is_null() &&
2649 (inst->src[0].abs || inst->src[0].negate)) {
2650 inst->src[0].abs = false;
2651 inst->src[0].negate = false;
2652 progress = true;
2653 break;
2654 }
2655
2656 if (inst->src[0].file != IMM)
2657 break;
2658
2659 if (inst->saturate) {
2660 /* Full mixed-type saturates don't happen. However, we can end up
2661 * with things like:
2662 *
2663 * mov.sat(8) g21<1>DF -1F
2664 *
2665 * Other mixed-size-but-same-base-type cases may also be possible.
2666 */
2667 if (inst->dst.type != inst->src[0].type &&
2668 inst->dst.type != BRW_REGISTER_TYPE_DF &&
2669 inst->src[0].type != BRW_REGISTER_TYPE_F)
2670 assert(!"unimplemented: saturate mixed types");
2671
2672 if (brw_saturate_immediate(inst->src[0].type,
2673 &inst->src[0].as_brw_reg())) {
2674 inst->saturate = false;
2675 progress = true;
2676 }
2677 }
2678 break;
2679
2680 case BRW_OPCODE_MUL:
2681 if (inst->src[1].file != IMM)
2682 continue;
2683
2684 /* a * 1.0 = a */
2685 if (inst->src[1].is_one()) {
2686 inst->opcode = BRW_OPCODE_MOV;
2687 inst->src[1] = reg_undef;
2688 progress = true;
2689 break;
2690 }
2691
2692 /* a * -1.0 = -a */
2693 if (inst->src[1].is_negative_one()) {
2694 inst->opcode = BRW_OPCODE_MOV;
2695 inst->src[0].negate = !inst->src[0].negate;
2696 inst->src[1] = reg_undef;
2697 progress = true;
2698 break;
2699 }
2700
2701 break;
2702 case BRW_OPCODE_ADD:
2703 if (inst->src[1].file != IMM)
2704 continue;
2705
2706 if (brw_reg_type_is_integer(inst->src[1].type) &&
2707 inst->src[1].is_zero()) {
2708 inst->opcode = BRW_OPCODE_MOV;
2709 inst->src[1] = reg_undef;
2710 progress = true;
2711 break;
2712 }
2713
2714 if (inst->src[0].file == IMM) {
2715 assert(inst->src[0].type == BRW_REGISTER_TYPE_F);
2716 inst->opcode = BRW_OPCODE_MOV;
2717 inst->src[0].f += inst->src[1].f;
2718 inst->src[1] = reg_undef;
2719 progress = true;
2720 break;
2721 }
2722 break;
2723 case BRW_OPCODE_OR:
2724 if (inst->src[0].equals(inst->src[1]) ||
2725 inst->src[1].is_zero()) {
2726 /* On Gen8+, the OR instruction can have a source modifier that
2727 * performs logical not on the operand. Cases of 'OR r0, ~r1, 0'
2728 * or 'OR r0, ~r1, ~r1' should become a NOT instead of a MOV.
2729 */
2730 if (inst->src[0].negate) {
2731 inst->opcode = BRW_OPCODE_NOT;
2732 inst->src[0].negate = false;
2733 } else {
2734 inst->opcode = BRW_OPCODE_MOV;
2735 }
2736 inst->src[1] = reg_undef;
2737 progress = true;
2738 break;
2739 }
2740 break;
2741 case BRW_OPCODE_CMP:
2742 if ((inst->conditional_mod == BRW_CONDITIONAL_Z ||
2743 inst->conditional_mod == BRW_CONDITIONAL_NZ) &&
2744 inst->src[1].is_zero() &&
2745 (inst->src[0].abs || inst->src[0].negate)) {
2746 inst->src[0].abs = false;
2747 inst->src[0].negate = false;
2748 progress = true;
2749 break;
2750 }
2751 break;
2752 case BRW_OPCODE_SEL:
2753 if (!devinfo->has_64bit_float &&
2754 !devinfo->has_64bit_int &&
2755 (inst->dst.type == BRW_REGISTER_TYPE_DF ||
2756 inst->dst.type == BRW_REGISTER_TYPE_UQ ||
2757 inst->dst.type == BRW_REGISTER_TYPE_Q)) {
2758 assert(inst->dst.type == inst->src[0].type);
2759 assert(!inst->saturate);
2760 assert(!inst->src[0].abs && !inst->src[0].negate);
2761 assert(!inst->src[1].abs && !inst->src[1].negate);
2762 const brw::fs_builder ibld(this, block, inst);
2763
2764 set_predicate(inst->predicate,
2765 ibld.SEL(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 0),
2766 subscript(inst->src[0], BRW_REGISTER_TYPE_UD, 0),
2767 subscript(inst->src[1], BRW_REGISTER_TYPE_UD, 0)));
2768 set_predicate(inst->predicate,
2769 ibld.SEL(subscript(inst->dst, BRW_REGISTER_TYPE_UD, 1),
2770 subscript(inst->src[0], BRW_REGISTER_TYPE_UD, 1),
2771 subscript(inst->src[1], BRW_REGISTER_TYPE_UD, 1)));
2772
2773 inst->remove(block);
2774 progress = true;
2775 }
2776 if (inst->src[0].equals(inst->src[1])) {
2777 inst->opcode = BRW_OPCODE_MOV;
2778 inst->src[1] = reg_undef;
2779 inst->predicate = BRW_PREDICATE_NONE;
2780 inst->predicate_inverse = false;
2781 progress = true;
2782 } else if (inst->saturate && inst->src[1].file == IMM) {
2783 switch (inst->conditional_mod) {
2784 case BRW_CONDITIONAL_LE:
2785 case BRW_CONDITIONAL_L:
2786 switch (inst->src[1].type) {
2787 case BRW_REGISTER_TYPE_F:
2788 if (inst->src[1].f >= 1.0f) {
2789 inst->opcode = BRW_OPCODE_MOV;
2790 inst->src[1] = reg_undef;
2791 inst->conditional_mod = BRW_CONDITIONAL_NONE;
2792 progress = true;
2793 }
2794 break;
2795 default:
2796 break;
2797 }
2798 break;
2799 case BRW_CONDITIONAL_GE:
2800 case BRW_CONDITIONAL_G:
2801 switch (inst->src[1].type) {
2802 case BRW_REGISTER_TYPE_F:
2803 if (inst->src[1].f <= 0.0f) {
2804 inst->opcode = BRW_OPCODE_MOV;
2805 inst->src[1] = reg_undef;
2806 inst->conditional_mod = BRW_CONDITIONAL_NONE;
2807 progress = true;
2808 }
2809 break;
2810 default:
2811 break;
2812 }
2813 default:
2814 break;
2815 }
2816 }
2817 break;
2818 case BRW_OPCODE_MAD:
2819 if (inst->src[0].type != BRW_REGISTER_TYPE_F ||
2820 inst->src[1].type != BRW_REGISTER_TYPE_F ||
2821 inst->src[2].type != BRW_REGISTER_TYPE_F)
2822 break;
2823 if (inst->src[1].is_one()) {
2824 inst->opcode = BRW_OPCODE_ADD;
2825 inst->src[1] = inst->src[2];
2826 inst->src[2] = reg_undef;
2827 progress = true;
2828 } else if (inst->src[2].is_one()) {
2829 inst->opcode = BRW_OPCODE_ADD;
2830 inst->src[2] = reg_undef;
2831 progress = true;
2832 }
2833 break;
2834 case SHADER_OPCODE_BROADCAST:
2835 if (is_uniform(inst->src[0])) {
2836 inst->opcode = BRW_OPCODE_MOV;
2837 inst->sources = 1;
2838 inst->force_writemask_all = true;
2839 progress = true;
2840 } else if (inst->src[1].file == IMM) {
2841 inst->opcode = BRW_OPCODE_MOV;
2842 /* It's possible that the selected component will be too large and
2843 * overflow the register. This can happen if someone does a
2844 * readInvocation() from GLSL or SPIR-V and provides an OOB
2845 * invocationIndex. If this happens and we some how manage
2846 * to constant fold it in and get here, then component() may cause
2847 * us to start reading outside of the VGRF which will lead to an
2848 * assert later. Instead, just let it wrap around if it goes over
2849 * exec_size.
2850 */
2851 const unsigned comp = inst->src[1].ud & (inst->exec_size - 1);
2852 inst->src[0] = component(inst->src[0], comp);
2853 inst->sources = 1;
2854 inst->force_writemask_all = true;
2855 progress = true;
2856 }
2857 break;
2858
2859 case SHADER_OPCODE_SHUFFLE:
2860 if (is_uniform(inst->src[0])) {
2861 inst->opcode = BRW_OPCODE_MOV;
2862 inst->sources = 1;
2863 progress = true;
2864 } else if (inst->src[1].file == IMM) {
2865 inst->opcode = BRW_OPCODE_MOV;
2866 inst->src[0] = component(inst->src[0],
2867 inst->src[1].ud);
2868 inst->sources = 1;
2869 progress = true;
2870 }
2871 break;
2872
2873 default:
2874 break;
2875 }
2876
2877 /* Swap if src[0] is immediate. */
2878 if (progress && inst->is_commutative()) {
2879 if (inst->src[0].file == IMM) {
2880 fs_reg tmp = inst->src[1];
2881 inst->src[1] = inst->src[0];
2882 inst->src[0] = tmp;
2883 }
2884 }
2885 }
2886
2887 if (progress)
2888 invalidate_analysis(DEPENDENCY_INSTRUCTION_DATA_FLOW |
2889 DEPENDENCY_INSTRUCTION_DETAIL);
2890
2891 return progress;
2892 }
2893
2894 /**
2895 * Optimize sample messages that have constant zero values for the trailing
2896 * texture coordinates. We can just reduce the message length for these
2897 * instructions instead of reserving a register for it. Trailing parameters
2898 * that aren't sent default to zero anyway. This will cause the dead code
2899 * eliminator to remove the MOV instruction that would otherwise be emitted to
2900 * set up the zero value.
2901 */
2902 bool
2903 fs_visitor::opt_zero_samples()
2904 {
2905 /* Gen4 infers the texturing opcode based on the message length so we can't
2906 * change it.
2907 */
2908 if (devinfo->gen < 5)
2909 return false;
2910
2911 bool progress = false;
2912
2913 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2914 if (!inst->is_tex())
2915 continue;
2916
2917 fs_inst *load_payload = (fs_inst *) inst->prev;
2918
2919 if (load_payload->is_head_sentinel() ||
2920 load_payload->opcode != SHADER_OPCODE_LOAD_PAYLOAD)
2921 continue;
2922
2923 /* We don't want to remove the message header or the first parameter.
2924 * Removing the first parameter is not allowed, see the Haswell PRM
2925 * volume 7, page 149:
2926 *
2927 * "Parameter 0 is required except for the sampleinfo message, which
2928 * has no parameter 0"
2929 */
2930 while (inst->mlen > inst->header_size + inst->exec_size / 8 &&
2931 load_payload->src[(inst->mlen - inst->header_size) /
2932 (inst->exec_size / 8) +
2933 inst->header_size - 1].is_zero()) {
2934 inst->mlen -= inst->exec_size / 8;
2935 progress = true;
2936 }
2937 }
2938
2939 if (progress)
2940 invalidate_analysis(DEPENDENCY_INSTRUCTION_DETAIL);
2941
2942 return progress;
2943 }
2944
2945 bool
2946 fs_visitor::opt_register_renaming()
2947 {
2948 bool progress = false;
2949 int depth = 0;
2950
2951 unsigned remap[alloc.count];
2952 memset(remap, ~0u, sizeof(unsigned) * alloc.count);
2953
2954 foreach_block_and_inst(block, fs_inst, inst, cfg) {
2955 if (inst->opcode == BRW_OPCODE_IF || inst->opcode == BRW_OPCODE_DO) {
2956 depth++;
2957 } else if (inst->opcode == BRW_OPCODE_ENDIF ||
2958 inst->opcode == BRW_OPCODE_WHILE) {
2959 depth--;
2960 }
2961
2962 /* Rewrite instruction sources. */
2963 for (int i = 0; i < inst->sources; i++) {
2964 if (inst->src[i].file == VGRF &&
2965 remap[inst->src[i].nr] != ~0u &&
2966 remap[inst->src[i].nr] != inst->src[i].nr) {
2967 inst->src[i].nr = remap[inst->src[i].nr];
2968 progress = true;
2969 }
2970 }
2971
2972 const unsigned dst = inst->dst.nr;
2973
2974 if (depth == 0 &&
2975 inst->dst.file == VGRF &&
2976 alloc.sizes[inst->dst.nr] * REG_SIZE == inst->size_written &&
2977 !inst->is_partial_write()) {
2978 if (remap[dst] == ~0u) {
2979 remap[dst] = dst;
2980 } else {
2981 remap[dst] = alloc.allocate(regs_written(inst));
2982 inst->dst.nr = remap[dst];
2983 progress = true;
2984 }
2985 } else if (inst->dst.file == VGRF &&
2986 remap[dst] != ~0u &&
2987 remap[dst] != dst) {
2988 inst->dst.nr = remap[dst];
2989 progress = true;
2990 }
2991 }
2992
2993 if (progress) {
2994 invalidate_analysis(DEPENDENCY_INSTRUCTION_DETAIL |
2995 DEPENDENCY_VARIABLES);
2996
2997 for (unsigned i = 0; i < ARRAY_SIZE(delta_xy); i++) {
2998 if (delta_xy[i].file == VGRF && remap[delta_xy[i].nr] != ~0u) {
2999 delta_xy[i].nr = remap[delta_xy[i].nr];
3000 }
3001 }
3002 }
3003
3004 return progress;
3005 }
3006
3007 /**
3008 * Remove redundant or useless discard jumps.
3009 *
3010 * For example, we can eliminate jumps in the following sequence:
3011 *
3012 * discard-jump (redundant with the next jump)
3013 * discard-jump (useless; jumps to the next instruction)
3014 * placeholder-halt
3015 */
3016 bool
3017 fs_visitor::opt_redundant_discard_jumps()
3018 {
3019 bool progress = false;
3020
3021 bblock_t *last_bblock = cfg->blocks[cfg->num_blocks - 1];
3022
3023 fs_inst *placeholder_halt = NULL;
3024 foreach_inst_in_block_reverse(fs_inst, inst, last_bblock) {
3025 if (inst->opcode == FS_OPCODE_PLACEHOLDER_HALT) {
3026 placeholder_halt = inst;
3027 break;
3028 }
3029 }
3030
3031 if (!placeholder_halt)
3032 return false;
3033
3034 /* Delete any HALTs immediately before the placeholder halt. */
3035 for (fs_inst *prev = (fs_inst *) placeholder_halt->prev;
3036 !prev->is_head_sentinel() && prev->opcode == FS_OPCODE_DISCARD_JUMP;
3037 prev = (fs_inst *) placeholder_halt->prev) {
3038 prev->remove(last_bblock);
3039 progress = true;
3040 }
3041
3042 if (progress)
3043 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
3044
3045 return progress;
3046 }
3047
3048 /**
3049 * Compute a bitmask with GRF granularity with a bit set for each GRF starting
3050 * from \p r.offset which overlaps the region starting at \p s.offset and
3051 * spanning \p ds bytes.
3052 */
3053 static inline unsigned
3054 mask_relative_to(const fs_reg &r, const fs_reg &s, unsigned ds)
3055 {
3056 const int rel_offset = reg_offset(s) - reg_offset(r);
3057 const int shift = rel_offset / REG_SIZE;
3058 const unsigned n = DIV_ROUND_UP(rel_offset % REG_SIZE + ds, REG_SIZE);
3059 assert(reg_space(r) == reg_space(s) &&
3060 shift >= 0 && shift < int(8 * sizeof(unsigned)));
3061 return ((1 << n) - 1) << shift;
3062 }
3063
3064 bool
3065 fs_visitor::compute_to_mrf()
3066 {
3067 bool progress = false;
3068 int next_ip = 0;
3069
3070 /* No MRFs on Gen >= 7. */
3071 if (devinfo->gen >= 7)
3072 return false;
3073
3074 const fs_live_variables &live = live_analysis.require();
3075
3076 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
3077 int ip = next_ip;
3078 next_ip++;
3079
3080 if (inst->opcode != BRW_OPCODE_MOV ||
3081 inst->is_partial_write() ||
3082 inst->dst.file != MRF || inst->src[0].file != VGRF ||
3083 inst->dst.type != inst->src[0].type ||
3084 inst->src[0].abs || inst->src[0].negate ||
3085 !inst->src[0].is_contiguous() ||
3086 inst->src[0].offset % REG_SIZE != 0)
3087 continue;
3088
3089 /* Can't compute-to-MRF this GRF if someone else was going to
3090 * read it later.
3091 */
3092 if (live.vgrf_end[inst->src[0].nr] > ip)
3093 continue;
3094
3095 /* Found a move of a GRF to a MRF. Let's see if we can go rewrite the
3096 * things that computed the value of all GRFs of the source region. The
3097 * regs_left bitset keeps track of the registers we haven't yet found a
3098 * generating instruction for.
3099 */
3100 unsigned regs_left = (1 << regs_read(inst, 0)) - 1;
3101
3102 foreach_inst_in_block_reverse_starting_from(fs_inst, scan_inst, inst) {
3103 if (regions_overlap(scan_inst->dst, scan_inst->size_written,
3104 inst->src[0], inst->size_read(0))) {
3105 /* Found the last thing to write our reg we want to turn
3106 * into a compute-to-MRF.
3107 */
3108
3109 /* If this one instruction didn't populate all the
3110 * channels, bail. We might be able to rewrite everything
3111 * that writes that reg, but it would require smarter
3112 * tracking.
3113 */
3114 if (scan_inst->is_partial_write())
3115 break;
3116
3117 /* Handling things not fully contained in the source of the copy
3118 * would need us to understand coalescing out more than one MOV at
3119 * a time.
3120 */
3121 if (!region_contained_in(scan_inst->dst, scan_inst->size_written,
3122 inst->src[0], inst->size_read(0)))
3123 break;
3124
3125 /* SEND instructions can't have MRF as a destination. */
3126 if (scan_inst->mlen)
3127 break;
3128
3129 if (devinfo->gen == 6) {
3130 /* gen6 math instructions must have the destination be
3131 * GRF, so no compute-to-MRF for them.
3132 */
3133 if (scan_inst->is_math()) {
3134 break;
3135 }
3136 }
3137
3138 /* Clear the bits for any registers this instruction overwrites. */
3139 regs_left &= ~mask_relative_to(
3140 inst->src[0], scan_inst->dst, scan_inst->size_written);
3141 if (!regs_left)
3142 break;
3143 }
3144
3145 /* We don't handle control flow here. Most computation of
3146 * values that end up in MRFs are shortly before the MRF
3147 * write anyway.
3148 */
3149 if (block->start() == scan_inst)
3150 break;
3151
3152 /* You can't read from an MRF, so if someone else reads our
3153 * MRF's source GRF that we wanted to rewrite, that stops us.
3154 */
3155 bool interfered = false;
3156 for (int i = 0; i < scan_inst->sources; i++) {
3157 if (regions_overlap(scan_inst->src[i], scan_inst->size_read(i),
3158 inst->src[0], inst->size_read(0))) {
3159 interfered = true;
3160 }
3161 }
3162 if (interfered)
3163 break;
3164
3165 if (regions_overlap(scan_inst->dst, scan_inst->size_written,
3166 inst->dst, inst->size_written)) {
3167 /* If somebody else writes our MRF here, we can't
3168 * compute-to-MRF before that.
3169 */
3170 break;
3171 }
3172
3173 if (scan_inst->mlen > 0 && scan_inst->base_mrf != -1 &&
3174 regions_overlap(fs_reg(MRF, scan_inst->base_mrf), scan_inst->mlen * REG_SIZE,
3175 inst->dst, inst->size_written)) {
3176 /* Found a SEND instruction, which means that there are
3177 * live values in MRFs from base_mrf to base_mrf +
3178 * scan_inst->mlen - 1. Don't go pushing our MRF write up
3179 * above it.
3180 */
3181 break;
3182 }
3183 }
3184
3185 if (regs_left)
3186 continue;
3187
3188 /* Found all generating instructions of our MRF's source value, so it
3189 * should be safe to rewrite them to point to the MRF directly.
3190 */
3191 regs_left = (1 << regs_read(inst, 0)) - 1;
3192
3193 foreach_inst_in_block_reverse_starting_from(fs_inst, scan_inst, inst) {
3194 if (regions_overlap(scan_inst->dst, scan_inst->size_written,
3195 inst->src[0], inst->size_read(0))) {
3196 /* Clear the bits for any registers this instruction overwrites. */
3197 regs_left &= ~mask_relative_to(
3198 inst->src[0], scan_inst->dst, scan_inst->size_written);
3199
3200 const unsigned rel_offset = reg_offset(scan_inst->dst) -
3201 reg_offset(inst->src[0]);
3202
3203 if (inst->dst.nr & BRW_MRF_COMPR4) {
3204 /* Apply the same address transformation done by the hardware
3205 * for COMPR4 MRF writes.
3206 */
3207 assert(rel_offset < 2 * REG_SIZE);
3208 scan_inst->dst.nr = inst->dst.nr + rel_offset / REG_SIZE * 4;
3209
3210 /* Clear the COMPR4 bit if the generating instruction is not
3211 * compressed.
3212 */
3213 if (scan_inst->size_written < 2 * REG_SIZE)
3214 scan_inst->dst.nr &= ~BRW_MRF_COMPR4;
3215
3216 } else {
3217 /* Calculate the MRF number the result of this instruction is
3218 * ultimately written to.
3219 */
3220 scan_inst->dst.nr = inst->dst.nr + rel_offset / REG_SIZE;
3221 }
3222
3223 scan_inst->dst.file = MRF;
3224 scan_inst->dst.offset = inst->dst.offset + rel_offset % REG_SIZE;
3225 scan_inst->saturate |= inst->saturate;
3226 if (!regs_left)
3227 break;
3228 }
3229 }
3230
3231 assert(!regs_left);
3232 inst->remove(block);
3233 progress = true;
3234 }
3235
3236 if (progress)
3237 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
3238
3239 return progress;
3240 }
3241
3242 /**
3243 * Eliminate FIND_LIVE_CHANNEL instructions occurring outside any control
3244 * flow. We could probably do better here with some form of divergence
3245 * analysis.
3246 */
3247 bool
3248 fs_visitor::eliminate_find_live_channel()
3249 {
3250 bool progress = false;
3251 unsigned depth = 0;
3252
3253 if (!brw_stage_has_packed_dispatch(devinfo, stage, stage_prog_data)) {
3254 /* The optimization below assumes that channel zero is live on thread
3255 * dispatch, which may not be the case if the fixed function dispatches
3256 * threads sparsely.
3257 */
3258 return false;
3259 }
3260
3261 foreach_block_and_inst_safe(block, fs_inst, inst, cfg) {
3262 switch (inst->opcode) {
3263 case BRW_OPCODE_IF:
3264 case BRW_OPCODE_DO:
3265 depth++;
3266 break;
3267
3268 case BRW_OPCODE_ENDIF:
3269 case BRW_OPCODE_WHILE:
3270 depth--;
3271 break;
3272
3273 case FS_OPCODE_DISCARD_JUMP:
3274 /* This can potentially make control flow non-uniform until the end
3275 * of the program.
3276 */
3277 return progress;
3278
3279 case SHADER_OPCODE_FIND_LIVE_CHANNEL:
3280 if (depth == 0) {
3281 inst->opcode = BRW_OPCODE_MOV;
3282 inst->src[0] = brw_imm_ud(0u);
3283 inst->sources = 1;
3284 inst->force_writemask_all = true;
3285 progress = true;
3286 }
3287 break;
3288
3289 default:
3290 break;
3291 }
3292 }
3293
3294 if (progress)
3295 invalidate_analysis(DEPENDENCY_INSTRUCTION_DETAIL);
3296
3297 return progress;
3298 }
3299
3300 /**
3301 * Once we've generated code, try to convert normal FS_OPCODE_FB_WRITE
3302 * instructions to FS_OPCODE_REP_FB_WRITE.
3303 */
3304 void
3305 fs_visitor::emit_repclear_shader()
3306 {
3307 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
3308 int base_mrf = 0;
3309 int color_mrf = base_mrf + 2;
3310 fs_inst *mov;
3311
3312 if (uniforms > 0) {
3313 mov = bld.exec_all().group(4, 0)
3314 .MOV(brw_message_reg(color_mrf),
3315 fs_reg(UNIFORM, 0, BRW_REGISTER_TYPE_F));
3316 } else {
3317 struct brw_reg reg =
3318 brw_reg(BRW_GENERAL_REGISTER_FILE, 2, 3, 0, 0, BRW_REGISTER_TYPE_F,
3319 BRW_VERTICAL_STRIDE_8, BRW_WIDTH_2, BRW_HORIZONTAL_STRIDE_4,
3320 BRW_SWIZZLE_XYZW, WRITEMASK_XYZW);
3321
3322 mov = bld.exec_all().group(4, 0)
3323 .MOV(vec4(brw_message_reg(color_mrf)), fs_reg(reg));
3324 }
3325
3326 fs_inst *write = NULL;
3327 if (key->nr_color_regions == 1) {
3328 write = bld.emit(FS_OPCODE_REP_FB_WRITE);
3329 write->saturate = key->clamp_fragment_color;
3330 write->base_mrf = color_mrf;
3331 write->target = 0;
3332 write->header_size = 0;
3333 write->mlen = 1;
3334 } else {
3335 assume(key->nr_color_regions > 0);
3336
3337 struct brw_reg header =
3338 retype(brw_message_reg(base_mrf), BRW_REGISTER_TYPE_UD);
3339 bld.exec_all().group(16, 0)
3340 .MOV(header, retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
3341
3342 for (int i = 0; i < key->nr_color_regions; ++i) {
3343 if (i > 0) {
3344 bld.exec_all().group(1, 0)
3345 .MOV(component(header, 2), brw_imm_ud(i));
3346 }
3347
3348 write = bld.emit(FS_OPCODE_REP_FB_WRITE);
3349 write->saturate = key->clamp_fragment_color;
3350 write->base_mrf = base_mrf;
3351 write->target = i;
3352 write->header_size = 2;
3353 write->mlen = 3;
3354 }
3355 }
3356 write->eot = true;
3357 write->last_rt = true;
3358
3359 calculate_cfg();
3360
3361 assign_constant_locations();
3362 assign_curb_setup();
3363
3364 /* Now that we have the uniform assigned, go ahead and force it to a vec4. */
3365 if (uniforms > 0) {
3366 assert(mov->src[0].file == FIXED_GRF);
3367 mov->src[0] = brw_vec4_grf(mov->src[0].nr, 0);
3368 }
3369
3370 lower_scoreboard();
3371 }
3372
3373 /**
3374 * Walks through basic blocks, looking for repeated MRF writes and
3375 * removing the later ones.
3376 */
3377 bool
3378 fs_visitor::remove_duplicate_mrf_writes()
3379 {
3380 fs_inst *last_mrf_move[BRW_MAX_MRF(devinfo->gen)];
3381 bool progress = false;
3382
3383 /* Need to update the MRF tracking for compressed instructions. */
3384 if (dispatch_width >= 16)
3385 return false;
3386
3387 memset(last_mrf_move, 0, sizeof(last_mrf_move));
3388
3389 foreach_block_and_inst_safe (block, fs_inst, inst, cfg) {
3390 if (inst->is_control_flow()) {
3391 memset(last_mrf_move, 0, sizeof(last_mrf_move));
3392 }
3393
3394 if (inst->opcode == BRW_OPCODE_MOV &&
3395 inst->dst.file == MRF) {
3396 fs_inst *prev_inst = last_mrf_move[inst->dst.nr];
3397 if (prev_inst && prev_inst->opcode == BRW_OPCODE_MOV &&
3398 inst->dst.equals(prev_inst->dst) &&
3399 inst->src[0].equals(prev_inst->src[0]) &&
3400 inst->saturate == prev_inst->saturate &&
3401 inst->predicate == prev_inst->predicate &&
3402 inst->conditional_mod == prev_inst->conditional_mod &&
3403 inst->exec_size == prev_inst->exec_size) {
3404 inst->remove(block);
3405 progress = true;
3406 continue;
3407 }
3408 }
3409
3410 /* Clear out the last-write records for MRFs that were overwritten. */
3411 if (inst->dst.file == MRF) {
3412 last_mrf_move[inst->dst.nr] = NULL;
3413 }
3414
3415 if (inst->mlen > 0 && inst->base_mrf != -1) {
3416 /* Found a SEND instruction, which will include two or fewer
3417 * implied MRF writes. We could do better here.
3418 */
3419 for (unsigned i = 0; i < inst->implied_mrf_writes(); i++) {
3420 last_mrf_move[inst->base_mrf + i] = NULL;
3421 }
3422 }
3423
3424 /* Clear out any MRF move records whose sources got overwritten. */
3425 for (unsigned i = 0; i < ARRAY_SIZE(last_mrf_move); i++) {
3426 if (last_mrf_move[i] &&
3427 regions_overlap(inst->dst, inst->size_written,
3428 last_mrf_move[i]->src[0],
3429 last_mrf_move[i]->size_read(0))) {
3430 last_mrf_move[i] = NULL;
3431 }
3432 }
3433
3434 if (inst->opcode == BRW_OPCODE_MOV &&
3435 inst->dst.file == MRF &&
3436 inst->src[0].file != ARF &&
3437 !inst->is_partial_write()) {
3438 last_mrf_move[inst->dst.nr] = inst;
3439 }
3440 }
3441
3442 if (progress)
3443 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
3444
3445 return progress;
3446 }
3447
3448 /**
3449 * Rounding modes for conversion instructions are included for each
3450 * conversion, but right now it is a state. So once it is set,
3451 * we don't need to call it again for subsequent calls.
3452 *
3453 * This is useful for vector/matrices conversions, as setting the
3454 * mode once is enough for the full vector/matrix
3455 */
3456 bool
3457 fs_visitor::remove_extra_rounding_modes()
3458 {
3459 bool progress = false;
3460 unsigned execution_mode = this->nir->info.float_controls_execution_mode;
3461
3462 brw_rnd_mode base_mode = BRW_RND_MODE_UNSPECIFIED;
3463 if ((FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 |
3464 FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32 |
3465 FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64) &
3466 execution_mode)
3467 base_mode = BRW_RND_MODE_RTNE;
3468 if ((FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 |
3469 FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 |
3470 FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64) &
3471 execution_mode)
3472 base_mode = BRW_RND_MODE_RTZ;
3473
3474 foreach_block (block, cfg) {
3475 brw_rnd_mode prev_mode = base_mode;
3476
3477 foreach_inst_in_block_safe (fs_inst, inst, block) {
3478 if (inst->opcode == SHADER_OPCODE_RND_MODE) {
3479 assert(inst->src[0].file == BRW_IMMEDIATE_VALUE);
3480 const brw_rnd_mode mode = (brw_rnd_mode) inst->src[0].d;
3481 if (mode == prev_mode) {
3482 inst->remove(block);
3483 progress = true;
3484 } else {
3485 prev_mode = mode;
3486 }
3487 }
3488 }
3489 }
3490
3491 if (progress)
3492 invalidate_analysis(DEPENDENCY_INSTRUCTIONS);
3493
3494 return progress;
3495 }
3496
3497 static void
3498 clear_deps_for_inst_src(fs_inst *inst, bool *deps, int first_grf, int grf_len)
3499 {
3500 /* Clear the flag for registers that actually got read (as expected). */
3501 for (int i = 0; i < inst->sources; i++) {
3502 int grf;
3503 if (inst->src[i].file == VGRF || inst->src[i].file == FIXED_GRF) {
3504 grf = inst->src[i].nr;
3505 } else {
3506 continue;
3507 }
3508
3509 if (grf >= first_grf &&
3510 grf < first_grf + grf_len) {
3511 deps[grf - first_grf] = false;
3512 if (inst->exec_size == 16)
3513 deps[grf - first_grf + 1] = false;
3514 }
3515 }
3516 }
3517
3518 /**
3519 * Implements this workaround for the original 965:
3520 *
3521 * "[DevBW, DevCL] Implementation Restrictions: As the hardware does not
3522 * check for post destination dependencies on this instruction, software
3523 * must ensure that there is no destination hazard for the case of ‘write
3524 * followed by a posted write’ shown in the following example.
3525 *
3526 * 1. mov r3 0
3527 * 2. send r3.xy <rest of send instruction>
3528 * 3. mov r2 r3
3529 *
3530 * Due to no post-destination dependency check on the ‘send’, the above
3531 * code sequence could have two instructions (1 and 2) in flight at the
3532 * same time that both consider ‘r3’ as the target of their final writes.
3533 */
3534 void
3535 fs_visitor::insert_gen4_pre_send_dependency_workarounds(bblock_t *block,
3536 fs_inst *inst)
3537 {
3538 int write_len = regs_written(inst);
3539 int first_write_grf = inst->dst.nr;
3540 bool needs_dep[BRW_MAX_MRF(devinfo->gen)];
3541 assert(write_len < (int)sizeof(needs_dep) - 1);
3542
3543 memset(needs_dep, false, sizeof(needs_dep));
3544 memset(needs_dep, true, write_len);
3545
3546 clear_deps_for_inst_src(inst, needs_dep, first_write_grf, write_len);
3547
3548 /* Walk backwards looking for writes to registers we're writing which
3549 * aren't read since being written. If we hit the start of the program,
3550 * we assume that there are no outstanding dependencies on entry to the
3551 * program.
3552 */
3553 foreach_inst_in_block_reverse_starting_from(fs_inst, scan_inst, inst) {
3554 /* If we hit control flow, assume that there *are* outstanding
3555 * dependencies, and force their cleanup before our instruction.
3556 */
3557 if (block->start() == scan_inst && block->num != 0) {
3558 for (int i = 0; i < write_len; i++) {
3559 if (needs_dep[i])
3560 DEP_RESOLVE_MOV(fs_builder(this, block, inst),
3561 first_write_grf + i);
3562 }
3563 return;
3564 }
3565
3566 /* We insert our reads as late as possible on the assumption that any
3567 * instruction but a MOV that might have left us an outstanding
3568 * dependency has more latency than a MOV.
3569 */
3570 if (scan_inst->dst.file == VGRF) {
3571 for (unsigned i = 0; i < regs_written(scan_inst); i++) {
3572 int reg = scan_inst->dst.nr + i;
3573
3574 if (reg >= first_write_grf &&
3575 reg < first_write_grf + write_len &&
3576 needs_dep[reg - first_write_grf]) {
3577 DEP_RESOLVE_MOV(fs_builder(this, block, inst), reg);
3578 needs_dep[reg - first_write_grf] = false;
3579 if (scan_inst->exec_size == 16)
3580 needs_dep[reg - first_write_grf + 1] = false;
3581 }
3582 }
3583 }
3584
3585 /* Clear the flag for registers that actually got read (as expected). */
3586 clear_deps_for_inst_src(scan_inst, needs_dep, first_write_grf, write_len);
3587
3588 /* Continue the loop only if we haven't resolved all the dependencies */
3589 int i;
3590 for (i = 0; i < write_len; i++) {
3591 if (needs_dep[i])
3592 break;
3593 }
3594 if (i == write_len)
3595 return;
3596 }
3597 }
3598
3599 /**
3600 * Implements this workaround for the original 965:
3601 *
3602 * "[DevBW, DevCL] Errata: A destination register from a send can not be
3603 * used as a destination register until after it has been sourced by an
3604 * instruction with a different destination register.
3605 */
3606 void
3607 fs_visitor::insert_gen4_post_send_dependency_workarounds(bblock_t *block, fs_inst *inst)
3608 {
3609 int write_len = regs_written(inst);
3610 unsigned first_write_grf = inst->dst.nr;
3611 bool needs_dep[BRW_MAX_MRF(devinfo->gen)];
3612 assert(write_len < (int)sizeof(needs_dep) - 1);
3613
3614 memset(needs_dep, false, sizeof(needs_dep));
3615 memset(needs_dep, true, write_len);
3616 /* Walk forwards looking for writes to registers we're writing which aren't
3617 * read before being written.
3618 */
3619 foreach_inst_in_block_starting_from(fs_inst, scan_inst, inst) {
3620 /* If we hit control flow, force resolve all remaining dependencies. */
3621 if (block->end() == scan_inst && block->num != cfg->num_blocks - 1) {
3622 for (int i = 0; i < write_len; i++) {
3623 if (needs_dep[i])
3624 DEP_RESOLVE_MOV(fs_builder(this, block, scan_inst),
3625 first_write_grf + i);
3626 }
3627 return;
3628 }
3629
3630 /* Clear the flag for registers that actually got read (as expected). */
3631 clear_deps_for_inst_src(scan_inst, needs_dep, first_write_grf, write_len);
3632
3633 /* We insert our reads as late as possible since they're reading the
3634 * result of a SEND, which has massive latency.
3635 */
3636 if (scan_inst->dst.file == VGRF &&
3637 scan_inst->dst.nr >= first_write_grf &&
3638 scan_inst->dst.nr < first_write_grf + write_len &&
3639 needs_dep[scan_inst->dst.nr - first_write_grf]) {
3640 DEP_RESOLVE_MOV(fs_builder(this, block, scan_inst),
3641 scan_inst->dst.nr);
3642 needs_dep[scan_inst->dst.nr - first_write_grf] = false;