projects / mesa.git / blob
? search:


ea92f6b85779b304e6221dd2e2e8c9e6360fd75e
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_cp.c
1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include "freedreno_util.h"
30
31 #include "ir3.h"
32 #include "ir3_shader.h"
33
34 /*
35 * Copy Propagate:
36 */
37
38 struct ir3_cp_ctx {
39 struct ir3 *shader;
40 struct ir3_shader_variant *so;
41 unsigned immediate_idx;
42 };
43
44 /* is it a type preserving mov, with ok flags? */
45 static bool is_eligible_mov(struct ir3_instruction *instr, bool allow_flags)
46 {
47 if (is_same_type_mov(instr)) {
48 struct ir3_register *dst = instr->regs[0];
49 struct ir3_register *src = instr->regs[1];
50 struct ir3_instruction *src_instr = ssa(src);
51
52 /* only if mov src is SSA (not const/immed): */
53 if (!src_instr)
54 return false;
55
56 /* no indirect: */
57 if (dst->flags & IR3_REG_RELATIV)
58 return false;
59 if (src->flags & IR3_REG_RELATIV)
60 return false;
61
62 if (src->flags & IR3_REG_ARRAY)
63 return false;
64
65 if (!allow_flags)
66 if (src->flags & (IR3_REG_FABS | IR3_REG_FNEG |
67 IR3_REG_SABS | IR3_REG_SNEG | IR3_REG_BNOT))
68 return false;
69
70 /* TODO: remove this hack: */
71 if (src_instr->opc == OPC_META_FO)
72 return false;
73
74 return true;
75 }
76 return false;
77 }
78
79 static unsigned cp_flags(unsigned flags)
80 {
81 /* only considering these flags (at least for now): */
82 flags &= (IR3_REG_CONST | IR3_REG_IMMED |
83 IR3_REG_FNEG | IR3_REG_FABS |
84 IR3_REG_SNEG | IR3_REG_SABS |
85 IR3_REG_BNOT | IR3_REG_RELATIV);
86 return flags;
87 }
88
89 static bool valid_flags(struct ir3_instruction *instr, unsigned n,
90 unsigned flags)
91 {
92 unsigned valid_flags;
93 flags = cp_flags(flags);
94
95 /* If destination is indirect, then source cannot be.. at least
96 * I don't think so..
97 */
98 if ((instr->regs[0]->flags & IR3_REG_RELATIV) &&
99 (flags & IR3_REG_RELATIV))
100 return false;
101
102 /* TODO it seems to *mostly* work to cp RELATIV, except we get some
103 * intermittent piglit variable-indexing fails. Newer blob driver
104 * doesn't seem to cp these. Possibly this is hw workaround? Not
105 * sure, but until that is understood better, lets just switch off
106 * cp for indirect src's:
107 */
108 if (flags & IR3_REG_RELATIV)
109 return false;
110
111 switch (opc_cat(instr->opc)) {
112 case 1:
113 valid_flags = IR3_REG_IMMED | IR3_REG_CONST | IR3_REG_RELATIV;
114 if (flags & ~valid_flags)
115 return false;
116 break;
117 case 2:
118 valid_flags = ir3_cat2_absneg(instr->opc) |
119 IR3_REG_CONST | IR3_REG_RELATIV;
120
121 if (ir3_cat2_int(instr->opc))
122 valid_flags |= IR3_REG_IMMED;
123
124 if (flags & ~valid_flags)
125 return false;
126
127 if (flags & (IR3_REG_CONST | IR3_REG_IMMED)) {
128 unsigned m = (n ^ 1) + 1;
129 /* cannot deal w/ const in both srcs:
130 * (note that some cat2 actually only have a single src)
131 */
132 if (m < instr->regs_count) {
133 struct ir3_register *reg = instr->regs[m];
134 if ((flags & IR3_REG_CONST) && (reg->flags & IR3_REG_CONST))
135 return false;
136 if ((flags & IR3_REG_IMMED) && (reg->flags & IR3_REG_IMMED))
137 return false;
138 }
139 /* cannot be const + ABS|NEG: */
140 if (flags & (IR3_REG_FABS | IR3_REG_FNEG |
141 IR3_REG_SABS | IR3_REG_SNEG | IR3_REG_BNOT))
142 return false;
143 }
144 break;
145 case 3:
146 valid_flags = ir3_cat3_absneg(instr->opc) |
147 IR3_REG_CONST | IR3_REG_RELATIV;
148
149 if (flags & ~valid_flags)
150 return false;
151
152 if (flags & (IR3_REG_CONST | IR3_REG_RELATIV)) {
153 /* cannot deal w/ const/relativ in 2nd src: */
154 if (n == 1)
155 return false;
156 }
157
158 if (flags & IR3_REG_CONST) {
159 /* cannot be const + ABS|NEG: */
160 if (flags & (IR3_REG_FABS | IR3_REG_FNEG |
161 IR3_REG_SABS | IR3_REG_SNEG | IR3_REG_BNOT))
162 return false;
163 }
164 break;
165 case 4:
166 /* seems like blob compiler avoids const as src.. */
167 /* TODO double check if this is still the case on a4xx */
168 if (flags & (IR3_REG_CONST | IR3_REG_IMMED))
169 return false;
170 if (flags & (IR3_REG_SABS | IR3_REG_SNEG))
171 return false;
172 break;
173 case 5:
174 /* no flags allowed */
175 if (flags)
176 return false;
177 break;
178 case 6:
179 valid_flags = IR3_REG_IMMED;
180 if (flags & ~valid_flags)
181 return false;
182
183 if (flags & IR3_REG_IMMED) {
184 /* doesn't seem like we can have immediate src for store
185 * instructions:
186 *
187 * TODO this restriction could also apply to load instructions,
188 * but for load instructions this arg is the address (and not
189 * really sure any good way to test a hard-coded immed addr src)
190 */
191 if (is_store(instr) && (n == 1))
192 return false;
193
194 if ((instr->opc == OPC_LDL) && (n != 1))
195 return false;
196
197 if ((instr->opc == OPC_STL) && (n != 2))
198 return false;
199
200 /* disallow CP into anything but the SSBO slot argument for
201 * atomics:
202 */
203 if (is_atomic(instr->opc) && (n != 0))
204 return false;
205
206 if (is_atomic(instr->opc) && !(instr->flags & IR3_INSTR_G))
207 return false;
208 }
209
210 break;
211 }
212
213 return true;
214 }
215
216 /* propagate register flags from src to dst.. negates need special
217 * handling to cancel each other out.
218 */
219 static void combine_flags(unsigned *dstflags, struct ir3_instruction *src)
220 {
221 unsigned srcflags = src->regs[1]->flags;
222
223 /* if what we are combining into already has (abs) flags,
224 * we can drop (neg) from src:
225 */
226 if (*dstflags & IR3_REG_FABS)
227 srcflags &= ~IR3_REG_FNEG;
228 if (*dstflags & IR3_REG_SABS)
229 srcflags &= ~IR3_REG_SNEG;
230
231 if (srcflags & IR3_REG_FABS)
232 *dstflags |= IR3_REG_FABS;
233 if (srcflags & IR3_REG_SABS)
234 *dstflags |= IR3_REG_SABS;
235 if (srcflags & IR3_REG_FNEG)
236 *dstflags ^= IR3_REG_FNEG;
237 if (srcflags & IR3_REG_SNEG)
238 *dstflags ^= IR3_REG_SNEG;
239 if (srcflags & IR3_REG_BNOT)
240 *dstflags ^= IR3_REG_BNOT;
241
242 *dstflags &= ~IR3_REG_SSA;
243 *dstflags |= srcflags & IR3_REG_SSA;
244 *dstflags |= srcflags & IR3_REG_CONST;
245 *dstflags |= srcflags & IR3_REG_IMMED;
246 *dstflags |= srcflags & IR3_REG_RELATIV;
247 *dstflags |= srcflags & IR3_REG_ARRAY;
248
249 /* if src of the src is boolean we can drop the (abs) since we know
250 * the source value is already a postitive integer. This cleans
251 * up the absnegs that get inserted when converting between nir and
252 * native boolean (see ir3_b2n/n2b)
253 */
254 struct ir3_instruction *srcsrc = ssa(src->regs[1]);
255 if (srcsrc && is_bool(srcsrc))
256 *dstflags &= ~IR3_REG_SABS;
257 }
258
259 static struct ir3_register *
260 lower_immed(struct ir3_cp_ctx *ctx, struct ir3_register *reg, unsigned new_flags)
261 {
262 unsigned swiz, idx, i;
263
264 reg = ir3_reg_clone(ctx->shader, reg);
265
266 /* in some cases, there are restrictions on (abs)/(neg) plus const..
267 * so just evaluate those and clear the flags:
268 */
269 if (new_flags & IR3_REG_SABS) {
270 reg->iim_val = abs(reg->iim_val);
271 new_flags &= ~IR3_REG_SABS;
272 }
273
274 if (new_flags & IR3_REG_FABS) {
275 reg->fim_val = fabs(reg->fim_val);
276 new_flags &= ~IR3_REG_FABS;
277 }
278
279 if (new_flags & IR3_REG_SNEG) {
280 reg->iim_val = -reg->iim_val;
281 new_flags &= ~IR3_REG_SNEG;
282 }
283
284 if (new_flags & IR3_REG_FNEG) {
285 reg->fim_val = -reg->fim_val;
286 new_flags &= ~IR3_REG_FNEG;
287 }
288
289 /* Reallocate for 4 more elements whenever it's necessary */
290 if (ctx->immediate_idx == ctx->so->immediates_size * 4) {
291 ctx->so->immediates_size += 4;
292 ctx->so->immediates = realloc (ctx->so->immediates,
293 ctx->so->immediates_size * sizeof (ctx->so->immediates[0]));
294 }
295
296 for (i = 0; i < ctx->immediate_idx; i++) {
297 swiz = i % 4;
298 idx = i / 4;
299
300 if (ctx->so->immediates[idx].val[swiz] == reg->uim_val) {
301 break;
302 }
303 }
304
305 if (i == ctx->immediate_idx) {
306 /* need to generate a new immediate: */
307 swiz = i % 4;
308 idx = i / 4;
309 ctx->so->immediates[idx].val[swiz] = reg->uim_val;
310 ctx->so->immediates_count = idx + 1;
311 ctx->immediate_idx++;
312 }
313
314 new_flags &= ~IR3_REG_IMMED;
315 new_flags |= IR3_REG_CONST;
316 reg->flags = new_flags;
317 reg->num = i + (4 * ctx->so->constbase.immediate);
318
319 return reg;
320 }
321
322 static void
323 unuse(struct ir3_instruction *instr)
324 {
325 debug_assert(instr->use_count > 0);
326
327 if (--instr->use_count == 0) {
328 struct ir3_block *block = instr->block;
329
330 instr->barrier_class = 0;
331 instr->barrier_conflict = 0;
332
333 /* we don't want to remove anything in keeps (which could
334 * be things like array store's)
335 */
336 for (unsigned i = 0; i < block->keeps_count; i++) {
337 debug_assert(block->keeps[i] != instr);
338 }
339 }
340 }
341
342 /**
343 * Handle cp for a given src register. This additionally handles
344 * the cases of collapsing immedate/const (which replace the src
345 * register with a non-ssa src) or collapsing mov's from relative
346 * src (which needs to also fixup the address src reference by the
347 * instruction).
348 */
349 static void
350 reg_cp(struct ir3_cp_ctx *ctx, struct ir3_instruction *instr,
351 struct ir3_register *reg, unsigned n)
352 {
353 struct ir3_instruction *src = ssa(reg);
354
355 if (is_eligible_mov(src, true)) {
356 /* simple case, no immed/const/relativ, only mov's w/ ssa src: */
357 struct ir3_register *src_reg = src->regs[1];
358 unsigned new_flags = reg->flags;
359
360 combine_flags(&new_flags, src);
361
362 if (valid_flags(instr, n, new_flags)) {
363 if (new_flags & IR3_REG_ARRAY) {
364 debug_assert(!(reg->flags & IR3_REG_ARRAY));
365 reg->array = src_reg->array;
366 }
367 reg->flags = new_flags;
368 reg->instr = ssa(src_reg);
369
370 instr->barrier_class |= src->barrier_class;
371 instr->barrier_conflict |= src->barrier_conflict;
372
373 unuse(src);
374 reg->instr->use_count++;
375 }
376
377 } else if (is_same_type_mov(src) &&
378 /* cannot collapse const/immed/etc into meta instrs: */
379 !is_meta(instr)) {
380 /* immed/const/etc cases, which require some special handling: */
381 struct ir3_register *src_reg = src->regs[1];
382 unsigned new_flags = reg->flags;
383
384 combine_flags(&new_flags, src);
385
386 if (!valid_flags(instr, n, new_flags)) {
387 /* See if lowering an immediate to const would help. */
388 if (valid_flags(instr, n, (new_flags & ~IR3_REG_IMMED) | IR3_REG_CONST)) {
389 debug_assert(new_flags & IR3_REG_IMMED);
390 instr->regs[n + 1] = lower_immed(ctx, src_reg, new_flags);
391 return;
392 }
393
394 /* special case for "normal" mad instructions, we can
395 * try swapping the first two args if that fits better.
396 *
397 * the "plain" MAD's (ie. the ones that don't shift first
398 * src prior to multiply) can swap their first two srcs if
399 * src[0] is !CONST and src[1] is CONST:
400 */
401 if ((n == 1) && is_mad(instr->opc) &&
402 !(instr->regs[0 + 1]->flags & (IR3_REG_CONST | IR3_REG_RELATIV)) &&
403 valid_flags(instr, 0, new_flags & ~IR3_REG_IMMED)) {
404 /* swap src[0] and src[1]: */
405 struct ir3_register *tmp;
406 tmp = instr->regs[0 + 1];
407 instr->regs[0 + 1] = instr->regs[1 + 1];
408 instr->regs[1 + 1] = tmp;
409
410 n = 0;
411 } else {
412 return;
413 }
414 }
415
416 /* Here we handle the special case of mov from
417 * CONST and/or RELATIV. These need to be handled
418 * specially, because in the case of move from CONST
419 * there is no src ir3_instruction so we need to
420 * replace the ir3_register. And in the case of
421 * RELATIV we need to handle the address register
422 * dependency.
423 */
424 if (src_reg->flags & IR3_REG_CONST) {
425 /* an instruction cannot reference two different
426 * address registers:
427 */
428 if ((src_reg->flags & IR3_REG_RELATIV) &&
429 conflicts(instr->address, reg->instr->address))
430 return;
431
432 /* This seems to be a hw bug, or something where the timings
433 * just somehow don't work out. This restriction may only
434 * apply if the first src is also CONST.
435 */
436 if ((opc_cat(instr->opc) == 3) && (n == 2) &&
437 (src_reg->flags & IR3_REG_RELATIV) &&
438 (src_reg->array.offset == 0))
439 return;
440
441 src_reg = ir3_reg_clone(instr->block->shader, src_reg);
442 src_reg->flags = new_flags;
443 instr->regs[n+1] = src_reg;
444
445 if (src_reg->flags & IR3_REG_RELATIV)
446 ir3_instr_set_address(instr, reg->instr->address);
447
448 return;
449 }
450
451 if ((src_reg->flags & IR3_REG_RELATIV) &&
452 !conflicts(instr->address, reg->instr->address)) {
453 src_reg = ir3_reg_clone(instr->block->shader, src_reg);
454 src_reg->flags = new_flags;
455 instr->regs[n+1] = src_reg;
456 ir3_instr_set_address(instr, reg->instr->address);
457
458 return;
459 }
460
461 /* NOTE: seems we can only do immed integers, so don't
462 * need to care about float. But we do need to handle
463 * abs/neg *before* checking that the immediate requires
464 * few enough bits to encode:
465 *
466 * TODO: do we need to do something to avoid accidentally
467 * catching a float immed?
468 */
469 if (src_reg->flags & IR3_REG_IMMED) {
470 int32_t iim_val = src_reg->iim_val;
471
472 debug_assert((opc_cat(instr->opc) == 1) ||
473 (opc_cat(instr->opc) == 6) ||
474 ir3_cat2_int(instr->opc) ||
475 (is_mad(instr->opc) && (n == 0)));
476
477 if (new_flags & IR3_REG_SABS)
478 iim_val = abs(iim_val);
479
480 if (new_flags & IR3_REG_SNEG)
481 iim_val = -iim_val;
482
483 if (new_flags & IR3_REG_BNOT)
484 iim_val = ~iim_val;
485
486 /* other than category 1 (mov) we can only encode up to 10 bits: */
487 if ((instr->opc == OPC_MOV) ||
488 !((iim_val & ~0x3ff) && (-iim_val & ~0x3ff))) {
489 new_flags &= ~(IR3_REG_SABS | IR3_REG_SNEG | IR3_REG_BNOT);
490 src_reg = ir3_reg_clone(instr->block->shader, src_reg);
491 src_reg->flags = new_flags;
492 src_reg->iim_val = iim_val;
493 instr->regs[n+1] = src_reg;
494 } else if (valid_flags(instr, n, (new_flags & ~IR3_REG_IMMED) | IR3_REG_CONST)) {
495 /* See if lowering an immediate to const would help. */
496 instr->regs[n+1] = lower_immed(ctx, src_reg, new_flags);
497 }
498
499 return;
500 }
501 }
502 }
503
504 /* Handle special case of eliminating output mov, and similar cases where
505 * there isn't a normal "consuming" instruction. In this case we cannot
506 * collapse flags (ie. output mov from const, or w/ abs/neg flags, cannot
507 * be eliminated)
508 */
509 static struct ir3_instruction *
510 eliminate_output_mov(struct ir3_instruction *instr)
511 {
512 if (is_eligible_mov(instr, false)) {
513 struct ir3_register *reg = instr->regs[1];
514 if (!(reg->flags & IR3_REG_ARRAY)) {
515 struct ir3_instruction *src_instr = ssa(reg);
516 debug_assert(src_instr);
517 return src_instr;
518 }
519 }
520 return instr;
521 }
522
523 /**
524 * Find instruction src's which are mov's that can be collapsed, replacing
525 * the mov dst with the mov src
526 */
527 static void
528 instr_cp(struct ir3_cp_ctx *ctx, struct ir3_instruction *instr)
529 {
530 struct ir3_register *reg;
531
532 if (instr->regs_count == 0)
533 return;
534
535 if (ir3_instr_check_mark(instr))
536 return;
537
538 /* walk down the graph from each src: */
539 foreach_src_n(reg, n, instr) {
540 struct ir3_instruction *src = ssa(reg);
541
542 if (!src)
543 continue;
544
545 instr_cp(ctx, src);
546
547 /* TODO non-indirect access we could figure out which register
548 * we actually want and allow cp..
549 */
550 if (reg->flags & IR3_REG_ARRAY)
551 continue;
552
553 /* Don't CP absneg into meta instructions, that won't end well: */
554 if (is_meta(instr) && (src->opc != OPC_MOV))
555 continue;
556
557 reg_cp(ctx, instr, reg, n);
558 }
559
560 if (instr->regs[0]->flags & IR3_REG_ARRAY) {
561 struct ir3_instruction *src = ssa(instr->regs[0]);
562 if (src)
563 instr_cp(ctx, src);
564 }
565
566 if (instr->address) {
567 instr_cp(ctx, instr->address);
568 ir3_instr_set_address(instr, eliminate_output_mov(instr->address));
569 }
570
571 /* we can end up with extra cmps.s from frontend, which uses a
572 *
573 * cmps.s p0.x, cond, 0
574 *
575 * as a way to mov into the predicate register. But frequently 'cond'
576 * is itself a cmps.s/cmps.f/cmps.u. So detect this special case and
577 * just re-write the instruction writing predicate register to get rid
578 * of the double cmps.
579 */
580 if ((instr->opc == OPC_CMPS_S) &&
581 (instr->regs[0]->num == regid(REG_P0, 0)) &&
582 ssa(instr->regs[1]) &&
583 (instr->regs[2]->flags & IR3_REG_IMMED) &&
584 (instr->regs[2]->iim_val == 0)) {
585 struct ir3_instruction *cond = ssa(instr->regs[1]);
586 switch (cond->opc) {
587 case OPC_CMPS_S:
588 case OPC_CMPS_F:
589 case OPC_CMPS_U:
590 instr->opc = cond->opc;
591 instr->flags = cond->flags;
592 instr->cat2 = cond->cat2;
593 instr->address = cond->address;
594 instr->regs[1] = cond->regs[1];
595 instr->regs[2] = cond->regs[2];
596 instr->barrier_class |= cond->barrier_class;
597 instr->barrier_conflict |= cond->barrier_conflict;
598 unuse(cond);
599 break;
600 default:
601 break;
602 }
603 }
604 }
605
606 void
607 ir3_cp(struct ir3 *ir, struct ir3_shader_variant *so)
608 {
609 struct ir3_cp_ctx ctx = {
610 .shader = ir,
611 .so = so,
612 };
613
614 /* This is a bit annoying, and probably wouldn't be necessary if we
615 * tracked a reverse link from producing instruction to consumer.
616 * But we need to know when we've eliminated the last consumer of
617 * a mov, so we need to do a pass to first count consumers of a
618 * mov.
619 */
620 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
621 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
622 struct ir3_instruction *src;
623
624 /* by the way, we don't account for false-dep's, so the CP
625 * pass should always happen before false-dep's are inserted
626 */
627 debug_assert(instr->deps_count == 0);
628
629 foreach_ssa_src(src, instr) {
630 src->use_count++;
631 }
632 }
633 }
634
635 ir3_clear_mark(ir);
636
637 for (unsigned i = 0; i < ir->noutputs; i++) {
638 if (ir->outputs[i]) {
639 instr_cp(&ctx, ir->outputs[i]);
640 ir->outputs[i] = eliminate_output_mov(ir->outputs[i]);
641 }
642 }
643
644 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
645 if (block->condition) {
646 instr_cp(&ctx, block->condition);
647 block->condition = eliminate_output_mov(block->condition);
648 }
649
650 for (unsigned i = 0; i < block->keeps_count; i++) {
651 instr_cp(&ctx, block->keeps[i]);
652 block->keeps[i] = eliminate_output_mov(block->keeps[i]);
653 }
654 }
655 }