projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Merge branch 'wip/nir-vtn' into vulkan
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_peephole.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_target.h"
25 #include "codegen/nv50_ir_build_util.h"
26
27 extern "C" {
28 #include "util/u_math.h"
29 }
30
31 namespace nv50_ir {
32
33 bool
34 Instruction::isNop() const
35 {
36 if (op == OP_PHI || op == OP_SPLIT || op == OP_MERGE || op == OP_CONSTRAINT)
37 return true;
38 if (terminator || join) // XXX: should terminator imply flow ?
39 return false;
40 if (op == OP_ATOM)
41 return false;
42 if (!fixed && op == OP_NOP)
43 return true;
44
45 if (defExists(0) && def(0).rep()->reg.data.id < 0) {
46 for (int d = 1; defExists(d); ++d)
47 if (def(d).rep()->reg.data.id >= 0)
48 WARN("part of vector result is unused !\n");
49 return true;
50 }
51
52 if (op == OP_MOV || op == OP_UNION) {
53 if (!getDef(0)->equals(getSrc(0)))
54 return false;
55 if (op == OP_UNION)
56 if (!def(0).rep()->equals(getSrc(1)))
57 return false;
58 return true;
59 }
60
61 return false;
62 }
63
64 bool Instruction::isDead() const
65 {
66 if (op == OP_STORE ||
67 op == OP_EXPORT ||
68 op == OP_ATOM ||
69 op == OP_SUSTB || op == OP_SUSTP || op == OP_SUREDP || op == OP_SUREDB ||
70 op == OP_WRSV)
71 return false;
72
73 for (int d = 0; defExists(d); ++d)
74 if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0)
75 return false;
76
77 if (terminator || asFlow())
78 return false;
79 if (fixed)
80 return false;
81
82 return true;
83 };
84
85 // =============================================================================
86
87 class CopyPropagation : public Pass
88 {
89 private:
90 virtual bool visit(BasicBlock *);
91 };
92
93 // Propagate all MOVs forward to make subsequent optimization easier, except if
94 // the sources stem from a phi, in which case we don't want to mess up potential
95 // swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def.
96 bool
97 CopyPropagation::visit(BasicBlock *bb)
98 {
99 Instruction *mov, *si, *next;
100
101 for (mov = bb->getEntry(); mov; mov = next) {
102 next = mov->next;
103 if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue())
104 continue;
105 if (mov->getPredicate())
106 continue;
107 if (mov->def(0).getFile() != mov->src(0).getFile())
108 continue;
109 si = mov->getSrc(0)->getInsn();
110 if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) {
111 // propagate
112 mov->def(0).replace(mov->getSrc(0), false);
113 delete_Instruction(prog, mov);
114 }
115 }
116 return true;
117 }
118
119 // =============================================================================
120
121 class MergeSplits : public Pass
122 {
123 private:
124 virtual bool visit(BasicBlock *);
125 };
126
127 // For SPLIT / MERGE pairs that operate on the same registers, replace the
128 // post-merge def with the SPLIT's source.
129 bool
130 MergeSplits::visit(BasicBlock *bb)
131 {
132 Instruction *i, *next, *si;
133
134 for (i = bb->getEntry(); i; i = next) {
135 next = i->next;
136 if (i->op != OP_MERGE || typeSizeof(i->dType) != 8)
137 continue;
138 si = i->getSrc(0)->getInsn();
139 if (si->op != OP_SPLIT || si != i->getSrc(1)->getInsn())
140 continue;
141 i->def(0).replace(si->getSrc(0), false);
142 delete_Instruction(prog, i);
143 }
144
145 return true;
146 }
147
148 // =============================================================================
149
150 class LoadPropagation : public Pass
151 {
152 private:
153 virtual bool visit(BasicBlock *);
154
155 void checkSwapSrc01(Instruction *);
156
157 bool isCSpaceLoad(Instruction *);
158 bool isImmd32Load(Instruction *);
159 bool isAttribOrSharedLoad(Instruction *);
160 };
161
162 bool
163 LoadPropagation::isCSpaceLoad(Instruction *ld)
164 {
165 return ld && ld->op == OP_LOAD && ld->src(0).getFile() == FILE_MEMORY_CONST;
166 }
167
168 bool
169 LoadPropagation::isImmd32Load(Instruction *ld)
170 {
171 if (!ld || (ld->op != OP_MOV) || (typeSizeof(ld->dType) != 4))
172 return false;
173 return ld->src(0).getFile() == FILE_IMMEDIATE;
174 }
175
176 bool
177 LoadPropagation::isAttribOrSharedLoad(Instruction *ld)
178 {
179 return ld &&
180 (ld->op == OP_VFETCH ||
181 (ld->op == OP_LOAD &&
182 (ld->src(0).getFile() == FILE_SHADER_INPUT ||
183 ld->src(0).getFile() == FILE_MEMORY_SHARED)));
184 }
185
186 void
187 LoadPropagation::checkSwapSrc01(Instruction *insn)
188 {
189 if (!prog->getTarget()->getOpInfo(insn).commutative)
190 if (insn->op != OP_SET && insn->op != OP_SLCT)
191 return;
192 if (insn->src(1).getFile() != FILE_GPR)
193 return;
194
195 Instruction *i0 = insn->getSrc(0)->getInsn();
196 Instruction *i1 = insn->getSrc(1)->getInsn();
197
198 if (isCSpaceLoad(i0)) {
199 if (!isCSpaceLoad(i1))
200 insn->swapSources(0, 1);
201 else
202 return;
203 } else
204 if (isImmd32Load(i0)) {
205 if (!isCSpaceLoad(i1) && !isImmd32Load(i1))
206 insn->swapSources(0, 1);
207 else
208 return;
209 } else
210 if (isAttribOrSharedLoad(i1)) {
211 if (!isAttribOrSharedLoad(i0))
212 insn->swapSources(0, 1);
213 else
214 return;
215 } else {
216 return;
217 }
218
219 if (insn->op == OP_SET || insn->op == OP_SET_AND ||
220 insn->op == OP_SET_OR || insn->op == OP_SET_XOR)
221 insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond);
222 else
223 if (insn->op == OP_SLCT)
224 insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond);
225 }
226
227 bool
228 LoadPropagation::visit(BasicBlock *bb)
229 {
230 const Target *targ = prog->getTarget();
231 Instruction *next;
232
233 for (Instruction *i = bb->getEntry(); i; i = next) {
234 next = i->next;
235
236 if (i->op == OP_CALL) // calls have args as sources, they must be in regs
237 continue;
238
239 if (i->op == OP_PFETCH) // pfetch expects arg1 to be a reg
240 continue;
241
242 if (i->srcExists(1))
243 checkSwapSrc01(i);
244
245 for (int s = 0; i->srcExists(s); ++s) {
246 Instruction *ld = i->getSrc(s)->getInsn();
247
248 if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV))
249 continue;
250 if (!targ->insnCanLoad(i, s, ld))
251 continue;
252
253 // propagate !
254 i->setSrc(s, ld->getSrc(0));
255 if (ld->src(0).isIndirect(0))
256 i->setIndirect(s, 0, ld->getIndirect(0, 0));
257
258 if (ld->getDef(0)->refCount() == 0)
259 delete_Instruction(prog, ld);
260 }
261 }
262 return true;
263 }
264
265 // =============================================================================
266
267 // Evaluate constant expressions.
268 class ConstantFolding : public Pass
269 {
270 public:
271 bool foldAll(Program *);
272
273 private:
274 virtual bool visit(BasicBlock *);
275
276 void expr(Instruction *, ImmediateValue&, ImmediateValue&);
277 void expr(Instruction *, ImmediateValue&, ImmediateValue&, ImmediateValue&);
278 void opnd(Instruction *, ImmediateValue&, int s);
279
280 void unary(Instruction *, const ImmediateValue&);
281
282 void tryCollapseChainedMULs(Instruction *, const int s, ImmediateValue&);
283
284 CmpInstruction *findOriginForTestWithZero(Value *);
285
286 unsigned int foldCount;
287
288 BuildUtil bld;
289 };
290
291 // TODO: remember generated immediates and only revisit these
292 bool
293 ConstantFolding::foldAll(Program *prog)
294 {
295 unsigned int iterCount = 0;
296 do {
297 foldCount = 0;
298 if (!run(prog))
299 return false;
300 } while (foldCount && ++iterCount < 2);
301 return true;
302 }
303
304 bool
305 ConstantFolding::visit(BasicBlock *bb)
306 {
307 Instruction *i, *next;
308
309 for (i = bb->getEntry(); i; i = next) {
310 next = i->next;
311 if (i->op == OP_MOV || i->op == OP_CALL)
312 continue;
313
314 ImmediateValue src0, src1, src2;
315
316 if (i->srcExists(2) &&
317 i->src(0).getImmediate(src0) &&
318 i->src(1).getImmediate(src1) &&
319 i->src(2).getImmediate(src2))
320 expr(i, src0, src1, src2);
321 else
322 if (i->srcExists(1) &&
323 i->src(0).getImmediate(src0) && i->src(1).getImmediate(src1))
324 expr(i, src0, src1);
325 else
326 if (i->srcExists(0) && i->src(0).getImmediate(src0))
327 opnd(i, src0, 0);
328 else
329 if (i->srcExists(1) && i->src(1).getImmediate(src1))
330 opnd(i, src1, 1);
331 }
332 return true;
333 }
334
335 CmpInstruction *
336 ConstantFolding::findOriginForTestWithZero(Value *value)
337 {
338 if (!value)
339 return NULL;
340 Instruction *insn = value->getInsn();
341
342 if (insn->asCmp() && insn->op != OP_SLCT)
343 return insn->asCmp();
344
345 /* Sometimes mov's will sneak in as a result of other folding. This gets
346 * cleaned up later.
347 */
348 if (insn->op == OP_MOV)
349 return findOriginForTestWithZero(insn->getSrc(0));
350
351 /* Deal with AND 1.0 here since nv50 can't fold into boolean float */
352 if (insn->op == OP_AND) {
353 int s = 0;
354 ImmediateValue imm;
355 if (!insn->src(s).getImmediate(imm)) {
356 s = 1;
357 if (!insn->src(s).getImmediate(imm))
358 return NULL;
359 }
360 if (imm.reg.data.f32 != 1.0f)
361 return NULL;
362 /* TODO: Come up with a way to handle the condition being inverted */
363 if (insn->src(!s).mod != Modifier(0))
364 return NULL;
365 return findOriginForTestWithZero(insn->getSrc(!s));
366 }
367
368 return NULL;
369 }
370
371 void
372 Modifier::applyTo(ImmediateValue& imm) const
373 {
374 if (!bits) // avoid failure if imm.reg.type is unhandled (e.g. b128)
375 return;
376 switch (imm.reg.type) {
377 case TYPE_F32:
378 if (bits & NV50_IR_MOD_ABS)
379 imm.reg.data.f32 = fabsf(imm.reg.data.f32);
380 if (bits & NV50_IR_MOD_NEG)
381 imm.reg.data.f32 = -imm.reg.data.f32;
382 if (bits & NV50_IR_MOD_SAT) {
383 if (imm.reg.data.f32 < 0.0f)
384 imm.reg.data.f32 = 0.0f;
385 else
386 if (imm.reg.data.f32 > 1.0f)
387 imm.reg.data.f32 = 1.0f;
388 }
389 assert(!(bits & NV50_IR_MOD_NOT));
390 break;
391
392 case TYPE_S8: // NOTE: will be extended
393 case TYPE_S16:
394 case TYPE_S32:
395 case TYPE_U8: // NOTE: treated as signed
396 case TYPE_U16:
397 case TYPE_U32:
398 if (bits & NV50_IR_MOD_ABS)
399 imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ?
400 imm.reg.data.s32 : -imm.reg.data.s32;
401 if (bits & NV50_IR_MOD_NEG)
402 imm.reg.data.s32 = -imm.reg.data.s32;
403 if (bits & NV50_IR_MOD_NOT)
404 imm.reg.data.s32 = ~imm.reg.data.s32;
405 break;
406
407 case TYPE_F64:
408 if (bits & NV50_IR_MOD_ABS)
409 imm.reg.data.f64 = fabs(imm.reg.data.f64);
410 if (bits & NV50_IR_MOD_NEG)
411 imm.reg.data.f64 = -imm.reg.data.f64;
412 if (bits & NV50_IR_MOD_SAT) {
413 if (imm.reg.data.f64 < 0.0)
414 imm.reg.data.f64 = 0.0;
415 else
416 if (imm.reg.data.f64 > 1.0)
417 imm.reg.data.f64 = 1.0;
418 }
419 assert(!(bits & NV50_IR_MOD_NOT));
420 break;
421
422 default:
423 assert(!"invalid/unhandled type");
424 imm.reg.data.u64 = 0;
425 break;
426 }
427 }
428
429 operation
430 Modifier::getOp() const
431 {
432 switch (bits) {
433 case NV50_IR_MOD_ABS: return OP_ABS;
434 case NV50_IR_MOD_NEG: return OP_NEG;
435 case NV50_IR_MOD_SAT: return OP_SAT;
436 case NV50_IR_MOD_NOT: return OP_NOT;
437 case 0:
438 return OP_MOV;
439 default:
440 return OP_CVT;
441 }
442 }
443
444 void
445 ConstantFolding::expr(Instruction *i,
446 ImmediateValue &imm0, ImmediateValue &imm1)
447 {
448 struct Storage *const a = &imm0.reg, *const b = &imm1.reg;
449 struct Storage res;
450
451 memset(&res.data, 0, sizeof(res.data));
452
453 switch (i->op) {
454 case OP_MAD:
455 case OP_FMA:
456 case OP_MUL:
457 if (i->dnz && i->dType == TYPE_F32) {
458 if (!isfinite(a->data.f32))
459 a->data.f32 = 0.0f;
460 if (!isfinite(b->data.f32))
461 b->data.f32 = 0.0f;
462 }
463 switch (i->dType) {
464 case TYPE_F32:
465 res.data.f32 = a->data.f32 * b->data.f32 * exp2f(i->postFactor);
466 break;
467 case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break;
468 case TYPE_S32:
469 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
470 res.data.s32 = ((int64_t)a->data.s32 * b->data.s32) >> 32;
471 break;
472 }
473 /* fallthrough */
474 case TYPE_U32:
475 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
476 res.data.u32 = ((uint64_t)a->data.u32 * b->data.u32) >> 32;
477 break;
478 }
479 res.data.u32 = a->data.u32 * b->data.u32; break;
480 default:
481 return;
482 }
483 break;
484 case OP_DIV:
485 if (b->data.u32 == 0)
486 break;
487 switch (i->dType) {
488 case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break;
489 case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break;
490 case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break;
491 case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break;
492 default:
493 return;
494 }
495 break;
496 case OP_ADD:
497 switch (i->dType) {
498 case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break;
499 case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break;
500 case TYPE_S32:
501 case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break;
502 default:
503 return;
504 }
505 break;
506 case OP_POW:
507 switch (i->dType) {
508 case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break;
509 case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break;
510 default:
511 return;
512 }
513 break;
514 case OP_MAX:
515 switch (i->dType) {
516 case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break;
517 case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break;
518 case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break;
519 case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break;
520 default:
521 return;
522 }
523 break;
524 case OP_MIN:
525 switch (i->dType) {
526 case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break;
527 case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break;
528 case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break;
529 case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break;
530 default:
531 return;
532 }
533 break;
534 case OP_AND:
535 res.data.u64 = a->data.u64 & b->data.u64;
536 break;
537 case OP_OR:
538 res.data.u64 = a->data.u64 | b->data.u64;
539 break;
540 case OP_XOR:
541 res.data.u64 = a->data.u64 ^ b->data.u64;
542 break;
543 case OP_SHL:
544 res.data.u32 = a->data.u32 << b->data.u32;
545 break;
546 case OP_SHR:
547 switch (i->dType) {
548 case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break;
549 case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break;
550 default:
551 return;
552 }
553 break;
554 case OP_SLCT:
555 if (a->data.u32 != b->data.u32)
556 return;
557 res.data.u32 = a->data.u32;
558 break;
559 case OP_EXTBF: {
560 int offset = b->data.u32 & 0xff;
561 int width = (b->data.u32 >> 8) & 0xff;
562 int rshift = offset;
563 int lshift = 0;
564 if (width == 0) {
565 res.data.u32 = 0;
566 break;
567 }
568 if (width + offset < 32) {
569 rshift = 32 - width;
570 lshift = 32 - width - offset;
571 }
572 if (i->subOp == NV50_IR_SUBOP_EXTBF_REV)
573 res.data.u32 = util_bitreverse(a->data.u32);
574 else
575 res.data.u32 = a->data.u32;
576 switch (i->dType) {
577 case TYPE_S32: res.data.s32 = (res.data.s32 << lshift) >> rshift; break;
578 case TYPE_U32: res.data.u32 = (res.data.u32 << lshift) >> rshift; break;
579 default:
580 return;
581 }
582 break;
583 }
584 case OP_POPCNT:
585 res.data.u32 = util_bitcount(a->data.u32 & b->data.u32);
586 break;
587 case OP_PFETCH:
588 // The two arguments to pfetch are logically added together. Normally
589 // the second argument will not be constant, but that can happen.
590 res.data.u32 = a->data.u32 + b->data.u32;
591 break;
592 default:
593 return;
594 }
595 ++foldCount;
596
597 i->src(0).mod = Modifier(0);
598 i->src(1).mod = Modifier(0);
599 i->postFactor = 0;
600
601 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
602 i->setSrc(1, NULL);
603
604 i->getSrc(0)->reg.data = res.data;
605
606 switch (i->op) {
607 case OP_MAD:
608 case OP_FMA: {
609 i->op = OP_ADD;
610
611 i->setSrc(1, i->getSrc(0));
612 i->src(1).mod = i->src(2).mod;
613 i->setSrc(0, i->getSrc(2));
614 i->setSrc(2, NULL);
615
616 ImmediateValue src0;
617 if (i->src(0).getImmediate(src0))
618 expr(i, src0, *i->getSrc(1)->asImm());
619 if (i->saturate && !prog->getTarget()->isSatSupported(i)) {
620 bld.setPosition(i, false);
621 i->setSrc(1, bld.loadImm(NULL, res.data.u32));
622 }
623 break;
624 }
625 case OP_PFETCH:
626 // Leave PFETCH alone... we just folded its 2 args into 1.
627 break;
628 default:
629 i->op = i->saturate ? OP_SAT : OP_MOV; /* SAT handled by unary() */
630 break;
631 }
632 i->subOp = 0;
633 }
634
635 void
636 ConstantFolding::expr(Instruction *i,
637 ImmediateValue &imm0,
638 ImmediateValue &imm1,
639 ImmediateValue &imm2)
640 {
641 struct Storage *const a = &imm0.reg, *const b = &imm1.reg, *const c = &imm2.reg;
642 struct Storage res;
643
644 memset(&res.data, 0, sizeof(res.data));
645
646 switch (i->op) {
647 case OP_INSBF: {
648 int offset = b->data.u32 & 0xff;
649 int width = (b->data.u32 >> 8) & 0xff;
650 unsigned bitmask = ((1 << width) - 1) << offset;
651 res.data.u32 = ((a->data.u32 << offset) & bitmask) | (c->data.u32 & ~bitmask);
652 break;
653 }
654 default:
655 return;
656 }
657
658 ++foldCount;
659 i->src(0).mod = Modifier(0);
660 i->src(1).mod = Modifier(0);
661 i->src(2).mod = Modifier(0);
662
663 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
664 i->setSrc(1, NULL);
665 i->setSrc(2, NULL);
666
667 i->getSrc(0)->reg.data = res.data;
668
669 i->op = OP_MOV;
670 }
671
672 void
673 ConstantFolding::unary(Instruction *i, const ImmediateValue &imm)
674 {
675 Storage res;
676
677 if (i->dType != TYPE_F32)
678 return;
679 switch (i->op) {
680 case OP_NEG: res.data.f32 = -imm.reg.data.f32; break;
681 case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break;
682 case OP_SAT: res.data.f32 = CLAMP(imm.reg.data.f32, 0.0f, 1.0f); break;
683 case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break;
684 case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break;
685 case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break;
686 case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break;
687 case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break;
688 case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break;
689 case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break;
690 case OP_PRESIN:
691 case OP_PREEX2:
692 // these should be handled in subsequent OP_SIN/COS/EX2
693 res.data.f32 = imm.reg.data.f32;
694 break;
695 default:
696 return;
697 }
698 i->op = OP_MOV;
699 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32));
700 i->src(0).mod = Modifier(0);
701 }
702
703 void
704 ConstantFolding::tryCollapseChainedMULs(Instruction *mul2,
705 const int s, ImmediateValue& imm2)
706 {
707 const int t = s ? 0 : 1;
708 Instruction *insn;
709 Instruction *mul1 = NULL; // mul1 before mul2
710 int e = 0;
711 float f = imm2.reg.data.f32 * exp2f(mul2->postFactor);
712 ImmediateValue imm1;
713
714 assert(mul2->op == OP_MUL && mul2->dType == TYPE_F32);
715
716 if (mul2->getSrc(t)->refCount() == 1) {
717 insn = mul2->getSrc(t)->getInsn();
718 if (!mul2->src(t).mod && insn->op == OP_MUL && insn->dType == TYPE_F32)
719 mul1 = insn;
720 if (mul1 && !mul1->saturate) {
721 int s1;
722
723 if (mul1->src(s1 = 0).getImmediate(imm1) ||
724 mul1->src(s1 = 1).getImmediate(imm1)) {
725 bld.setPosition(mul1, false);
726 // a = mul r, imm1
727 // d = mul a, imm2 -> d = mul r, (imm1 * imm2)
728 mul1->setSrc(s1, bld.loadImm(NULL, f * imm1.reg.data.f32));
729 mul1->src(s1).mod = Modifier(0);
730 mul2->def(0).replace(mul1->getDef(0), false);
731 mul1->saturate = mul2->saturate;
732 } else
733 if (prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
734 // c = mul a, b
735 // d = mul c, imm -> d = mul_x_imm a, b
736 mul1->postFactor = e;
737 mul2->def(0).replace(mul1->getDef(0), false);
738 if (f < 0)
739 mul1->src(0).mod *= Modifier(NV50_IR_MOD_NEG);
740 mul1->saturate = mul2->saturate;
741 }
742 return;
743 }
744 }
745 if (mul2->getDef(0)->refCount() == 1 && !mul2->saturate) {
746 // b = mul a, imm
747 // d = mul b, c -> d = mul_x_imm a, c
748 int s2, t2;
749 insn = (*mul2->getDef(0)->uses.begin())->getInsn();
750 if (!insn)
751 return;
752 mul1 = mul2;
753 mul2 = NULL;
754 s2 = insn->getSrc(0) == mul1->getDef(0) ? 0 : 1;
755 t2 = s2 ? 0 : 1;
756 if (insn->op == OP_MUL && insn->dType == TYPE_F32)
757 if (!insn->src(s2).mod && !insn->src(t2).getImmediate(imm1))
758 mul2 = insn;
759 if (mul2 && prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
760 mul2->postFactor = e;
761 mul2->setSrc(s2, mul1->src(t));
762 if (f < 0)
763 mul2->src(s2).mod *= Modifier(NV50_IR_MOD_NEG);
764 }
765 }
766 }
767
768 void
769 ConstantFolding::opnd(Instruction *i, ImmediateValue &imm0, int s)
770 {
771 const int t = !s;
772 const operation op = i->op;
773 Instruction *newi = i;
774
775 switch (i->op) {
776 case OP_MUL:
777 if (i->dType == TYPE_F32)
778 tryCollapseChainedMULs(i, s, imm0);
779
780 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
781 assert(!isFloatType(i->sType));
782 if (imm0.isInteger(1) && i->dType == TYPE_S32) {
783 bld.setPosition(i, false);
784 // Need to set to the sign value, which is a compare.
785 newi = bld.mkCmp(OP_SET, CC_LT, TYPE_S32, i->getDef(0),
786 TYPE_S32, i->getSrc(t), bld.mkImm(0));
787 delete_Instruction(prog, i);
788 } else if (imm0.isInteger(0) || imm0.isInteger(1)) {
789 // The high bits can't be set in this case (either mul by 0 or
790 // unsigned by 1)
791 i->op = OP_MOV;
792 i->subOp = 0;
793 i->setSrc(0, new_ImmediateValue(prog, 0u));
794 i->src(0).mod = Modifier(0);
795 i->setSrc(1, NULL);
796 } else if (!imm0.isNegative() && imm0.isPow2()) {
797 // Translate into a shift
798 imm0.applyLog2();
799 i->op = OP_SHR;
800 i->subOp = 0;
801 imm0.reg.data.u32 = 32 - imm0.reg.data.u32;
802 i->setSrc(0, i->getSrc(t));
803 i->src(0).mod = i->src(t).mod;
804 i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
805 i->src(1).mod = 0;
806 }
807 } else
808 if (imm0.isInteger(0)) {
809 i->op = OP_MOV;
810 i->setSrc(0, new_ImmediateValue(prog, 0u));
811 i->src(0).mod = Modifier(0);
812 i->postFactor = 0;
813 i->setSrc(1, NULL);
814 } else
815 if (!i->postFactor && (imm0.isInteger(1) || imm0.isInteger(-1))) {
816 if (imm0.isNegative())
817 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
818 i->op = i->src(t).mod.getOp();
819 if (s == 0) {
820 i->setSrc(0, i->getSrc(1));
821 i->src(0).mod = i->src(1).mod;
822 i->src(1).mod = 0;
823 }
824 if (i->op != OP_CVT)
825 i->src(0).mod = 0;
826 i->setSrc(1, NULL);
827 } else
828 if (!i->postFactor && (imm0.isInteger(2) || imm0.isInteger(-2))) {
829 if (imm0.isNegative())
830 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
831 i->op = OP_ADD;
832 i->setSrc(s, i->getSrc(t));
833 i->src(s).mod = i->src(t).mod;
834 } else
835 if (!isFloatType(i->sType) && !imm0.isNegative() && imm0.isPow2()) {
836 i->op = OP_SHL;
837 imm0.applyLog2();
838 i->setSrc(0, i->getSrc(t));
839 i->src(0).mod = i->src(t).mod;
840 i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
841 i->src(1).mod = 0;
842 }
843 break;
844 case OP_MAD:
845 if (imm0.isInteger(0)) {
846 i->setSrc(0, i->getSrc(2));
847 i->src(0).mod = i->src(2).mod;
848 i->setSrc(1, NULL);
849 i->setSrc(2, NULL);
850 i->op = i->src(0).mod.getOp();
851 if (i->op != OP_CVT)
852 i->src(0).mod = 0;
853 } else
854 if (imm0.isInteger(1) || imm0.isInteger(-1)) {
855 if (imm0.isNegative())
856 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
857 if (s == 0) {
858 i->setSrc(0, i->getSrc(1));
859 i->src(0).mod = i->src(1).mod;
860 }
861 i->setSrc(1, i->getSrc(2));
862 i->src(1).mod = i->src(2).mod;
863 i->setSrc(2, NULL);
864 i->op = OP_ADD;
865 }
866 break;
867 case OP_ADD:
868 if (i->usesFlags())
869 break;
870 if (imm0.isInteger(0)) {
871 if (s == 0) {
872 i->setSrc(0, i->getSrc(1));
873 i->src(0).mod = i->src(1).mod;
874 }
875 i->setSrc(1, NULL);
876 i->op = i->src(0).mod.getOp();
877 if (i->op != OP_CVT)
878 i->src(0).mod = Modifier(0);
879 }
880 break;
881
882 case OP_DIV:
883 if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32))
884 break;
885 bld.setPosition(i, false);
886 if (imm0.reg.data.u32 == 0) {
887 break;
888 } else
889 if (imm0.reg.data.u32 == 1) {
890 i->op = OP_MOV;
891 i->setSrc(1, NULL);
892 } else
893 if (i->dType == TYPE_U32 && imm0.isPow2()) {
894 i->op = OP_SHR;
895 i->setSrc(1, bld.mkImm(util_logbase2(imm0.reg.data.u32)));
896 } else
897 if (i->dType == TYPE_U32) {
898 Instruction *mul;
899 Value *tA, *tB;
900 const uint32_t d = imm0.reg.data.u32;
901 uint32_t m;
902 int r, s;
903 uint32_t l = util_logbase2(d);
904 if (((uint32_t)1 << l) < d)
905 ++l;
906 m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1;
907 r = l ? 1 : 0;
908 s = l ? (l - 1) : 0;
909
910 tA = bld.getSSA();
911 tB = bld.getSSA();
912 mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0),
913 bld.loadImm(NULL, m));
914 mul->subOp = NV50_IR_SUBOP_MUL_HIGH;
915 bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA);
916 tA = bld.getSSA();
917 if (r)
918 bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r));
919 else
920 tA = tB;
921 tB = s ? bld.getSSA() : i->getDef(0);
922 newi = bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA);
923 if (s)
924 bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s));
925
926 delete_Instruction(prog, i);
927 } else
928 if (imm0.reg.data.s32 == -1) {
929 i->op = OP_NEG;
930 i->setSrc(1, NULL);
931 } else {
932 LValue *tA, *tB;
933 LValue *tD;
934 const int32_t d = imm0.reg.data.s32;
935 int32_t m;
936 int32_t l = util_logbase2(static_cast<unsigned>(abs(d)));
937 if ((1 << l) < abs(d))
938 ++l;
939 if (!l)
940 l = 1;
941 m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32);
942
943 tA = bld.getSSA();
944 tB = bld.getSSA();
945 bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m),
946 i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH;
947 if (l > 1)
948 bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1));
949 else
950 tB = tA;
951 tA = bld.getSSA();
952 bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, TYPE_S32, i->getSrc(0), bld.mkImm(0));
953 tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue();
954 newi = bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA);
955 if (d < 0)
956 bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB);
957
958 delete_Instruction(prog, i);
959 }
960 break;
961
962 case OP_MOD:
963 if (i->sType == TYPE_U32 && imm0.isPow2()) {
964 bld.setPosition(i, false);
965 i->op = OP_AND;
966 i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 - 1));
967 }
968 break;
969
970 case OP_SET: // TODO: SET_AND,OR,XOR
971 {
972 /* This optimizes the case where the output of a set is being compared
973 * to zero. Since the set can only produce 0/-1 (int) or 0/1 (float), we
974 * can be a lot cleverer in our comparison.
975 */
976 CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t));
977 CondCode cc, ccZ;
978 if (imm0.reg.data.u32 != 0 || !si)
979 return;
980 cc = si->setCond;
981 ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U);
982 // We do everything assuming var (cmp) 0, reverse the condition if 0 is
983 // first.
984 if (s == 0)
985 ccZ = reverseCondCode(ccZ);
986 // If there is a negative modifier, we need to undo that, by flipping
987 // the comparison to zero.
988 if (i->src(t).mod.neg())
989 ccZ = reverseCondCode(ccZ);
990 // If this is a signed comparison, we expect the input to be a regular
991 // boolean, i.e. 0/-1. However the rest of the logic assumes that true
992 // is positive, so just flip the sign.
993 if (i->sType == TYPE_S32) {
994 assert(!isFloatType(si->dType));
995 ccZ = reverseCondCode(ccZ);
996 }
997 switch (ccZ) {
998 case CC_LT: cc = CC_FL; break; // bool < 0 -- this is never true
999 case CC_GE: cc = CC_TR; break; // bool >= 0 -- this is always true
1000 case CC_EQ: cc = inverseCondCode(cc); break; // bool == 0 -- !bool
1001 case CC_LE: cc = inverseCondCode(cc); break; // bool <= 0 -- !bool
1002 case CC_GT: break; // bool > 0 -- bool
1003 case CC_NE: break; // bool != 0 -- bool
1004 default:
1005 return;
1006 }
1007
1008 // Update the condition of this SET to be identical to the origin set,
1009 // but with the updated condition code. The original SET should get
1010 // DCE'd, ideally.
1011 i->op = si->op;
1012 i->asCmp()->setCond = cc;
1013 i->setSrc(0, si->src(0));
1014 i->setSrc(1, si->src(1));
1015 if (si->srcExists(2))
1016 i->setSrc(2, si->src(2));
1017 i->sType = si->sType;
1018 }
1019 break;
1020
1021 case OP_AND:
1022 {
1023 CmpInstruction *cmp = i->getSrc(t)->getInsn()->asCmp();
1024 if (!cmp || cmp->op == OP_SLCT || cmp->getDef(0)->refCount() > 1)
1025 return;
1026 if (!prog->getTarget()->isOpSupported(cmp->op, TYPE_F32))
1027 return;
1028 if (imm0.reg.data.f32 != 1.0)
1029 return;
1030 if (i->getSrc(t)->getInsn()->dType != TYPE_U32)
1031 return;
1032
1033 i->getSrc(t)->getInsn()->dType = TYPE_F32;
1034 if (i->src(t).mod != Modifier(0)) {
1035 assert(i->src(t).mod == Modifier(NV50_IR_MOD_NOT));
1036 i->src(t).mod = Modifier(0);
1037 cmp->setCond = inverseCondCode(cmp->setCond);
1038 }
1039 i->op = OP_MOV;
1040 i->setSrc(s, NULL);
1041 if (t) {
1042 i->setSrc(0, i->getSrc(t));
1043 i->setSrc(t, NULL);
1044 }
1045 }
1046 break;
1047
1048 case OP_SHL:
1049 {
1050 if (s != 1 || i->src(0).mod != Modifier(0))
1051 break;
1052 // try to concatenate shifts
1053 Instruction *si = i->getSrc(0)->getInsn();
1054 if (!si || si->op != OP_SHL)
1055 break;
1056 ImmediateValue imm1;
1057 if (si->src(1).getImmediate(imm1)) {
1058 bld.setPosition(i, false);
1059 i->setSrc(0, si->getSrc(0));
1060 i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 + imm1.reg.data.u32));
1061 }
1062 }
1063 break;
1064
1065 case OP_ABS:
1066 case OP_NEG:
1067 case OP_SAT:
1068 case OP_LG2:
1069 case OP_RCP:
1070 case OP_SQRT:
1071 case OP_RSQ:
1072 case OP_PRESIN:
1073 case OP_SIN:
1074 case OP_COS:
1075 case OP_PREEX2:
1076 case OP_EX2:
1077 unary(i, imm0);
1078 break;
1079 case OP_BFIND: {
1080 int32_t res;
1081 switch (i->dType) {
1082 case TYPE_S32: res = util_last_bit_signed(imm0.reg.data.s32) - 1; break;
1083 case TYPE_U32: res = util_last_bit(imm0.reg.data.u32) - 1; break;
1084 default:
1085 return;
1086 }
1087 if (i->subOp == NV50_IR_SUBOP_BFIND_SAMT && res >= 0)
1088 res = 31 - res;
1089 bld.setPosition(i, false); /* make sure bld is init'ed */
1090 i->setSrc(0, bld.mkImm(res));
1091 i->setSrc(1, NULL);
1092 i->op = OP_MOV;
1093 i->subOp = 0;
1094 break;
1095 }
1096 case OP_POPCNT: {
1097 // Only deal with 1-arg POPCNT here
1098 if (i->srcExists(1))
1099 break;
1100 uint32_t res = util_bitcount(imm0.reg.data.u32);
1101 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res));
1102 i->setSrc(1, NULL);
1103 i->op = OP_MOV;
1104 break;
1105 }
1106 default:
1107 return;
1108 }
1109 if (newi->op != op)
1110 foldCount++;
1111 }
1112
1113 // =============================================================================
1114
1115 // Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed.
1116 class ModifierFolding : public Pass
1117 {
1118 private:
1119 virtual bool visit(BasicBlock *);
1120 };
1121
1122 bool
1123 ModifierFolding::visit(BasicBlock *bb)
1124 {
1125 const Target *target = prog->getTarget();
1126
1127 Instruction *i, *next, *mi;
1128 Modifier mod;
1129
1130 for (i = bb->getEntry(); i; i = next) {
1131 next = i->next;
1132
1133 if (0 && i->op == OP_SUB) {
1134 // turn "sub" into "add neg" (do we really want this ?)
1135 i->op = OP_ADD;
1136 i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
1137 }
1138
1139 for (int s = 0; s < 3 && i->srcExists(s); ++s) {
1140 mi = i->getSrc(s)->getInsn();
1141 if (!mi ||
1142 mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8)
1143 continue;
1144 if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) {
1145 if ((i->op != OP_ADD &&
1146 i->op != OP_MUL) ||
1147 (mi->op != OP_ABS &&
1148 mi->op != OP_NEG))
1149 continue;
1150 } else
1151 if (i->sType != mi->dType) {
1152 continue;
1153 }
1154 if ((mod = Modifier(mi->op)) == Modifier(0))
1155 continue;
1156 mod *= mi->src(0).mod;
1157
1158 if ((i->op == OP_ABS) || i->src(s).mod.abs()) {
1159 // abs neg [abs] = abs
1160 mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS));
1161 } else
1162 if ((i->op == OP_NEG) && mod.neg()) {
1163 assert(s == 0);
1164 // neg as both opcode and modifier on same insn is prohibited
1165 // neg neg abs = abs, neg neg = identity
1166 mod = mod & Modifier(~NV50_IR_MOD_NEG);
1167 i->op = mod.getOp();
1168 mod = mod & Modifier(~NV50_IR_MOD_ABS);
1169 if (mod == Modifier(0))
1170 i->op = OP_MOV;
1171 }
1172
1173 if (target->isModSupported(i, s, mod)) {
1174 i->setSrc(s, mi->getSrc(0));
1175 i->src(s).mod *= mod;
1176 }
1177 }
1178
1179 if (i->op == OP_SAT) {
1180 mi = i->getSrc(0)->getInsn();
1181 if (mi &&
1182 mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) {
1183 mi->saturate = 1;
1184 mi->setDef(0, i->getDef(0));
1185 delete_Instruction(prog, i);
1186 }
1187 }
1188 }
1189
1190 return true;
1191 }
1192
1193 // =============================================================================
1194
1195 // MUL + ADD -> MAD/FMA
1196 // MIN/MAX(a, a) -> a, etc.
1197 // SLCT(a, b, const) -> cc(const) ? a : b
1198 // RCP(RCP(a)) -> a
1199 // MUL(MUL(a, b), const) -> MUL_Xconst(a, b)
1200 class AlgebraicOpt : public Pass
1201 {
1202 private:
1203 virtual bool visit(BasicBlock *);
1204
1205 void handleABS(Instruction *);
1206 bool handleADD(Instruction *);
1207 bool tryADDToMADOrSAD(Instruction *, operation toOp);
1208 void handleMINMAX(Instruction *);
1209 void handleRCP(Instruction *);
1210 void handleSLCT(Instruction *);
1211 void handleLOGOP(Instruction *);
1212 void handleCVT(Instruction *);
1213 void handleSUCLAMP(Instruction *);
1214
1215 BuildUtil bld;
1216 };
1217
1218 void
1219 AlgebraicOpt::handleABS(Instruction *abs)
1220 {
1221 Instruction *sub = abs->getSrc(0)->getInsn();
1222 DataType ty;
1223 if (!sub ||
1224 !prog->getTarget()->isOpSupported(OP_SAD, abs->dType))
1225 return;
1226 // expect not to have mods yet, if we do, bail
1227 if (sub->src(0).mod || sub->src(1).mod)
1228 return;
1229 // hidden conversion ?
1230 ty = intTypeToSigned(sub->dType);
1231 if (abs->dType != abs->sType || ty != abs->sType)
1232 return;
1233
1234 if ((sub->op != OP_ADD && sub->op != OP_SUB) ||
1235 sub->src(0).getFile() != FILE_GPR || sub->src(0).mod ||
1236 sub->src(1).getFile() != FILE_GPR || sub->src(1).mod)
1237 return;
1238
1239 Value *src0 = sub->getSrc(0);
1240 Value *src1 = sub->getSrc(1);
1241
1242 if (sub->op == OP_ADD) {
1243 Instruction *neg = sub->getSrc(1)->getInsn();
1244 if (neg && neg->op != OP_NEG) {
1245 neg = sub->getSrc(0)->getInsn();
1246 src0 = sub->getSrc(1);
1247 }
1248 if (!neg || neg->op != OP_NEG ||
1249 neg->dType != neg->sType || neg->sType != ty)
1250 return;
1251 src1 = neg->getSrc(0);
1252 }
1253
1254 // found ABS(SUB))
1255 abs->moveSources(1, 2); // move sources >=1 up by 2
1256 abs->op = OP_SAD;
1257 abs->setType(sub->dType);
1258 abs->setSrc(0, src0);
1259 abs->setSrc(1, src1);
1260 bld.setPosition(abs, false);
1261 abs->setSrc(2, bld.loadImm(bld.getSSA(typeSizeof(ty)), 0));
1262 }
1263
1264 bool
1265 AlgebraicOpt::handleADD(Instruction *add)
1266 {
1267 Value *src0 = add->getSrc(0);
1268 Value *src1 = add->getSrc(1);
1269
1270 if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
1271 return false;
1272
1273 bool changed = false;
1274 if (!changed && prog->getTarget()->isOpSupported(OP_MAD, add->dType))
1275 changed = tryADDToMADOrSAD(add, OP_MAD);
1276 if (!changed && prog->getTarget()->isOpSupported(OP_SAD, add->dType))
1277 changed = tryADDToMADOrSAD(add, OP_SAD);
1278 return changed;
1279 }
1280
1281 // ADD(SAD(a,b,0), c) -> SAD(a,b,c)
1282 // ADD(MUL(a,b), c) -> MAD(a,b,c)
1283 bool
1284 AlgebraicOpt::tryADDToMADOrSAD(Instruction *add, operation toOp)
1285 {
1286 Value *src0 = add->getSrc(0);
1287 Value *src1 = add->getSrc(1);
1288 Value *src;
1289 int s;
1290 const operation srcOp = toOp == OP_SAD ? OP_SAD : OP_MUL;
1291 const Modifier modBad = Modifier(~((toOp == OP_MAD) ? NV50_IR_MOD_NEG : 0));
1292 Modifier mod[4];
1293
1294 if (src0->refCount() == 1 &&
1295 src0->getUniqueInsn() && src0->getUniqueInsn()->op == srcOp)
1296 s = 0;
1297 else
1298 if (src1->refCount() == 1 &&
1299 src1->getUniqueInsn() && src1->getUniqueInsn()->op == srcOp)
1300 s = 1;
1301 else
1302 return false;
1303
1304 if ((src0->getUniqueInsn() && src0->getUniqueInsn()->bb != add->bb) ||
1305 (src1->getUniqueInsn() && src1->getUniqueInsn()->bb != add->bb))
1306 return false;
1307
1308 src = add->getSrc(s);
1309
1310 if (src->getInsn()->postFactor)
1311 return false;
1312 if (toOp == OP_SAD) {
1313 ImmediateValue imm;
1314 if (!src->getInsn()->src(2).getImmediate(imm))
1315 return false;
1316 if (!imm.isInteger(0))
1317 return false;
1318 }
1319
1320 mod[0] = add->src(0).mod;
1321 mod[1] = add->src(1).mod;
1322 mod[2] = src->getUniqueInsn()->src(0).mod;
1323 mod[3] = src->getUniqueInsn()->src(1).mod;
1324
1325 if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & modBad)
1326 return false;
1327
1328 add->op = toOp;
1329 add->subOp = src->getInsn()->subOp; // potentially mul-high
1330
1331 add->setSrc(2, add->src(s ? 0 : 1));
1332
1333 add->setSrc(0, src->getInsn()->getSrc(0));
1334 add->src(0).mod = mod[2] ^ mod[s];
1335 add->setSrc(1, src->getInsn()->getSrc(1));
1336 add->src(1).mod = mod[3];
1337
1338 return true;
1339 }
1340
1341 void
1342 AlgebraicOpt::handleMINMAX(Instruction *minmax)
1343 {
1344 Value *src0 = minmax->getSrc(0);
1345 Value *src1 = minmax->getSrc(1);
1346
1347 if (src0 != src1 || src0->reg.file != FILE_GPR)
1348 return;
1349 if (minmax->src(0).mod == minmax->src(1).mod) {
1350 if (minmax->def(0).mayReplace(minmax->src(0))) {
1351 minmax->def(0).replace(minmax->src(0), false);
1352 minmax->bb->remove(minmax);
1353 } else {
1354 minmax->op = OP_CVT;
1355 minmax->setSrc(1, NULL);
1356 }
1357 } else {
1358 // TODO:
1359 // min(x, -x) = -abs(x)
1360 // min(x, -abs(x)) = -abs(x)
1361 // min(x, abs(x)) = x
1362 // max(x, -abs(x)) = x
1363 // max(x, abs(x)) = abs(x)
1364 // max(x, -x) = abs(x)
1365 }
1366 }
1367
1368 void
1369 AlgebraicOpt::handleRCP(Instruction *rcp)
1370 {
1371 Instruction *si = rcp->getSrc(0)->getUniqueInsn();
1372
1373 if (si && si->op == OP_RCP) {
1374 Modifier mod = rcp->src(0).mod * si->src(0).mod;
1375 rcp->op = mod.getOp();
1376 rcp->setSrc(0, si->getSrc(0));
1377 }
1378 }
1379
1380 void
1381 AlgebraicOpt::handleSLCT(Instruction *slct)
1382 {
1383 if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) {
1384 if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f))
1385 slct->setSrc(0, slct->getSrc(1));
1386 } else
1387 if (slct->getSrc(0) != slct->getSrc(1)) {
1388 return;
1389 }
1390 slct->op = OP_MOV;
1391 slct->setSrc(1, NULL);
1392 slct->setSrc(2, NULL);
1393 }
1394
1395 void
1396 AlgebraicOpt::handleLOGOP(Instruction *logop)
1397 {
1398 Value *src0 = logop->getSrc(0);
1399 Value *src1 = logop->getSrc(1);
1400
1401 if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
1402 return;
1403
1404 if (src0 == src1) {
1405 if ((logop->op == OP_AND || logop->op == OP_OR) &&
1406 logop->def(0).mayReplace(logop->src(0))) {
1407 logop->def(0).replace(logop->src(0), false);
1408 delete_Instruction(prog, logop);
1409 }
1410 } else {
1411 // try AND(SET, SET) -> SET_AND(SET)
1412 Instruction *set0 = src0->getInsn();
1413 Instruction *set1 = src1->getInsn();
1414
1415 if (!set0 || set0->fixed || !set1 || set1->fixed)
1416 return;
1417 if (set1->op != OP_SET) {
1418 Instruction *xchg = set0;
1419 set0 = set1;
1420 set1 = xchg;
1421 if (set1->op != OP_SET)
1422 return;
1423 }
1424 operation redOp = (logop->op == OP_AND ? OP_SET_AND :
1425 logop->op == OP_XOR ? OP_SET_XOR : OP_SET_OR);
1426 if (!prog->getTarget()->isOpSupported(redOp, set1->sType))
1427 return;
1428 if (set0->op != OP_SET &&
1429 set0->op != OP_SET_AND &&
1430 set0->op != OP_SET_OR &&
1431 set0->op != OP_SET_XOR)
1432 return;
1433 if (set0->getDef(0)->refCount() > 1 &&
1434 set1->getDef(0)->refCount() > 1)
1435 return;
1436 if (set0->getPredicate() || set1->getPredicate())
1437 return;
1438 // check that they don't source each other
1439 for (int s = 0; s < 2; ++s)
1440 if (set0->getSrc(s) == set1->getDef(0) ||
1441 set1->getSrc(s) == set0->getDef(0))
1442 return;
1443
1444 set0 = cloneForward(func, set0);
1445 set1 = cloneShallow(func, set1);
1446 logop->bb->insertAfter(logop, set1);
1447 logop->bb->insertAfter(logop, set0);
1448
1449 set0->dType = TYPE_U8;
1450 set0->getDef(0)->reg.file = FILE_PREDICATE;
1451 set0->getDef(0)->reg.size = 1;
1452 set1->setSrc(2, set0->getDef(0));
1453 set1->op = redOp;
1454 set1->setDef(0, logop->getDef(0));
1455 delete_Instruction(prog, logop);
1456 }
1457 }
1458
1459 // F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0
1460 // nv50:
1461 // F2I(NEG(I2F(ABS(SET))))
1462 void
1463 AlgebraicOpt::handleCVT(Instruction *cvt)
1464 {
1465 if (cvt->sType != TYPE_F32 ||
1466 cvt->dType != TYPE_S32 || cvt->src(0).mod != Modifier(0))
1467 return;
1468 Instruction *insn = cvt->getSrc(0)->getInsn();
1469 if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32)
1470 return;
1471 if (insn->src(0).mod != Modifier(0))
1472 return;
1473 insn = insn->getSrc(0)->getInsn();
1474
1475 // check for nv50 SET(-1,0) -> SET(1.0f/0.0f) chain and nvc0's f32 SET
1476 if (insn && insn->op == OP_CVT &&
1477 insn->dType == TYPE_F32 &&
1478 insn->sType == TYPE_S32) {
1479 insn = insn->getSrc(0)->getInsn();
1480 if (!insn || insn->op != OP_ABS || insn->sType != TYPE_S32 ||
1481 insn->src(0).mod)
1482 return;
1483 insn = insn->getSrc(0)->getInsn();
1484 if (!insn || insn->op != OP_SET || insn->dType != TYPE_U32)
1485 return;
1486 } else
1487 if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32) {
1488 return;
1489 }
1490
1491 Instruction *bset = cloneShallow(func, insn);
1492 bset->dType = TYPE_U32;
1493 bset->setDef(0, cvt->getDef(0));
1494 cvt->bb->insertAfter(cvt, bset);
1495 delete_Instruction(prog, cvt);
1496 }
1497
1498 // SUCLAMP dst, (ADD b imm), k, 0 -> SUCLAMP dst, b, k, imm (if imm fits s6)
1499 void
1500 AlgebraicOpt::handleSUCLAMP(Instruction *insn)
1501 {
1502 ImmediateValue imm;
1503 int32_t val = insn->getSrc(2)->asImm()->reg.data.s32;
1504 int s;
1505 Instruction *add;
1506
1507 assert(insn->srcExists(0) && insn->src(0).getFile() == FILE_GPR);
1508
1509 // look for ADD (TODO: only count references by non-SUCLAMP)
1510 if (insn->getSrc(0)->refCount() > 1)
1511 return;
1512 add = insn->getSrc(0)->getInsn();
1513 if (!add || add->op != OP_ADD ||
1514 (add->dType != TYPE_U32 &&
1515 add->dType != TYPE_S32))
1516 return;
1517
1518 // look for immediate
1519 for (s = 0; s < 2; ++s)
1520 if (add->src(s).getImmediate(imm))
1521 break;
1522 if (s >= 2)
1523 return;
1524 s = s ? 0 : 1;
1525 // determine if immediate fits
1526 val += imm.reg.data.s32;
1527 if (val > 31 || val < -32)
1528 return;
1529 // determine if other addend fits
1530 if (add->src(s).getFile() != FILE_GPR || add->src(s).mod != Modifier(0))
1531 return;
1532
1533 bld.setPosition(insn, false); // make sure bld is init'ed
1534 // replace sources
1535 insn->setSrc(2, bld.mkImm(val));
1536 insn->setSrc(0, add->getSrc(s));
1537 }
1538
1539 bool
1540 AlgebraicOpt::visit(BasicBlock *bb)
1541 {
1542 Instruction *next;
1543 for (Instruction *i = bb->getEntry(); i; i = next) {
1544 next = i->next;
1545 switch (i->op) {
1546 case OP_ABS:
1547 handleABS(i);
1548 break;
1549 case OP_ADD:
1550 handleADD(i);
1551 break;
1552 case OP_RCP:
1553 handleRCP(i);
1554 break;
1555 case OP_MIN:
1556 case OP_MAX:
1557 handleMINMAX(i);
1558 break;
1559 case OP_SLCT:
1560 handleSLCT(i);
1561 break;
1562 case OP_AND:
1563 case OP_OR:
1564 case OP_XOR:
1565 handleLOGOP(i);
1566 break;
1567 case OP_CVT:
1568 handleCVT(i);
1569 break;
1570 case OP_SUCLAMP:
1571 handleSUCLAMP(i);
1572 break;
1573 default:
1574 break;
1575 }
1576 }
1577
1578 return true;
1579 }
1580
1581 // =============================================================================
1582
1583 static inline void
1584 updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn)
1585 {
1586 if (offset != ldst->getSrc(0)->reg.data.offset) {
1587 if (ldst->getSrc(0)->refCount() > 1)
1588 ldst->setSrc(0, cloneShallow(fn, ldst->getSrc(0)));
1589 ldst->getSrc(0)->reg.data.offset = offset;
1590 }
1591 }
1592
1593 // Combine loads and stores, forward stores to loads where possible.
1594 class MemoryOpt : public Pass
1595 {
1596 private:
1597 class Record
1598 {
1599 public:
1600 Record *next;
1601 Instruction *insn;
1602 const Value *rel[2];
1603 const Value *base;
1604 int32_t offset;
1605 int8_t fileIndex;
1606 uint8_t size;
1607 bool locked;
1608 Record *prev;
1609
1610 bool overlaps(const Instruction *ldst) const;
1611
1612 inline void link(Record **);
1613 inline void unlink(Record **);
1614 inline void set(const Instruction *ldst);
1615 };
1616
1617 public:
1618 MemoryOpt();
1619
1620 Record *loads[DATA_FILE_COUNT];
1621 Record *stores[DATA_FILE_COUNT];
1622
1623 MemoryPool recordPool;
1624
1625 private:
1626 virtual bool visit(BasicBlock *);
1627 bool runOpt(BasicBlock *);
1628
1629 Record **getList(const Instruction *);
1630
1631 Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const;
1632
1633 // merge @insn into load/store instruction from @rec
1634 bool combineLd(Record *rec, Instruction *ld);
1635 bool combineSt(Record *rec, Instruction *st);
1636
1637 bool replaceLdFromLd(Instruction *ld, Record *ldRec);
1638 bool replaceLdFromSt(Instruction *ld, Record *stRec);
1639 bool replaceStFromSt(Instruction *restrict st, Record *stRec);
1640
1641 void addRecord(Instruction *ldst);
1642 void purgeRecords(Instruction *const st, DataFile);
1643 void lockStores(Instruction *const ld);
1644 void reset();
1645
1646 private:
1647 Record *prevRecord;
1648 };
1649
1650 MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6)
1651 {
1652 for (int i = 0; i < DATA_FILE_COUNT; ++i) {
1653 loads[i] = NULL;
1654 stores[i] = NULL;
1655 }
1656 prevRecord = NULL;
1657 }
1658
1659 void
1660 MemoryOpt::reset()
1661 {
1662 for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) {
1663 Record *it, *next;
1664 for (it = loads[i]; it; it = next) {
1665 next = it->next;
1666 recordPool.release(it);
1667 }
1668 loads[i] = NULL;
1669 for (it = stores[i]; it; it = next) {
1670 next = it->next;
1671 recordPool.release(it);
1672 }
1673 stores[i] = NULL;
1674 }
1675 }
1676
1677 bool
1678 MemoryOpt::combineLd(Record *rec, Instruction *ld)
1679 {
1680 int32_t offRc = rec->offset;
1681 int32_t offLd = ld->getSrc(0)->reg.data.offset;
1682 int sizeRc = rec->size;
1683 int sizeLd = typeSizeof(ld->dType);
1684 int size = sizeRc + sizeLd;
1685 int d, j;
1686
1687 if (!prog->getTarget()->
1688 isAccessSupported(ld->getSrc(0)->reg.file, typeOfSize(size)))
1689 return false;
1690 // no unaligned loads
1691 if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) ||
1692 ((size == 0xc) && (MIN2(offLd, offRc) & 0xf)))
1693 return false;
1694
1695 assert(sizeRc + sizeLd <= 16 && offRc != offLd);
1696
1697 for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j);
1698
1699 if (offLd < offRc) {
1700 int sz;
1701 for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d);
1702 // d: nr of definitions in ld
1703 // j: nr of definitions in rec->insn, move:
1704 for (d = d + j - 1; j > 0; --j, --d)
1705 rec->insn->setDef(d, rec->insn->getDef(j - 1));
1706
1707 if (rec->insn->getSrc(0)->refCount() > 1)
1708 rec->insn->setSrc(0, cloneShallow(func, rec->insn->getSrc(0)));
1709 rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd;
1710
1711 d = 0;
1712 } else {
1713 d = j;
1714 }
1715 // move definitions of @ld to @rec->insn
1716 for (j = 0; sizeLd; ++j, ++d) {
1717 sizeLd -= ld->getDef(j)->reg.size;
1718 rec->insn->setDef(d, ld->getDef(j));
1719 }
1720
1721 rec->size = size;
1722 rec->insn->getSrc(0)->reg.size = size;
1723 rec->insn->setType(typeOfSize(size));
1724
1725 delete_Instruction(prog, ld);
1726
1727 return true;
1728 }
1729
1730 bool
1731 MemoryOpt::combineSt(Record *rec, Instruction *st)
1732 {
1733 int32_t offRc = rec->offset;
1734 int32_t offSt = st->getSrc(0)->reg.data.offset;
1735 int sizeRc = rec->size;
1736 int sizeSt = typeSizeof(st->dType);
1737 int s = sizeSt / 4;
1738 int size = sizeRc + sizeSt;
1739 int j, k;
1740 Value *src[4]; // no modifiers in ValueRef allowed for st
1741 Value *extra[3];
1742
1743 if (!prog->getTarget()->
1744 isAccessSupported(st->getSrc(0)->reg.file, typeOfSize(size)))
1745 return false;
1746 if (size == 8 && MIN2(offRc, offSt) & 0x7)
1747 return false;
1748
1749 st->takeExtraSources(0, extra); // save predicate and indirect address
1750
1751 if (offRc < offSt) {
1752 // save values from @st
1753 for (s = 0; sizeSt; ++s) {
1754 sizeSt -= st->getSrc(s + 1)->reg.size;
1755 src[s] = st->getSrc(s + 1);
1756 }
1757 // set record's values as low sources of @st
1758 for (j = 1; sizeRc; ++j) {
1759 sizeRc -= rec->insn->getSrc(j)->reg.size;
1760 st->setSrc(j, rec->insn->getSrc(j));
1761 }
1762 // set saved values as high sources of @st
1763 for (k = j, j = 0; j < s; ++j)
1764 st->setSrc(k++, src[j]);
1765
1766 updateLdStOffset(st, offRc, func);
1767 } else {
1768 for (j = 1; sizeSt; ++j)
1769 sizeSt -= st->getSrc(j)->reg.size;
1770 for (s = 1; sizeRc; ++j, ++s) {
1771 sizeRc -= rec->insn->getSrc(s)->reg.size;
1772 st->setSrc(j, rec->insn->getSrc(s));
1773 }
1774 rec->offset = offSt;
1775 }
1776 st->putExtraSources(0, extra); // restore pointer and predicate
1777
1778 delete_Instruction(prog, rec->insn);
1779 rec->insn = st;
1780 rec->size = size;
1781 rec->insn->getSrc(0)->reg.size = size;
1782 rec->insn->setType(typeOfSize(size));
1783 return true;
1784 }
1785
1786 void
1787 MemoryOpt::Record::set(const Instruction *ldst)
1788 {
1789 const Symbol *mem = ldst->getSrc(0)->asSym();
1790 fileIndex = mem->reg.fileIndex;
1791 rel[0] = ldst->getIndirect(0, 0);
1792 rel[1] = ldst->getIndirect(0, 1);
1793 offset = mem->reg.data.offset;
1794 base = mem->getBase();
1795 size = typeSizeof(ldst->sType);
1796 }
1797
1798 void
1799 MemoryOpt::Record::link(Record **list)
1800 {
1801 next = *list;
1802 if (next)
1803 next->prev = this;
1804 prev = NULL;
1805 *list = this;
1806 }
1807
1808 void
1809 MemoryOpt::Record::unlink(Record **list)
1810 {
1811 if (next)
1812 next->prev = prev;
1813 if (prev)
1814 prev->next = next;
1815 else
1816 *list = next;
1817 }
1818
1819 MemoryOpt::Record **
1820 MemoryOpt::getList(const Instruction *insn)
1821 {
1822 if (insn->op == OP_LOAD || insn->op == OP_VFETCH)
1823 return &loads[insn->src(0).getFile()];
1824 return &stores[insn->src(0).getFile()];
1825 }
1826
1827 void
1828 MemoryOpt::addRecord(Instruction *i)
1829 {
1830 Record **list = getList(i);
1831 Record *it = reinterpret_cast<Record *>(recordPool.allocate());
1832
1833 it->link(list);
1834 it->set(i);
1835 it->insn = i;
1836 it->locked = false;
1837 }
1838
1839 MemoryOpt::Record *
1840 MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const
1841 {
1842 const Symbol *sym = insn->getSrc(0)->asSym();
1843 const int size = typeSizeof(insn->sType);
1844 Record *rec = NULL;
1845 Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file];
1846
1847 for (; it; it = it->next) {
1848 if (it->locked && insn->op != OP_LOAD)
1849 continue;
1850 if ((it->offset >> 4) != (sym->reg.data.offset >> 4) ||
1851 it->rel[0] != insn->getIndirect(0, 0) ||
1852 it->fileIndex != sym->reg.fileIndex ||
1853 it->rel[1] != insn->getIndirect(0, 1))
1854 continue;
1855
1856 if (it->offset < sym->reg.data.offset) {
1857 if (it->offset + it->size >= sym->reg.data.offset) {
1858 isAdj = (it->offset + it->size == sym->reg.data.offset);
1859 if (!isAdj)
1860 return it;
1861 if (!(it->offset & 0x7))
1862 rec = it;
1863 }
1864 } else {
1865 isAdj = it->offset != sym->reg.data.offset;
1866 if (size <= it->size && !isAdj)
1867 return it;
1868 else
1869 if (!(sym->reg.data.offset & 0x7))
1870 if (it->offset - size <= sym->reg.data.offset)
1871 rec = it;
1872 }
1873 }
1874 return rec;
1875 }
1876
1877 bool
1878 MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec)
1879 {
1880 Instruction *st = rec->insn;
1881 int32_t offSt = rec->offset;
1882 int32_t offLd = ld->getSrc(0)->reg.data.offset;
1883 int d, s;
1884
1885 for (s = 1; offSt != offLd && st->srcExists(s); ++s)
1886 offSt += st->getSrc(s)->reg.size;
1887 if (offSt != offLd)
1888 return false;
1889
1890 for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) {
1891 if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size)
1892 return false;
1893 if (st->getSrc(s)->reg.file != FILE_GPR)
1894 return false;
1895 ld->def(d).replace(st->src(s), false);
1896 }
1897 ld->bb->remove(ld);
1898 return true;
1899 }
1900
1901 bool
1902 MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec)
1903 {
1904 Instruction *ldR = rec->insn;
1905 int32_t offR = rec->offset;
1906 int32_t offE = ldE->getSrc(0)->reg.data.offset;
1907 int dR, dE;
1908
1909 assert(offR <= offE);
1910 for (dR = 0; offR < offE && ldR->defExists(dR); ++dR)
1911 offR += ldR->getDef(dR)->reg.size;
1912 if (offR != offE)
1913 return false;
1914
1915 for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) {
1916 if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size)
1917 return false;
1918 ldE->def(dE).replace(ldR->getDef(dR), false);
1919 }
1920
1921 delete_Instruction(prog, ldE);
1922 return true;
1923 }
1924
1925 bool
1926 MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec)
1927 {
1928 const Instruction *const ri = rec->insn;
1929 Value *extra[3];
1930
1931 int32_t offS = st->getSrc(0)->reg.data.offset;
1932 int32_t offR = rec->offset;
1933 int32_t endS = offS + typeSizeof(st->dType);
1934 int32_t endR = offR + typeSizeof(ri->dType);
1935
1936 rec->size = MAX2(endS, endR) - MIN2(offS, offR);
1937
1938 st->takeExtraSources(0, extra);
1939
1940 if (offR < offS) {
1941 Value *vals[10];
1942 int s, n;
1943 int k = 0;
1944 // get non-replaced sources of ri
1945 for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s)
1946 vals[k++] = ri->getSrc(s);
1947 n = s;
1948 // get replaced sources of st
1949 for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s)
1950 vals[k++] = st->getSrc(s);
1951 // skip replaced sources of ri
1952 for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s);
1953 // get non-replaced sources after values covered by st
1954 for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s)
1955 vals[k++] = ri->getSrc(s);
1956 assert((unsigned int)k <= Elements(vals));
1957 for (s = 0; s < k; ++s)
1958 st->setSrc(s + 1, vals[s]);
1959 st->setSrc(0, ri->getSrc(0));
1960 } else
1961 if (endR > endS) {
1962 int j, s;
1963 for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size);
1964 for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size);
1965 for (; offR < endR; offR += ri->getSrc(j++)->reg.size)
1966 st->setSrc(s++, ri->getSrc(j));
1967 }
1968 st->putExtraSources(0, extra);
1969
1970 delete_Instruction(prog, rec->insn);
1971
1972 rec->insn = st;
1973 rec->offset = st->getSrc(0)->reg.data.offset;
1974
1975 st->setType(typeOfSize(rec->size));
1976
1977 return true;
1978 }
1979
1980 bool
1981 MemoryOpt::Record::overlaps(const Instruction *ldst) const
1982 {
1983 Record that;
1984 that.set(ldst);
1985
1986 if (this->fileIndex != that.fileIndex)
1987 return false;
1988
1989 if (this->rel[0] || that.rel[0])
1990 return this->base == that.base;
1991 return
1992 (this->offset < that.offset + that.size) &&
1993 (this->offset + this->size > that.offset);
1994 }
1995
1996 // We must not eliminate stores that affect the result of @ld if
1997 // we find later stores to the same location, and we may no longer
1998 // merge them with later stores.
1999 // The stored value can, however, still be used to determine the value
2000 // returned by future loads.
2001 void
2002 MemoryOpt::lockStores(Instruction *const ld)
2003 {
2004 for (Record *r = stores[ld->src(0).getFile()]; r; r = r->next)
2005 if (!r->locked && r->overlaps(ld))
2006 r->locked = true;
2007 }
2008
2009 // Prior loads from the location of @st are no longer valid.
2010 // Stores to the location of @st may no longer be used to derive
2011 // the value at it nor be coalesced into later stores.
2012 void
2013 MemoryOpt::purgeRecords(Instruction *const st, DataFile f)
2014 {
2015 if (st)
2016 f = st->src(0).getFile();
2017
2018 for (Record *r = loads[f]; r; r = r->next)
2019 if (!st || r->overlaps(st))
2020 r->unlink(&loads[f]);
2021
2022 for (Record *r = stores[f]; r; r = r->next)
2023 if (!st || r->overlaps(st))
2024 r->unlink(&stores[f]);
2025 }
2026
2027 bool
2028 MemoryOpt::visit(BasicBlock *bb)
2029 {
2030 bool ret = runOpt(bb);
2031 // Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st
2032 // where 96 bit memory operations are forbidden.
2033 if (ret)
2034 ret = runOpt(bb);
2035 return ret;
2036 }
2037
2038 bool
2039 MemoryOpt::runOpt(BasicBlock *bb)
2040 {
2041 Instruction *ldst, *next;
2042 Record *rec;
2043 bool isAdjacent = true;
2044
2045 for (ldst = bb->getEntry(); ldst; ldst = next) {
2046 bool keep = true;
2047 bool isLoad = true;
2048 next = ldst->next;
2049
2050 if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) {
2051 if (ldst->isDead()) {
2052 // might have been produced by earlier optimization
2053 delete_Instruction(prog, ldst);
2054 continue;
2055 }
2056 } else
2057 if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) {
2058 isLoad = false;
2059 } else {
2060 // TODO: maybe have all fixed ops act as barrier ?
2061 if (ldst->op == OP_CALL ||
2062 ldst->op == OP_BAR ||
2063 ldst->op == OP_MEMBAR) {
2064 purgeRecords(NULL, FILE_MEMORY_LOCAL);
2065 purgeRecords(NULL, FILE_MEMORY_GLOBAL);
2066 purgeRecords(NULL, FILE_MEMORY_SHARED);
2067 purgeRecords(NULL, FILE_SHADER_OUTPUT);
2068 } else
2069 if (ldst->op == OP_ATOM || ldst->op == OP_CCTL) {
2070 if (ldst->src(0).getFile() == FILE_MEMORY_GLOBAL) {
2071 purgeRecords(NULL, FILE_MEMORY_LOCAL);
2072 purgeRecords(NULL, FILE_MEMORY_GLOBAL);
2073 purgeRecords(NULL, FILE_MEMORY_SHARED);
2074 } else {
2075 purgeRecords(NULL, ldst->src(0).getFile());
2076 }
2077 } else
2078 if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) {
2079 purgeRecords(NULL, FILE_SHADER_OUTPUT);
2080 }
2081 continue;
2082 }
2083 if (ldst->getPredicate()) // TODO: handle predicated ld/st
2084 continue;
2085
2086 if (isLoad) {
2087 DataFile file = ldst->src(0).getFile();
2088
2089 // if ld l[]/g[] look for previous store to eliminate the reload
2090 if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) {
2091 // TODO: shared memory ?
2092 rec = findRecord(ldst, false, isAdjacent);
2093 if (rec && !isAdjacent)
2094 keep = !replaceLdFromSt(ldst, rec);
2095 }
2096
2097 // or look for ld from the same location and replace this one
2098 rec = keep ? findRecord(ldst, true, isAdjacent) : NULL;
2099 if (rec) {
2100 if (!isAdjacent)
2101 keep = !replaceLdFromLd(ldst, rec);
2102 else
2103 // or combine a previous load with this one
2104 keep = !combineLd(rec, ldst);
2105 }
2106 if (keep)
2107 lockStores(ldst);
2108 } else {
2109 rec = findRecord(ldst, false, isAdjacent);
2110 if (rec) {
2111 if (!isAdjacent)
2112 keep = !replaceStFromSt(ldst, rec);
2113 else
2114 keep = !combineSt(rec, ldst);
2115 }
2116 if (keep)
2117 purgeRecords(ldst, DATA_FILE_COUNT);
2118 }
2119 if (keep)
2120 addRecord(ldst);
2121 }
2122 reset();
2123
2124 return true;
2125 }
2126
2127 // =============================================================================
2128
2129 // Turn control flow into predicated instructions (after register allocation !).
2130 // TODO:
2131 // Could move this to before register allocation on NVC0 and also handle nested
2132 // constructs.
2133 class FlatteningPass : public Pass
2134 {
2135 private:
2136 virtual bool visit(BasicBlock *);
2137
2138 bool tryPredicateConditional(BasicBlock *);
2139 void predicateInstructions(BasicBlock *, Value *pred, CondCode cc);
2140 void tryPropagateBranch(BasicBlock *);
2141 inline bool isConstantCondition(Value *pred);
2142 inline bool mayPredicate(const Instruction *, const Value *pred) const;
2143 inline void removeFlow(Instruction *);
2144 };
2145
2146 bool
2147 FlatteningPass::isConstantCondition(Value *pred)
2148 {
2149 Instruction *insn = pred->getUniqueInsn();
2150 assert(insn);
2151 if (insn->op != OP_SET || insn->srcExists(2))
2152 return false;
2153
2154 for (int s = 0; s < 2 && insn->srcExists(s); ++s) {
2155 Instruction *ld = insn->getSrc(s)->getUniqueInsn();
2156 DataFile file;
2157 if (ld) {
2158 if (ld->op != OP_MOV && ld->op != OP_LOAD)
2159 return false;
2160 if (ld->src(0).isIndirect(0))
2161 return false;
2162 file = ld->src(0).getFile();
2163 } else {
2164 file = insn->src(s).getFile();
2165 // catch $r63 on NVC0
2166 if (file == FILE_GPR && insn->getSrc(s)->reg.data.id > prog->maxGPR)
2167 file = FILE_IMMEDIATE;
2168 }
2169 if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST)
2170 return false;
2171 }
2172 return true;
2173 }
2174
2175 void
2176 FlatteningPass::removeFlow(Instruction *insn)
2177 {
2178 FlowInstruction *term = insn ? insn->asFlow() : NULL;
2179 if (!term)
2180 return;
2181 Graph::Edge::Type ty = term->bb->cfg.outgoing().getType();
2182
2183 if (term->op == OP_BRA) {
2184 // TODO: this might get more difficult when we get arbitrary BRAs
2185 if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK)
2186 return;
2187 } else
2188 if (term->op != OP_JOIN)
2189 return;
2190
2191 Value *pred = term->getPredicate();
2192
2193 delete_Instruction(prog, term);
2194
2195 if (pred && pred->refCount() == 0) {
2196 Instruction *pSet = pred->getUniqueInsn();
2197 pred->join->reg.data.id = -1; // deallocate
2198 if (pSet->isDead())
2199 delete_Instruction(prog, pSet);
2200 }
2201 }
2202
2203 void
2204 FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc)
2205 {
2206 for (Instruction *i = bb->getEntry(); i; i = i->next) {
2207 if (i->isNop())
2208 continue;
2209 assert(!i->getPredicate());
2210 i->setPredicate(cc, pred);
2211 }
2212 removeFlow(bb->getExit());
2213 }
2214
2215 bool
2216 FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const
2217 {
2218 if (insn->isPseudo())
2219 return true;
2220 // TODO: calls where we don't know which registers are modified
2221
2222 if (!prog->getTarget()->mayPredicate(insn, pred))
2223 return false;
2224 for (int d = 0; insn->defExists(d); ++d)
2225 if (insn->getDef(d)->equals(pred))
2226 return false;
2227 return true;
2228 }
2229
2230 // If we jump to BRA/RET/EXIT, replace the jump with it.
2231 // NOTE: We do not update the CFG anymore here !
2232 //
2233 // TODO: Handle cases where we skip over a branch (maybe do that elsewhere ?):
2234 // BB:0
2235 // @p0 bra BB:2 -> @!p0 bra BB:3 iff (!) BB:2 immediately adjoins BB:1
2236 // BB1:
2237 // bra BB:3
2238 // BB2:
2239 // ...
2240 // BB3:
2241 // ...
2242 void
2243 FlatteningPass::tryPropagateBranch(BasicBlock *bb)
2244 {
2245 for (Instruction *i = bb->getExit(); i && i->op == OP_BRA; i = i->prev) {
2246 BasicBlock *bf = i->asFlow()->target.bb;
2247
2248 if (bf->getInsnCount() != 1)
2249 continue;
2250
2251 FlowInstruction *bra = i->asFlow();
2252 FlowInstruction *rep = bf->getExit()->asFlow();
2253
2254 if (!rep || rep->getPredicate())
2255 continue;
2256 if (rep->op != OP_BRA &&
2257 rep->op != OP_JOIN &&
2258 rep->op != OP_EXIT)
2259 continue;
2260
2261 // TODO: If there are multiple branches to @rep, only the first would
2262 // be replaced, so only remove them after this pass is done ?
2263 // Also, need to check all incident blocks for fall-through exits and
2264 // add the branch there.
2265 bra->op = rep->op;
2266 bra->target.bb = rep->target.bb;
2267 if (bf->cfg.incidentCount() == 1)
2268 bf->remove(rep);
2269 }
2270 }
2271
2272 bool
2273 FlatteningPass::visit(BasicBlock *bb)
2274 {
2275 if (tryPredicateConditional(bb))
2276 return true;
2277
2278 // try to attach join to previous instruction
2279 if (prog->getTarget()->hasJoin) {
2280 Instruction *insn = bb->getExit();
2281 if (insn && insn->op == OP_JOIN && !insn->getPredicate()) {
2282 insn = insn->prev;
2283 if (insn && !insn->getPredicate() &&
2284 !insn->asFlow() &&
2285 insn->op != OP_TEXBAR &&
2286 !isTextureOp(insn->op) && // probably just nve4
2287 !isSurfaceOp(insn->op) && // not confirmed
2288 insn->op != OP_LINTERP && // probably just nve4
2289 insn->op != OP_PINTERP && // probably just nve4
2290 ((insn->op != OP_LOAD && insn->op != OP_STORE) ||
2291 (typeSizeof(insn->dType) <= 4 && !insn->src(0).isIndirect(0))) &&
2292 !insn->isNop()) {
2293 insn->join = 1;
2294 bb->remove(bb->getExit());
2295 return true;
2296 }
2297 }
2298 }
2299
2300 tryPropagateBranch(bb);
2301
2302 return true;
2303 }
2304
2305 bool
2306 FlatteningPass::tryPredicateConditional(BasicBlock *bb)
2307 {
2308 BasicBlock *bL = NULL, *bR = NULL;
2309 unsigned int nL = 0, nR = 0, limit = 12;
2310 Instruction *insn;
2311 unsigned int mask;
2312
2313 mask = bb->initiatesSimpleConditional();
2314 if (!mask)
2315 return false;
2316
2317 assert(bb->getExit());
2318 Value *pred = bb->getExit()->getPredicate();
2319 assert(pred);
2320
2321 if (isConstantCondition(pred))
2322 limit = 4;
2323
2324 Graph::EdgeIterator ei = bb->cfg.outgoing();
2325
2326 if (mask & 1) {
2327 bL = BasicBlock::get(ei.getNode());
2328 for (insn = bL->getEntry(); insn; insn = insn->next, ++nL)
2329 if (!mayPredicate(insn, pred))
2330 return false;
2331 if (nL > limit)
2332 return false; // too long, do a real branch
2333 }
2334 ei.next();
2335
2336 if (mask & 2) {
2337 bR = BasicBlock::get(ei.getNode());
2338 for (insn = bR->getEntry(); insn; insn = insn->next, ++nR)
2339 if (!mayPredicate(insn, pred))
2340 return false;
2341 if (nR > limit)
2342 return false; // too long, do a real branch
2343 }
2344
2345 if (bL)
2346 predicateInstructions(bL, pred, bb->getExit()->cc);
2347 if (bR)
2348 predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc));
2349
2350 if (bb->joinAt) {
2351 bb->remove(bb->joinAt);
2352 bb->joinAt = NULL;
2353 }
2354 removeFlow(bb->getExit()); // delete the branch/join at the fork point
2355
2356 // remove potential join operations at the end of the conditional
2357 if (prog->getTarget()->joinAnterior) {
2358 bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode());
2359 if (bb->getEntry() && bb->getEntry()->op == OP_JOIN)
2360 removeFlow(bb->getEntry());
2361 }
2362
2363 return true;
2364 }
2365
2366 // =============================================================================
2367
2368 // Fold Immediate into MAD; must be done after register allocation due to
2369 // constraint SDST == SSRC2
2370 // TODO:
2371 // Does NVC0+ have other situations where this pass makes sense?
2372 class NV50PostRaConstantFolding : public Pass
2373 {
2374 private:
2375 virtual bool visit(BasicBlock *);
2376 };
2377
2378 bool
2379 NV50PostRaConstantFolding::visit(BasicBlock *bb)
2380 {
2381 Value *vtmp;
2382 Instruction *def;
2383
2384 for (Instruction *i = bb->getFirst(); i; i = i->next) {
2385 switch (i->op) {
2386 case OP_MAD:
2387 if (i->def(0).getFile() != FILE_GPR ||
2388 i->src(0).getFile() != FILE_GPR ||
2389 i->src(1).getFile() != FILE_GPR ||
2390 i->src(2).getFile() != FILE_GPR ||
2391 i->getDef(0)->reg.data.id != i->getSrc(2)->reg.data.id ||
2392 !isFloatType(i->dType))
2393 break;
2394
2395 def = i->getSrc(1)->getInsn();
2396 if (def->op == OP_MOV && def->src(0).getFile() == FILE_IMMEDIATE) {
2397 vtmp = i->getSrc(1);
2398 i->setSrc(1, def->getSrc(0));
2399
2400 /* There's no post-RA dead code elimination, so do it here
2401 * XXX: if we add more code-removing post-RA passes, we might
2402 * want to create a post-RA dead-code elim pass */
2403 if (vtmp->refCount() == 0)
2404 delete_Instruction(bb->getProgram(), def);
2405
2406 break;
2407 }
2408 break;
2409 default:
2410 break;
2411 }
2412 }
2413
2414 return true;
2415 }
2416
2417 // =============================================================================
2418
2419 // Common subexpression elimination. Stupid O^2 implementation.
2420 class LocalCSE : public Pass
2421 {
2422 private:
2423 virtual bool visit(BasicBlock *);
2424
2425 inline bool tryReplace(Instruction **, Instruction *);
2426
2427 DLList ops[OP_LAST + 1];
2428 };
2429
2430 class GlobalCSE : public Pass
2431 {
2432 private:
2433 virtual bool visit(BasicBlock *);
2434 };
2435
2436 bool
2437 Instruction::isActionEqual(const Instruction *that) const
2438 {
2439 if (this->op != that->op ||
2440 this->dType != that->dType ||
2441 this->sType != that->sType)
2442 return false;
2443 if (this->cc != that->cc)
2444 return false;
2445
2446 if (this->asTex()) {
2447 if (memcmp(&this->asTex()->tex,
2448 &that->asTex()->tex,
2449 sizeof(this->asTex()->tex)))
2450 return false;
2451 } else
2452 if (this->asCmp()) {
2453 if (this->asCmp()->setCond != that->asCmp()->setCond)
2454 return false;
2455 } else
2456 if (this->asFlow()) {
2457 return false;
2458 } else {
2459 if (this->ipa != that->ipa ||
2460 this->lanes != that->lanes ||
2461 this->perPatch != that->perPatch)
2462 return false;
2463 if (this->postFactor != that->postFactor)
2464 return false;
2465 }
2466
2467 if (this->subOp != that->subOp ||
2468 this->saturate != that->saturate ||
2469 this->rnd != that->rnd ||
2470 this->ftz != that->ftz ||
2471 this->dnz != that->dnz ||
2472 this->cache != that->cache ||
2473 this->mask != that->mask)
2474 return false;
2475
2476 return true;
2477 }
2478
2479 bool
2480 Instruction::isResultEqual(const Instruction *that) const
2481 {
2482 unsigned int d, s;
2483
2484 // NOTE: location of discard only affects tex with liveOnly and quadops
2485 if (!this->defExists(0) && this->op != OP_DISCARD)
2486 return false;
2487
2488 if (!isActionEqual(that))
2489 return false;
2490
2491 if (this->predSrc != that->predSrc)
2492 return false;
2493
2494 for (d = 0; this->defExists(d); ++d) {
2495 if (!that->defExists(d) ||
2496 !this->getDef(d)->equals(that->getDef(d), false))
2497 return false;
2498 }
2499 if (that->defExists(d))
2500 return false;
2501
2502 for (s = 0; this->srcExists(s); ++s) {
2503 if (!that->srcExists(s))
2504 return false;
2505 if (this->src(s).mod != that->src(s).mod)
2506 return false;
2507 if (!this->getSrc(s)->equals(that->getSrc(s), true))
2508 return false;
2509 }
2510 if (that->srcExists(s))
2511 return false;
2512
2513 if (op == OP_LOAD || op == OP_VFETCH) {
2514 switch (src(0).getFile()) {
2515 case FILE_MEMORY_CONST:
2516 case FILE_SHADER_INPUT:
2517 return true;
2518 default:
2519 return false;
2520 }
2521 }
2522
2523 return true;
2524 }
2525
2526 // pull through common expressions from different in-blocks
2527 bool
2528 GlobalCSE::visit(BasicBlock *bb)
2529 {
2530 Instruction *phi, *next, *ik;
2531 int s;
2532
2533 // TODO: maybe do this with OP_UNION, too
2534
2535 for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) {
2536 next = phi->next;
2537 if (phi->getSrc(0)->refCount() > 1)
2538 continue;
2539 ik = phi->getSrc(0)->getInsn();
2540 if (!ik)
2541 continue; // probably a function input
2542 for (s = 1; phi->srcExists(s); ++s) {
2543 if (phi->getSrc(s)->refCount() > 1)
2544 break;
2545 if (!phi->getSrc(s)->getInsn() ||
2546 !phi->getSrc(s)->getInsn()->isResultEqual(ik))
2547 break;
2548 }
2549 if (!phi->srcExists(s)) {
2550 Instruction *entry = bb->getEntry();
2551 ik->bb->remove(ik);
2552 if (!entry || entry->op != OP_JOIN)
2553 bb->insertHead(ik);
2554 else
2555 bb->insertAfter(entry, ik);
2556 ik->setDef(0, phi->getDef(0));
2557 delete_Instruction(prog, phi);
2558 }
2559 }
2560
2561 return true;
2562 }
2563
2564 bool
2565 LocalCSE::tryReplace(Instruction **ptr, Instruction *i)
2566 {
2567 Instruction *old = *ptr;
2568
2569 // TODO: maybe relax this later (causes trouble with OP_UNION)
2570 if (i->isPredicated())
2571 return false;
2572
2573 if (!old->isResultEqual(i))
2574 return false;
2575
2576 for (int d = 0; old->defExists(d); ++d)
2577 old->def(d).replace(i->getDef(d), false);
2578 delete_Instruction(prog, old);
2579 *ptr = NULL;
2580 return true;
2581 }
2582
2583 bool
2584 LocalCSE::visit(BasicBlock *bb)
2585 {
2586 unsigned int replaced;
2587
2588 do {
2589 Instruction *ir, *next;
2590
2591 replaced = 0;
2592
2593 // will need to know the order of instructions
2594 int serial = 0;
2595 for (ir = bb->getFirst(); ir; ir = ir->next)
2596 ir->serial = serial++;
2597
2598 for (ir = bb->getEntry(); ir; ir = next) {
2599 int s;
2600 Value *src = NULL;
2601
2602 next = ir->next;
2603
2604 if (ir->fixed) {
2605 ops[ir->op].insert(ir);
2606 continue;
2607 }
2608
2609 for (s = 0; ir->srcExists(s); ++s)
2610 if (ir->getSrc(s)->asLValue())
2611 if (!src || ir->getSrc(s)->refCount() < src->refCount())
2612 src = ir->getSrc(s);
2613
2614 if (src) {
2615 for (Value::UseIterator it = src->uses.begin();
2616 it != src->uses.end(); ++it) {
2617 Instruction *ik = (*it)->getInsn();
2618 if (ik && ik->bb == ir->bb && ik->serial < ir->serial)
2619 if (tryReplace(&ir, ik))
2620 break;
2621 }
2622 } else {
2623 DLLIST_FOR_EACH(&ops[ir->op], iter)
2624 {
2625 Instruction *ik = reinterpret_cast<Instruction *>(iter.get());
2626 if (tryReplace(&ir, ik))
2627 break;
2628 }
2629 }
2630
2631 if (ir)
2632 ops[ir->op].insert(ir);
2633 else
2634 ++replaced;
2635 }
2636 for (unsigned int i = 0; i <= OP_LAST; ++i)
2637 ops[i].clear();
2638
2639 } while (replaced);
2640
2641 return true;
2642 }
2643
2644 // =============================================================================
2645
2646 // Remove computations of unused values.
2647 class DeadCodeElim : public Pass
2648 {
2649 public:
2650 bool buryAll(Program *);
2651
2652 private:
2653 virtual bool visit(BasicBlock *);
2654
2655 void checkSplitLoad(Instruction *ld); // for partially dead loads
2656
2657 unsigned int deadCount;
2658 };
2659
2660 bool
2661 DeadCodeElim::buryAll(Program *prog)
2662 {
2663 do {
2664 deadCount = 0;
2665 if (!this->run(prog, false, false))
2666 return false;
2667 } while (deadCount);
2668
2669 return true;
2670 }
2671
2672 bool
2673 DeadCodeElim::visit(BasicBlock *bb)
2674 {
2675 Instruction *next;
2676
2677 for (Instruction *i = bb->getFirst(); i; i = next) {
2678 next = i->next;
2679 if (i->isDead()) {
2680 ++deadCount;
2681 delete_Instruction(prog, i);
2682 } else
2683 if (i->defExists(1) && (i->op == OP_VFETCH || i->op == OP_LOAD)) {
2684 checkSplitLoad(i);
2685 } else
2686 if (i->defExists(0) && !i->getDef(0)->refCount()) {
2687 if (i->op == OP_ATOM ||
2688 i->op == OP_SUREDP ||
2689 i->op == OP_SUREDB)
2690 i->setDef(0, NULL);
2691 }
2692 }
2693 return true;
2694 }
2695
2696 void
2697 DeadCodeElim::checkSplitLoad(Instruction *ld1)
2698 {
2699 Instruction *ld2 = NULL; // can get at most 2 loads
2700 Value *def1[4];
2701 Value *def2[4];
2702 int32_t addr1, addr2;
2703 int32_t size1, size2;
2704 int d, n1, n2;
2705 uint32_t mask = 0xffffffff;
2706
2707 for (d = 0; ld1->defExists(d); ++d)
2708 if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0)
2709 mask &= ~(1 << d);
2710 if (mask == 0xffffffff)
2711 return;
2712
2713 addr1 = ld1->getSrc(0)->reg.data.offset;
2714 n1 = n2 = 0;
2715 size1 = size2 = 0;
2716 for (d = 0; ld1->defExists(d); ++d) {
2717 if (mask & (1 << d)) {
2718 if (size1 && (addr1 & 0x7))
2719 break;
2720 def1[n1] = ld1->getDef(d);
2721 size1 += def1[n1++]->reg.size;
2722 } else
2723 if (!n1) {
2724 addr1 += ld1->getDef(d)->reg.size;
2725 } else {
2726 break;
2727 }
2728 }
2729 for (addr2 = addr1 + size1; ld1->defExists(d); ++d) {
2730 if (mask & (1 << d)) {
2731 def2[n2] = ld1->getDef(d);
2732 size2 += def2[n2++]->reg.size;
2733 } else {
2734 assert(!n2);
2735 addr2 += ld1->getDef(d)->reg.size;
2736 }
2737 }
2738
2739 updateLdStOffset(ld1, addr1, func);
2740 ld1->setType(typeOfSize(size1));
2741 for (d = 0; d < 4; ++d)
2742 ld1->setDef(d, (d < n1) ? def1[d] : NULL);
2743
2744 if (!n2)
2745 return;
2746
2747 ld2 = cloneShallow(func, ld1);
2748 updateLdStOffset(ld2, addr2, func);
2749 ld2->setType(typeOfSize(size2));
2750 for (d = 0; d < 4; ++d)
2751 ld2->setDef(d, (d < n2) ? def2[d] : NULL);
2752
2753 ld1->bb->insertAfter(ld1, ld2);
2754 }
2755
2756 // =============================================================================
2757
2758 #define RUN_PASS(l, n, f) \
2759 if (level >= (l)) { \
2760 if (dbgFlags & NV50_IR_DEBUG_VERBOSE) \
2761 INFO("PEEPHOLE: %s\n", #n); \
2762 n pass; \
2763 if (!pass.f(this)) \
2764 return false; \
2765 }
2766
2767 bool
2768 Program::optimizeSSA(int level)
2769 {
2770 RUN_PASS(1, DeadCodeElim, buryAll);
2771 RUN_PASS(1, CopyPropagation, run);
2772 RUN_PASS(1, MergeSplits, run);
2773 RUN_PASS(2, GlobalCSE, run);
2774 RUN_PASS(1, LocalCSE, run);
2775 RUN_PASS(2, AlgebraicOpt, run);
2776 RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks
2777 RUN_PASS(1, ConstantFolding, foldAll);
2778 RUN_PASS(1, LoadPropagation, run);
2779 RUN_PASS(2, MemoryOpt, run);
2780 RUN_PASS(2, LocalCSE, run);
2781 RUN_PASS(0, DeadCodeElim, buryAll);
2782
2783 return true;
2784 }
2785
2786 bool
2787 Program::optimizePostRA(int level)
2788 {
2789 RUN_PASS(2, FlatteningPass, run);
2790 if (getTarget()->getChipset() < 0xc0)
2791 RUN_PASS(2, NV50PostRaConstantFolding, run);
2792
2793 return true;
2794 }
2795
2796 }