projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nvc0/ir: return 0 in imageLoad on incomplete textures
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_peephole.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_target.h"
25 #include "codegen/nv50_ir_build_util.h"
26
27 extern "C" {
28 #include "util/u_math.h"
29 }
30
31 namespace nv50_ir {
32
33 bool
34 Instruction::isNop() const
35 {
36 if (op == OP_PHI || op == OP_SPLIT || op == OP_MERGE || op == OP_CONSTRAINT)
37 return true;
38 if (terminator || join) // XXX: should terminator imply flow ?
39 return false;
40 if (op == OP_ATOM)
41 return false;
42 if (!fixed && op == OP_NOP)
43 return true;
44
45 if (defExists(0) && def(0).rep()->reg.data.id < 0) {
46 for (int d = 1; defExists(d); ++d)
47 if (def(d).rep()->reg.data.id >= 0)
48 WARN("part of vector result is unused !\n");
49 return true;
50 }
51
52 if (op == OP_MOV || op == OP_UNION) {
53 if (!getDef(0)->equals(getSrc(0)))
54 return false;
55 if (op == OP_UNION)
56 if (!def(0).rep()->equals(getSrc(1)))
57 return false;
58 return true;
59 }
60
61 return false;
62 }
63
64 bool Instruction::isDead() const
65 {
66 if (op == OP_STORE ||
67 op == OP_EXPORT ||
68 op == OP_ATOM ||
69 op == OP_SUSTB || op == OP_SUSTP || op == OP_SUREDP || op == OP_SUREDB ||
70 op == OP_WRSV)
71 return false;
72
73 for (int d = 0; defExists(d); ++d)
74 if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0)
75 return false;
76
77 if (terminator || asFlow())
78 return false;
79 if (fixed)
80 return false;
81
82 return true;
83 };
84
85 // =============================================================================
86
87 class CopyPropagation : public Pass
88 {
89 private:
90 virtual bool visit(BasicBlock *);
91 };
92
93 // Propagate all MOVs forward to make subsequent optimization easier, except if
94 // the sources stem from a phi, in which case we don't want to mess up potential
95 // swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def.
96 bool
97 CopyPropagation::visit(BasicBlock *bb)
98 {
99 Instruction *mov, *si, *next;
100
101 for (mov = bb->getEntry(); mov; mov = next) {
102 next = mov->next;
103 if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue())
104 continue;
105 if (mov->getPredicate())
106 continue;
107 if (mov->def(0).getFile() != mov->src(0).getFile())
108 continue;
109 si = mov->getSrc(0)->getInsn();
110 if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) {
111 // propagate
112 mov->def(0).replace(mov->getSrc(0), false);
113 delete_Instruction(prog, mov);
114 }
115 }
116 return true;
117 }
118
119 // =============================================================================
120
121 class MergeSplits : public Pass
122 {
123 private:
124 virtual bool visit(BasicBlock *);
125 };
126
127 // For SPLIT / MERGE pairs that operate on the same registers, replace the
128 // post-merge def with the SPLIT's source.
129 bool
130 MergeSplits::visit(BasicBlock *bb)
131 {
132 Instruction *i, *next, *si;
133
134 for (i = bb->getEntry(); i; i = next) {
135 next = i->next;
136 if (i->op != OP_MERGE || typeSizeof(i->dType) != 8)
137 continue;
138 si = i->getSrc(0)->getInsn();
139 if (si->op != OP_SPLIT || si != i->getSrc(1)->getInsn())
140 continue;
141 i->def(0).replace(si->getSrc(0), false);
142 delete_Instruction(prog, i);
143 }
144
145 return true;
146 }
147
148 // =============================================================================
149
150 class LoadPropagation : public Pass
151 {
152 private:
153 virtual bool visit(BasicBlock *);
154
155 void checkSwapSrc01(Instruction *);
156
157 bool isCSpaceLoad(Instruction *);
158 bool isImmdLoad(Instruction *);
159 bool isAttribOrSharedLoad(Instruction *);
160 };
161
162 bool
163 LoadPropagation::isCSpaceLoad(Instruction *ld)
164 {
165 return ld && ld->op == OP_LOAD && ld->src(0).getFile() == FILE_MEMORY_CONST;
166 }
167
168 bool
169 LoadPropagation::isImmdLoad(Instruction *ld)
170 {
171 if (!ld || (ld->op != OP_MOV) ||
172 ((typeSizeof(ld->dType) != 4) && (typeSizeof(ld->dType) != 8)))
173 return false;
174
175 // A 0 can be replaced with a register, so it doesn't count as an immediate.
176 ImmediateValue val;
177 return ld->src(0).getImmediate(val) && !val.isInteger(0);
178 }
179
180 bool
181 LoadPropagation::isAttribOrSharedLoad(Instruction *ld)
182 {
183 return ld &&
184 (ld->op == OP_VFETCH ||
185 (ld->op == OP_LOAD &&
186 (ld->src(0).getFile() == FILE_SHADER_INPUT ||
187 ld->src(0).getFile() == FILE_MEMORY_SHARED)));
188 }
189
190 void
191 LoadPropagation::checkSwapSrc01(Instruction *insn)
192 {
193 const Target *targ = prog->getTarget();
194 if (!targ->getOpInfo(insn).commutative)
195 if (insn->op != OP_SET && insn->op != OP_SLCT && insn->op != OP_SUB)
196 return;
197 if (insn->src(1).getFile() != FILE_GPR)
198 return;
199 // This is the special OP_SET used for alphatesting, we can't reverse its
200 // arguments as that will confuse the fixup code.
201 if (insn->op == OP_SET && insn->subOp)
202 return;
203
204 Instruction *i0 = insn->getSrc(0)->getInsn();
205 Instruction *i1 = insn->getSrc(1)->getInsn();
206
207 // Swap sources to inline the less frequently used source. That way,
208 // optimistically, it will eventually be able to remove the instruction.
209 int i0refs = insn->getSrc(0)->refCount();
210 int i1refs = insn->getSrc(1)->refCount();
211
212 if ((isCSpaceLoad(i0) || isImmdLoad(i0)) && targ->insnCanLoad(insn, 1, i0)) {
213 if ((!isImmdLoad(i1) && !isCSpaceLoad(i1)) ||
214 !targ->insnCanLoad(insn, 1, i1) ||
215 i0refs < i1refs)
216 insn->swapSources(0, 1);
217 else
218 return;
219 } else
220 if (isAttribOrSharedLoad(i1)) {
221 if (!isAttribOrSharedLoad(i0))
222 insn->swapSources(0, 1);
223 else
224 return;
225 } else {
226 return;
227 }
228
229 if (insn->op == OP_SET || insn->op == OP_SET_AND ||
230 insn->op == OP_SET_OR || insn->op == OP_SET_XOR)
231 insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond);
232 else
233 if (insn->op == OP_SLCT)
234 insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond);
235 else
236 if (insn->op == OP_SUB) {
237 insn->src(0).mod = insn->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
238 insn->src(1).mod = insn->src(1).mod ^ Modifier(NV50_IR_MOD_NEG);
239 }
240 }
241
242 bool
243 LoadPropagation::visit(BasicBlock *bb)
244 {
245 const Target *targ = prog->getTarget();
246 Instruction *next;
247
248 for (Instruction *i = bb->getEntry(); i; i = next) {
249 next = i->next;
250
251 if (i->op == OP_CALL) // calls have args as sources, they must be in regs
252 continue;
253
254 if (i->op == OP_PFETCH) // pfetch expects arg1 to be a reg
255 continue;
256
257 if (i->srcExists(1))
258 checkSwapSrc01(i);
259
260 for (int s = 0; i->srcExists(s); ++s) {
261 Instruction *ld = i->getSrc(s)->getInsn();
262
263 if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV))
264 continue;
265 if (!targ->insnCanLoad(i, s, ld))
266 continue;
267
268 // propagate !
269 i->setSrc(s, ld->getSrc(0));
270 if (ld->src(0).isIndirect(0))
271 i->setIndirect(s, 0, ld->getIndirect(0, 0));
272
273 if (ld->getDef(0)->refCount() == 0)
274 delete_Instruction(prog, ld);
275 }
276 }
277 return true;
278 }
279
280 // =============================================================================
281
282 class IndirectPropagation : public Pass
283 {
284 private:
285 virtual bool visit(BasicBlock *);
286
287 BuildUtil bld;
288 };
289
290 bool
291 IndirectPropagation::visit(BasicBlock *bb)
292 {
293 const Target *targ = prog->getTarget();
294 Instruction *next;
295
296 for (Instruction *i = bb->getEntry(); i; i = next) {
297 next = i->next;
298
299 bld.setPosition(i, false);
300
301 for (int s = 0; i->srcExists(s); ++s) {
302 Instruction *insn;
303 ImmediateValue imm;
304 if (!i->src(s).isIndirect(0))
305 continue;
306 insn = i->getIndirect(s, 0)->getInsn();
307 if (!insn)
308 continue;
309 if (insn->op == OP_ADD && !isFloatType(insn->dType)) {
310 if (insn->src(0).getFile() != targ->nativeFile(FILE_ADDRESS) ||
311 !insn->src(1).getImmediate(imm) ||
312 !targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
313 continue;
314 i->setIndirect(s, 0, insn->getSrc(0));
315 i->setSrc(s, cloneShallow(func, i->getSrc(s)));
316 i->src(s).get()->reg.data.offset += imm.reg.data.u32;
317 } else if (insn->op == OP_SUB && !isFloatType(insn->dType)) {
318 if (insn->src(0).getFile() != targ->nativeFile(FILE_ADDRESS) ||
319 !insn->src(1).getImmediate(imm) ||
320 !targ->insnCanLoadOffset(i, s, -imm.reg.data.s32))
321 continue;
322 i->setIndirect(s, 0, insn->getSrc(0));
323 i->setSrc(s, cloneShallow(func, i->getSrc(s)));
324 i->src(s).get()->reg.data.offset -= imm.reg.data.u32;
325 } else if (insn->op == OP_MOV) {
326 if (!insn->src(0).getImmediate(imm) ||
327 !targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
328 continue;
329 i->setIndirect(s, 0, NULL);
330 i->setSrc(s, cloneShallow(func, i->getSrc(s)));
331 i->src(s).get()->reg.data.offset += imm.reg.data.u32;
332 } else if (insn->op == OP_SHLADD) {
333 if (!insn->src(2).getImmediate(imm) ||
334 !targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
335 continue;
336 i->setIndirect(s, 0, bld.mkOp2v(
337 OP_SHL, TYPE_U32, bld.getSSA(), insn->getSrc(0), insn->getSrc(1)));
338 i->setSrc(s, cloneShallow(func, i->getSrc(s)));
339 i->src(s).get()->reg.data.offset += imm.reg.data.u32;
340 }
341 }
342 }
343 return true;
344 }
345
346 // =============================================================================
347
348 // Evaluate constant expressions.
349 class ConstantFolding : public Pass
350 {
351 public:
352 bool foldAll(Program *);
353
354 private:
355 virtual bool visit(BasicBlock *);
356
357 void expr(Instruction *, ImmediateValue&, ImmediateValue&);
358 void expr(Instruction *, ImmediateValue&, ImmediateValue&, ImmediateValue&);
359 void opnd(Instruction *, ImmediateValue&, int s);
360 void opnd3(Instruction *, ImmediateValue&);
361
362 void unary(Instruction *, const ImmediateValue&);
363
364 void tryCollapseChainedMULs(Instruction *, const int s, ImmediateValue&);
365
366 CmpInstruction *findOriginForTestWithZero(Value *);
367
368 unsigned int foldCount;
369
370 BuildUtil bld;
371 };
372
373 // TODO: remember generated immediates and only revisit these
374 bool
375 ConstantFolding::foldAll(Program *prog)
376 {
377 unsigned int iterCount = 0;
378 do {
379 foldCount = 0;
380 if (!run(prog))
381 return false;
382 } while (foldCount && ++iterCount < 2);
383 return true;
384 }
385
386 bool
387 ConstantFolding::visit(BasicBlock *bb)
388 {
389 Instruction *i, *next;
390
391 for (i = bb->getEntry(); i; i = next) {
392 next = i->next;
393 if (i->op == OP_MOV || i->op == OP_CALL)
394 continue;
395
396 ImmediateValue src0, src1, src2;
397
398 if (i->srcExists(2) &&
399 i->src(0).getImmediate(src0) &&
400 i->src(1).getImmediate(src1) &&
401 i->src(2).getImmediate(src2))
402 expr(i, src0, src1, src2);
403 else
404 if (i->srcExists(1) &&
405 i->src(0).getImmediate(src0) && i->src(1).getImmediate(src1))
406 expr(i, src0, src1);
407 else
408 if (i->srcExists(0) && i->src(0).getImmediate(src0))
409 opnd(i, src0, 0);
410 else
411 if (i->srcExists(1) && i->src(1).getImmediate(src1))
412 opnd(i, src1, 1);
413 if (i->srcExists(2) && i->src(2).getImmediate(src2))
414 opnd3(i, src2);
415 }
416 return true;
417 }
418
419 CmpInstruction *
420 ConstantFolding::findOriginForTestWithZero(Value *value)
421 {
422 if (!value)
423 return NULL;
424 Instruction *insn = value->getInsn();
425 if (!insn)
426 return NULL;
427
428 if (insn->asCmp() && insn->op != OP_SLCT)
429 return insn->asCmp();
430
431 /* Sometimes mov's will sneak in as a result of other folding. This gets
432 * cleaned up later.
433 */
434 if (insn->op == OP_MOV)
435 return findOriginForTestWithZero(insn->getSrc(0));
436
437 /* Deal with AND 1.0 here since nv50 can't fold into boolean float */
438 if (insn->op == OP_AND) {
439 int s = 0;
440 ImmediateValue imm;
441 if (!insn->src(s).getImmediate(imm)) {
442 s = 1;
443 if (!insn->src(s).getImmediate(imm))
444 return NULL;
445 }
446 if (imm.reg.data.f32 != 1.0f)
447 return NULL;
448 /* TODO: Come up with a way to handle the condition being inverted */
449 if (insn->src(!s).mod != Modifier(0))
450 return NULL;
451 return findOriginForTestWithZero(insn->getSrc(!s));
452 }
453
454 return NULL;
455 }
456
457 void
458 Modifier::applyTo(ImmediateValue& imm) const
459 {
460 if (!bits) // avoid failure if imm.reg.type is unhandled (e.g. b128)
461 return;
462 switch (imm.reg.type) {
463 case TYPE_F32:
464 if (bits & NV50_IR_MOD_ABS)
465 imm.reg.data.f32 = fabsf(imm.reg.data.f32);
466 if (bits & NV50_IR_MOD_NEG)
467 imm.reg.data.f32 = -imm.reg.data.f32;
468 if (bits & NV50_IR_MOD_SAT) {
469 if (imm.reg.data.f32 < 0.0f)
470 imm.reg.data.f32 = 0.0f;
471 else
472 if (imm.reg.data.f32 > 1.0f)
473 imm.reg.data.f32 = 1.0f;
474 }
475 assert(!(bits & NV50_IR_MOD_NOT));
476 break;
477
478 case TYPE_S8: // NOTE: will be extended
479 case TYPE_S16:
480 case TYPE_S32:
481 case TYPE_U8: // NOTE: treated as signed
482 case TYPE_U16:
483 case TYPE_U32:
484 if (bits & NV50_IR_MOD_ABS)
485 imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ?
486 imm.reg.data.s32 : -imm.reg.data.s32;
487 if (bits & NV50_IR_MOD_NEG)
488 imm.reg.data.s32 = -imm.reg.data.s32;
489 if (bits & NV50_IR_MOD_NOT)
490 imm.reg.data.s32 = ~imm.reg.data.s32;
491 break;
492
493 case TYPE_F64:
494 if (bits & NV50_IR_MOD_ABS)
495 imm.reg.data.f64 = fabs(imm.reg.data.f64);
496 if (bits & NV50_IR_MOD_NEG)
497 imm.reg.data.f64 = -imm.reg.data.f64;
498 if (bits & NV50_IR_MOD_SAT) {
499 if (imm.reg.data.f64 < 0.0)
500 imm.reg.data.f64 = 0.0;
501 else
502 if (imm.reg.data.f64 > 1.0)
503 imm.reg.data.f64 = 1.0;
504 }
505 assert(!(bits & NV50_IR_MOD_NOT));
506 break;
507
508 default:
509 assert(!"invalid/unhandled type");
510 imm.reg.data.u64 = 0;
511 break;
512 }
513 }
514
515 operation
516 Modifier::getOp() const
517 {
518 switch (bits) {
519 case NV50_IR_MOD_ABS: return OP_ABS;
520 case NV50_IR_MOD_NEG: return OP_NEG;
521 case NV50_IR_MOD_SAT: return OP_SAT;
522 case NV50_IR_MOD_NOT: return OP_NOT;
523 case 0:
524 return OP_MOV;
525 default:
526 return OP_CVT;
527 }
528 }
529
530 void
531 ConstantFolding::expr(Instruction *i,
532 ImmediateValue &imm0, ImmediateValue &imm1)
533 {
534 struct Storage *const a = &imm0.reg, *const b = &imm1.reg;
535 struct Storage res;
536 DataType type = i->dType;
537
538 memset(&res.data, 0, sizeof(res.data));
539
540 switch (i->op) {
541 case OP_MAD:
542 case OP_FMA:
543 case OP_MUL:
544 if (i->dnz && i->dType == TYPE_F32) {
545 if (!isfinite(a->data.f32))
546 a->data.f32 = 0.0f;
547 if (!isfinite(b->data.f32))
548 b->data.f32 = 0.0f;
549 }
550 switch (i->dType) {
551 case TYPE_F32:
552 res.data.f32 = a->data.f32 * b->data.f32 * exp2f(i->postFactor);
553 break;
554 case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break;
555 case TYPE_S32:
556 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
557 res.data.s32 = ((int64_t)a->data.s32 * b->data.s32) >> 32;
558 break;
559 }
560 /* fallthrough */
561 case TYPE_U32:
562 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
563 res.data.u32 = ((uint64_t)a->data.u32 * b->data.u32) >> 32;
564 break;
565 }
566 res.data.u32 = a->data.u32 * b->data.u32; break;
567 default:
568 return;
569 }
570 break;
571 case OP_DIV:
572 if (b->data.u32 == 0)
573 break;
574 switch (i->dType) {
575 case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break;
576 case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break;
577 case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break;
578 case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break;
579 default:
580 return;
581 }
582 break;
583 case OP_ADD:
584 switch (i->dType) {
585 case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break;
586 case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break;
587 case TYPE_S32:
588 case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break;
589 default:
590 return;
591 }
592 break;
593 case OP_SUB:
594 switch (i->dType) {
595 case TYPE_F32: res.data.f32 = a->data.f32 - b->data.f32; break;
596 case TYPE_F64: res.data.f64 = a->data.f64 - b->data.f64; break;
597 case TYPE_S32:
598 case TYPE_U32: res.data.u32 = a->data.u32 - b->data.u32; break;
599 default:
600 return;
601 }
602 break;
603 case OP_POW:
604 switch (i->dType) {
605 case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break;
606 case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break;
607 default:
608 return;
609 }
610 break;
611 case OP_MAX:
612 switch (i->dType) {
613 case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break;
614 case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break;
615 case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break;
616 case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break;
617 default:
618 return;
619 }
620 break;
621 case OP_MIN:
622 switch (i->dType) {
623 case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break;
624 case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break;
625 case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break;
626 case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break;
627 default:
628 return;
629 }
630 break;
631 case OP_AND:
632 res.data.u64 = a->data.u64 & b->data.u64;
633 break;
634 case OP_OR:
635 res.data.u64 = a->data.u64 | b->data.u64;
636 break;
637 case OP_XOR:
638 res.data.u64 = a->data.u64 ^ b->data.u64;
639 break;
640 case OP_SHL:
641 res.data.u32 = a->data.u32 << b->data.u32;
642 break;
643 case OP_SHR:
644 switch (i->dType) {
645 case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break;
646 case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break;
647 default:
648 return;
649 }
650 break;
651 case OP_SLCT:
652 if (a->data.u32 != b->data.u32)
653 return;
654 res.data.u32 = a->data.u32;
655 break;
656 case OP_EXTBF: {
657 int offset = b->data.u32 & 0xff;
658 int width = (b->data.u32 >> 8) & 0xff;
659 int rshift = offset;
660 int lshift = 0;
661 if (width == 0) {
662 res.data.u32 = 0;
663 break;
664 }
665 if (width + offset < 32) {
666 rshift = 32 - width;
667 lshift = 32 - width - offset;
668 }
669 if (i->subOp == NV50_IR_SUBOP_EXTBF_REV)
670 res.data.u32 = util_bitreverse(a->data.u32);
671 else
672 res.data.u32 = a->data.u32;
673 switch (i->dType) {
674 case TYPE_S32: res.data.s32 = (res.data.s32 << lshift) >> rshift; break;
675 case TYPE_U32: res.data.u32 = (res.data.u32 << lshift) >> rshift; break;
676 default:
677 return;
678 }
679 break;
680 }
681 case OP_POPCNT:
682 res.data.u32 = util_bitcount(a->data.u32 & b->data.u32);
683 break;
684 case OP_PFETCH:
685 // The two arguments to pfetch are logically added together. Normally
686 // the second argument will not be constant, but that can happen.
687 res.data.u32 = a->data.u32 + b->data.u32;
688 type = TYPE_U32;
689 break;
690 case OP_MERGE:
691 switch (i->dType) {
692 case TYPE_U64:
693 case TYPE_S64:
694 case TYPE_F64:
695 res.data.u64 = (((uint64_t)b->data.u32) << 32) | a->data.u32;
696 break;
697 default:
698 return;
699 }
700 break;
701 default:
702 return;
703 }
704 ++foldCount;
705
706 i->src(0).mod = Modifier(0);
707 i->src(1).mod = Modifier(0);
708 i->postFactor = 0;
709
710 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
711 i->setSrc(1, NULL);
712
713 i->getSrc(0)->reg.data = res.data;
714 i->getSrc(0)->reg.type = type;
715 i->getSrc(0)->reg.size = typeSizeof(type);
716
717 switch (i->op) {
718 case OP_MAD:
719 case OP_FMA: {
720 ImmediateValue src0, src1 = *i->getSrc(0)->asImm();
721
722 // Move the immediate into position 1, where we know it might be
723 // emittable. However it might not be anyways, as there may be other
724 // restrictions, so move it into a separate LValue.
725 bld.setPosition(i, false);
726 i->op = OP_ADD;
727 i->setSrc(1, bld.mkMov(bld.getSSA(type), i->getSrc(0), type)->getDef(0));
728 i->setSrc(0, i->getSrc(2));
729 i->src(0).mod = i->src(2).mod;
730 i->setSrc(2, NULL);
731
732 if (i->src(0).getImmediate(src0))
733 expr(i, src0, src1);
734 else
735 opnd(i, src1, 1);
736 break;
737 }
738 case OP_PFETCH:
739 // Leave PFETCH alone... we just folded its 2 args into 1.
740 break;
741 default:
742 i->op = i->saturate ? OP_SAT : OP_MOV;
743 if (i->saturate)
744 unary(i, *i->getSrc(0)->asImm());
745 break;
746 }
747 i->subOp = 0;
748 }
749
750 void
751 ConstantFolding::expr(Instruction *i,
752 ImmediateValue &imm0,
753 ImmediateValue &imm1,
754 ImmediateValue &imm2)
755 {
756 struct Storage *const a = &imm0.reg, *const b = &imm1.reg, *const c = &imm2.reg;
757 struct Storage res;
758
759 memset(&res.data, 0, sizeof(res.data));
760
761 switch (i->op) {
762 case OP_INSBF: {
763 int offset = b->data.u32 & 0xff;
764 int width = (b->data.u32 >> 8) & 0xff;
765 unsigned bitmask = ((1 << width) - 1) << offset;
766 res.data.u32 = ((a->data.u32 << offset) & bitmask) | (c->data.u32 & ~bitmask);
767 break;
768 }
769 case OP_MAD:
770 case OP_FMA: {
771 switch (i->dType) {
772 case TYPE_F32:
773 res.data.f32 = a->data.f32 * b->data.f32 * exp2f(i->postFactor) +
774 c->data.f32;
775 break;
776 case TYPE_F64:
777 res.data.f64 = a->data.f64 * b->data.f64 + c->data.f64;
778 break;
779 case TYPE_S32:
780 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
781 res.data.s32 = ((int64_t)a->data.s32 * b->data.s32 >> 32) + c->data.s32;
782 break;
783 }
784 /* fallthrough */
785 case TYPE_U32:
786 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
787 res.data.u32 = ((uint64_t)a->data.u32 * b->data.u32 >> 32) + c->data.u32;
788 break;
789 }
790 res.data.u32 = a->data.u32 * b->data.u32 + c->data.u32;
791 break;
792 default:
793 return;
794 }
795 break;
796 }
797 case OP_SHLADD:
798 res.data.u32 = (a->data.u32 << b->data.u32) + c->data.u32;
799 break;
800 default:
801 return;
802 }
803
804 ++foldCount;
805 i->src(0).mod = Modifier(0);
806 i->src(1).mod = Modifier(0);
807 i->src(2).mod = Modifier(0);
808
809 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
810 i->setSrc(1, NULL);
811 i->setSrc(2, NULL);
812
813 i->getSrc(0)->reg.data = res.data;
814 i->getSrc(0)->reg.type = i->dType;
815 i->getSrc(0)->reg.size = typeSizeof(i->dType);
816
817 i->op = OP_MOV;
818 }
819
820 void
821 ConstantFolding::unary(Instruction *i, const ImmediateValue &imm)
822 {
823 Storage res;
824
825 if (i->dType != TYPE_F32)
826 return;
827 switch (i->op) {
828 case OP_NEG: res.data.f32 = -imm.reg.data.f32; break;
829 case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break;
830 case OP_SAT: res.data.f32 = CLAMP(imm.reg.data.f32, 0.0f, 1.0f); break;
831 case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break;
832 case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break;
833 case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break;
834 case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break;
835 case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break;
836 case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break;
837 case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break;
838 case OP_PRESIN:
839 case OP_PREEX2:
840 // these should be handled in subsequent OP_SIN/COS/EX2
841 res.data.f32 = imm.reg.data.f32;
842 break;
843 default:
844 return;
845 }
846 i->op = OP_MOV;
847 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32));
848 i->src(0).mod = Modifier(0);
849 }
850
851 void
852 ConstantFolding::tryCollapseChainedMULs(Instruction *mul2,
853 const int s, ImmediateValue& imm2)
854 {
855 const int t = s ? 0 : 1;
856 Instruction *insn;
857 Instruction *mul1 = NULL; // mul1 before mul2
858 int e = 0;
859 float f = imm2.reg.data.f32 * exp2f(mul2->postFactor);
860 ImmediateValue imm1;
861
862 assert(mul2->op == OP_MUL && mul2->dType == TYPE_F32);
863
864 if (mul2->getSrc(t)->refCount() == 1) {
865 insn = mul2->getSrc(t)->getInsn();
866 if (!mul2->src(t).mod && insn->op == OP_MUL && insn->dType == TYPE_F32)
867 mul1 = insn;
868 if (mul1 && !mul1->saturate) {
869 int s1;
870
871 if (mul1->src(s1 = 0).getImmediate(imm1) ||
872 mul1->src(s1 = 1).getImmediate(imm1)) {
873 bld.setPosition(mul1, false);
874 // a = mul r, imm1
875 // d = mul a, imm2 -> d = mul r, (imm1 * imm2)
876 mul1->setSrc(s1, bld.loadImm(NULL, f * imm1.reg.data.f32));
877 mul1->src(s1).mod = Modifier(0);
878 mul2->def(0).replace(mul1->getDef(0), false);
879 mul1->saturate = mul2->saturate;
880 } else
881 if (prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
882 // c = mul a, b
883 // d = mul c, imm -> d = mul_x_imm a, b
884 mul1->postFactor = e;
885 mul2->def(0).replace(mul1->getDef(0), false);
886 if (f < 0)
887 mul1->src(0).mod *= Modifier(NV50_IR_MOD_NEG);
888 mul1->saturate = mul2->saturate;
889 }
890 return;
891 }
892 }
893 if (mul2->getDef(0)->refCount() == 1 && !mul2->saturate) {
894 // b = mul a, imm
895 // d = mul b, c -> d = mul_x_imm a, c
896 int s2, t2;
897 insn = (*mul2->getDef(0)->uses.begin())->getInsn();
898 if (!insn)
899 return;
900 mul1 = mul2;
901 mul2 = NULL;
902 s2 = insn->getSrc(0) == mul1->getDef(0) ? 0 : 1;
903 t2 = s2 ? 0 : 1;
904 if (insn->op == OP_MUL && insn->dType == TYPE_F32)
905 if (!insn->src(s2).mod && !insn->src(t2).getImmediate(imm1))
906 mul2 = insn;
907 if (mul2 && prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
908 mul2->postFactor = e;
909 mul2->setSrc(s2, mul1->src(t));
910 if (f < 0)
911 mul2->src(s2).mod *= Modifier(NV50_IR_MOD_NEG);
912 }
913 }
914 }
915
916 void
917 ConstantFolding::opnd3(Instruction *i, ImmediateValue &imm2)
918 {
919 switch (i->op) {
920 case OP_MAD:
921 case OP_FMA:
922 if (imm2.isInteger(0)) {
923 i->op = OP_MUL;
924 i->setSrc(2, NULL);
925 foldCount++;
926 return;
927 }
928 break;
929 case OP_SHLADD:
930 if (imm2.isInteger(0)) {
931 i->op = OP_SHL;
932 i->setSrc(2, NULL);
933 foldCount++;
934 return;
935 }
936 break;
937 default:
938 return;
939 }
940 }
941
942 void
943 ConstantFolding::opnd(Instruction *i, ImmediateValue &imm0, int s)
944 {
945 const Target *target = prog->getTarget();
946 const int t = !s;
947 const operation op = i->op;
948 Instruction *newi = i;
949
950 switch (i->op) {
951 case OP_SPLIT: {
952 bld.setPosition(i, false);
953
954 uint8_t size = i->getDef(0)->reg.size;
955 uint8_t bitsize = size * 8;
956 uint32_t mask = (1ULL << bitsize) - 1;
957 assert(bitsize <= 32);
958
959 uint64_t val = imm0.reg.data.u64;
960 for (int8_t d = 0; i->defExists(d); ++d) {
961 Value *def = i->getDef(d);
962 assert(def->reg.size == size);
963
964 newi = bld.mkMov(def, bld.mkImm((uint32_t)(val & mask)), TYPE_U32);
965 val >>= bitsize;
966 }
967 delete_Instruction(prog, i);
968 break;
969 }
970 case OP_MUL:
971 if (i->dType == TYPE_F32)
972 tryCollapseChainedMULs(i, s, imm0);
973
974 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
975 assert(!isFloatType(i->sType));
976 if (imm0.isInteger(1) && i->dType == TYPE_S32) {
977 bld.setPosition(i, false);
978 // Need to set to the sign value, which is a compare.
979 newi = bld.mkCmp(OP_SET, CC_LT, TYPE_S32, i->getDef(0),
980 TYPE_S32, i->getSrc(t), bld.mkImm(0));
981 delete_Instruction(prog, i);
982 } else if (imm0.isInteger(0) || imm0.isInteger(1)) {
983 // The high bits can't be set in this case (either mul by 0 or
984 // unsigned by 1)
985 i->op = OP_MOV;
986 i->subOp = 0;
987 i->setSrc(0, new_ImmediateValue(prog, 0u));
988 i->src(0).mod = Modifier(0);
989 i->setSrc(1, NULL);
990 } else if (!imm0.isNegative() && imm0.isPow2()) {
991 // Translate into a shift
992 imm0.applyLog2();
993 i->op = OP_SHR;
994 i->subOp = 0;
995 imm0.reg.data.u32 = 32 - imm0.reg.data.u32;
996 i->setSrc(0, i->getSrc(t));
997 i->src(0).mod = i->src(t).mod;
998 i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
999 i->src(1).mod = 0;
1000 }
1001 } else
1002 if (imm0.isInteger(0)) {
1003 i->op = OP_MOV;
1004 i->setSrc(0, new_ImmediateValue(prog, 0u));
1005 i->src(0).mod = Modifier(0);
1006 i->postFactor = 0;
1007 i->setSrc(1, NULL);
1008 } else
1009 if (!i->postFactor && (imm0.isInteger(1) || imm0.isInteger(-1))) {
1010 if (imm0.isNegative())
1011 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1012 i->op = i->src(t).mod.getOp();
1013 if (s == 0) {
1014 i->setSrc(0, i->getSrc(1));
1015 i->src(0).mod = i->src(1).mod;
1016 i->src(1).mod = 0;
1017 }
1018 if (i->op != OP_CVT)
1019 i->src(0).mod = 0;
1020 i->setSrc(1, NULL);
1021 } else
1022 if (!i->postFactor && (imm0.isInteger(2) || imm0.isInteger(-2))) {
1023 if (imm0.isNegative())
1024 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1025 i->op = OP_ADD;
1026 i->setSrc(s, i->getSrc(t));
1027 i->src(s).mod = i->src(t).mod;
1028 } else
1029 if (!isFloatType(i->sType) && !imm0.isNegative() && imm0.isPow2()) {
1030 i->op = OP_SHL;
1031 imm0.applyLog2();
1032 i->setSrc(0, i->getSrc(t));
1033 i->src(0).mod = i->src(t).mod;
1034 i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
1035 i->src(1).mod = 0;
1036 } else
1037 if (i->postFactor && i->sType == TYPE_F32) {
1038 /* Can't emit a postfactor with an immediate, have to fold it in */
1039 i->setSrc(s, new_ImmediateValue(
1040 prog, imm0.reg.data.f32 * exp2f(i->postFactor)));
1041 i->postFactor = 0;
1042 }
1043 break;
1044 case OP_FMA:
1045 case OP_MAD:
1046 if (imm0.isInteger(0)) {
1047 i->setSrc(0, i->getSrc(2));
1048 i->src(0).mod = i->src(2).mod;
1049 i->setSrc(1, NULL);
1050 i->setSrc(2, NULL);
1051 i->op = i->src(0).mod.getOp();
1052 if (i->op != OP_CVT)
1053 i->src(0).mod = 0;
1054 } else
1055 if (i->subOp != NV50_IR_SUBOP_MUL_HIGH &&
1056 (imm0.isInteger(1) || imm0.isInteger(-1))) {
1057 if (imm0.isNegative())
1058 i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1059 if (s == 0) {
1060 i->setSrc(0, i->getSrc(1));
1061 i->src(0).mod = i->src(1).mod;
1062 }
1063 i->setSrc(1, i->getSrc(2));
1064 i->src(1).mod = i->src(2).mod;
1065 i->setSrc(2, NULL);
1066 i->op = OP_ADD;
1067 } else
1068 if (s == 1 && !imm0.isNegative() && imm0.isPow2() &&
1069 !isFloatType(i->dType) &&
1070 target->isOpSupported(OP_SHLADD, i->dType) &&
1071 !i->subOp) {
1072 i->op = OP_SHLADD;
1073 imm0.applyLog2();
1074 i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
1075 }
1076 break;
1077 case OP_SUB:
1078 if (imm0.isInteger(0) && s == 0 && typeSizeof(i->dType) == 8 &&
1079 !isFloatType(i->dType))
1080 break;
1081 /* fallthrough */
1082 case OP_ADD:
1083 if (i->usesFlags())
1084 break;
1085 if (imm0.isInteger(0)) {
1086 if (s == 0) {
1087 i->setSrc(0, i->getSrc(1));
1088 i->src(0).mod = i->src(1).mod;
1089 if (i->op == OP_SUB)
1090 i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
1091 }
1092 i->setSrc(1, NULL);
1093 i->op = i->src(0).mod.getOp();
1094 if (i->op != OP_CVT)
1095 i->src(0).mod = Modifier(0);
1096 }
1097 break;
1098
1099 case OP_DIV:
1100 if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32))
1101 break;
1102 bld.setPosition(i, false);
1103 if (imm0.reg.data.u32 == 0) {
1104 break;
1105 } else
1106 if (imm0.reg.data.u32 == 1) {
1107 i->op = OP_MOV;
1108 i->setSrc(1, NULL);
1109 } else
1110 if (i->dType == TYPE_U32 && imm0.isPow2()) {
1111 i->op = OP_SHR;
1112 i->setSrc(1, bld.mkImm(util_logbase2(imm0.reg.data.u32)));
1113 } else
1114 if (i->dType == TYPE_U32) {
1115 Instruction *mul;
1116 Value *tA, *tB;
1117 const uint32_t d = imm0.reg.data.u32;
1118 uint32_t m;
1119 int r, s;
1120 uint32_t l = util_logbase2(d);
1121 if (((uint32_t)1 << l) < d)
1122 ++l;
1123 m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1;
1124 r = l ? 1 : 0;
1125 s = l ? (l - 1) : 0;
1126
1127 tA = bld.getSSA();
1128 tB = bld.getSSA();
1129 mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0),
1130 bld.loadImm(NULL, m));
1131 mul->subOp = NV50_IR_SUBOP_MUL_HIGH;
1132 bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA);
1133 tA = bld.getSSA();
1134 if (r)
1135 bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r));
1136 else
1137 tA = tB;
1138 tB = s ? bld.getSSA() : i->getDef(0);
1139 newi = bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA);
1140 if (s)
1141 bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s));
1142
1143 delete_Instruction(prog, i);
1144 } else
1145 if (imm0.reg.data.s32 == -1) {
1146 i->op = OP_NEG;
1147 i->setSrc(1, NULL);
1148 } else {
1149 LValue *tA, *tB;
1150 LValue *tD;
1151 const int32_t d = imm0.reg.data.s32;
1152 int32_t m;
1153 int32_t l = util_logbase2(static_cast<unsigned>(abs(d)));
1154 if ((1 << l) < abs(d))
1155 ++l;
1156 if (!l)
1157 l = 1;
1158 m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32);
1159
1160 tA = bld.getSSA();
1161 tB = bld.getSSA();
1162 bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m),
1163 i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH;
1164 if (l > 1)
1165 bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1));
1166 else
1167 tB = tA;
1168 tA = bld.getSSA();
1169 bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, TYPE_S32, i->getSrc(0), bld.mkImm(0));
1170 tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue();
1171 newi = bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA);
1172 if (d < 0)
1173 bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB);
1174
1175 delete_Instruction(prog, i);
1176 }
1177 break;
1178
1179 case OP_MOD:
1180 if (s == 1 && imm0.isPow2()) {
1181 bld.setPosition(i, false);
1182 if (i->sType == TYPE_U32) {
1183 i->op = OP_AND;
1184 i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 - 1));
1185 } else if (i->sType == TYPE_S32) {
1186 // Do it on the absolute value of the input, and then restore the
1187 // sign. The only odd case is MIN_INT, but that should work out
1188 // as well, since MIN_INT mod any power of 2 is 0.
1189 //
1190 // Technically we don't have to do any of this since MOD is
1191 // undefined with negative arguments in GLSL, but this seems like
1192 // the nice thing to do.
1193 Value *abs = bld.mkOp1v(OP_ABS, TYPE_S32, bld.getSSA(), i->getSrc(0));
1194 Value *neg, *v1, *v2;
1195 bld.mkCmp(OP_SET, CC_LT, TYPE_S32,
1196 (neg = bld.getSSA(1, prog->getTarget()->nativeFile(FILE_PREDICATE))),
1197 TYPE_S32, i->getSrc(0), bld.loadImm(NULL, 0));
1198 Value *mod = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), abs,
1199 bld.loadImm(NULL, imm0.reg.data.u32 - 1));
1200 bld.mkOp1(OP_NEG, TYPE_S32, (v1 = bld.getSSA()), mod)
1201 ->setPredicate(CC_P, neg);
1202 bld.mkOp1(OP_MOV, TYPE_S32, (v2 = bld.getSSA()), mod)
1203 ->setPredicate(CC_NOT_P, neg);
1204 newi = bld.mkOp2(OP_UNION, TYPE_S32, i->getDef(0), v1, v2);
1205
1206 delete_Instruction(prog, i);
1207 }
1208 } else if (s == 1) {
1209 // In this case, we still want the optimized lowering that we get
1210 // from having division by an immediate.
1211 //
1212 // a % b == a - (a/b) * b
1213 bld.setPosition(i, false);
1214 Value *div = bld.mkOp2v(OP_DIV, i->sType, bld.getSSA(),
1215 i->getSrc(0), i->getSrc(1));
1216 newi = bld.mkOp2(OP_ADD, i->sType, i->getDef(0), i->getSrc(0),
1217 bld.mkOp2v(OP_MUL, i->sType, bld.getSSA(), div, i->getSrc(1)));
1218 // TODO: Check that target supports this. In this case, we know that
1219 // all backends do.
1220 newi->src(1).mod = Modifier(NV50_IR_MOD_NEG);
1221
1222 delete_Instruction(prog, i);
1223 }
1224 break;
1225
1226 case OP_SET: // TODO: SET_AND,OR,XOR
1227 {
1228 /* This optimizes the case where the output of a set is being compared
1229 * to zero. Since the set can only produce 0/-1 (int) or 0/1 (float), we
1230 * can be a lot cleverer in our comparison.
1231 */
1232 CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t));
1233 CondCode cc, ccZ;
1234 if (imm0.reg.data.u32 != 0 || !si)
1235 return;
1236 cc = si->setCond;
1237 ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U);
1238 // We do everything assuming var (cmp) 0, reverse the condition if 0 is
1239 // first.
1240 if (s == 0)
1241 ccZ = reverseCondCode(ccZ);
1242 // If there is a negative modifier, we need to undo that, by flipping
1243 // the comparison to zero.
1244 if (i->src(t).mod.neg())
1245 ccZ = reverseCondCode(ccZ);
1246 // If this is a signed comparison, we expect the input to be a regular
1247 // boolean, i.e. 0/-1. However the rest of the logic assumes that true
1248 // is positive, so just flip the sign.
1249 if (i->sType == TYPE_S32) {
1250 assert(!isFloatType(si->dType));
1251 ccZ = reverseCondCode(ccZ);
1252 }
1253 switch (ccZ) {
1254 case CC_LT: cc = CC_FL; break; // bool < 0 -- this is never true
1255 case CC_GE: cc = CC_TR; break; // bool >= 0 -- this is always true
1256 case CC_EQ: cc = inverseCondCode(cc); break; // bool == 0 -- !bool
1257 case CC_LE: cc = inverseCondCode(cc); break; // bool <= 0 -- !bool
1258 case CC_GT: break; // bool > 0 -- bool
1259 case CC_NE: break; // bool != 0 -- bool
1260 default:
1261 return;
1262 }
1263
1264 // Update the condition of this SET to be identical to the origin set,
1265 // but with the updated condition code. The original SET should get
1266 // DCE'd, ideally.
1267 i->op = si->op;
1268 i->asCmp()->setCond = cc;
1269 i->setSrc(0, si->src(0));
1270 i->setSrc(1, si->src(1));
1271 if (si->srcExists(2))
1272 i->setSrc(2, si->src(2));
1273 i->sType = si->sType;
1274 }
1275 break;
1276
1277 case OP_AND:
1278 {
1279 Instruction *src = i->getSrc(t)->getInsn();
1280 ImmediateValue imm1;
1281 if (imm0.reg.data.u32 == 0) {
1282 i->op = OP_MOV;
1283 i->setSrc(0, new_ImmediateValue(prog, 0u));
1284 i->src(0).mod = Modifier(0);
1285 i->setSrc(1, NULL);
1286 } else if (imm0.reg.data.u32 == ~0U) {
1287 i->op = i->src(t).mod.getOp();
1288 if (t) {
1289 i->setSrc(0, i->getSrc(t));
1290 i->src(0).mod = i->src(t).mod;
1291 }
1292 i->setSrc(1, NULL);
1293 } else if (src->asCmp()) {
1294 CmpInstruction *cmp = src->asCmp();
1295 if (!cmp || cmp->op == OP_SLCT || cmp->getDef(0)->refCount() > 1)
1296 return;
1297 if (!prog->getTarget()->isOpSupported(cmp->op, TYPE_F32))
1298 return;
1299 if (imm0.reg.data.f32 != 1.0)
1300 return;
1301 if (cmp->dType != TYPE_U32)
1302 return;
1303
1304 cmp->dType = TYPE_F32;
1305 if (i->src(t).mod != Modifier(0)) {
1306 assert(i->src(t).mod == Modifier(NV50_IR_MOD_NOT));
1307 i->src(t).mod = Modifier(0);
1308 cmp->setCond = inverseCondCode(cmp->setCond);
1309 }
1310 i->op = OP_MOV;
1311 i->setSrc(s, NULL);
1312 if (t) {
1313 i->setSrc(0, i->getSrc(t));
1314 i->setSrc(t, NULL);
1315 }
1316 } else if (prog->getTarget()->isOpSupported(OP_EXTBF, TYPE_U32) &&
1317 src->op == OP_SHR &&
1318 src->src(1).getImmediate(imm1) &&
1319 i->src(t).mod == Modifier(0) &&
1320 util_is_power_of_two_or_zero(imm0.reg.data.u32 + 1)) {
1321 // low byte = offset, high byte = width
1322 uint32_t ext = (util_last_bit(imm0.reg.data.u32) << 8) | imm1.reg.data.u32;
1323 i->op = OP_EXTBF;
1324 i->setSrc(0, src->getSrc(0));
1325 i->setSrc(1, new_ImmediateValue(prog, ext));
1326 } else if (src->op == OP_SHL &&
1327 src->src(1).getImmediate(imm1) &&
1328 i->src(t).mod == Modifier(0) &&
1329 util_is_power_of_two_or_zero(~imm0.reg.data.u32 + 1) &&
1330 util_last_bit(~imm0.reg.data.u32) <= imm1.reg.data.u32) {
1331 i->op = OP_MOV;
1332 i->setSrc(s, NULL);
1333 if (t) {
1334 i->setSrc(0, i->getSrc(t));
1335 i->setSrc(t, NULL);
1336 }
1337 }
1338 }
1339 break;
1340
1341 case OP_SHL:
1342 {
1343 if (s != 1 || i->src(0).mod != Modifier(0))
1344 break;
1345 // try to concatenate shifts
1346 Instruction *si = i->getSrc(0)->getInsn();
1347 if (!si)
1348 break;
1349 ImmediateValue imm1;
1350 switch (si->op) {
1351 case OP_SHL:
1352 if (si->src(1).getImmediate(imm1)) {
1353 bld.setPosition(i, false);
1354 i->setSrc(0, si->getSrc(0));
1355 i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 + imm1.reg.data.u32));
1356 }
1357 break;
1358 case OP_SHR:
1359 if (si->src(1).getImmediate(imm1) && imm0.reg.data.u32 == imm1.reg.data.u32) {
1360 bld.setPosition(i, false);
1361 i->op = OP_AND;
1362 i->setSrc(0, si->getSrc(0));
1363 i->setSrc(1, bld.loadImm(NULL, ~((1 << imm0.reg.data.u32) - 1)));
1364 }
1365 break;
1366 case OP_MUL:
1367 int muls;
1368 if (isFloatType(si->dType))
1369 return;
1370 if (si->src(1).getImmediate(imm1))
1371 muls = 1;
1372 else if (si->src(0).getImmediate(imm1))
1373 muls = 0;
1374 else
1375 return;
1376
1377 bld.setPosition(i, false);
1378 i->op = OP_MUL;
1379 i->setSrc(0, si->getSrc(!muls));
1380 i->setSrc(1, bld.loadImm(NULL, imm1.reg.data.u32 << imm0.reg.data.u32));
1381 break;
1382 case OP_SUB:
1383 case OP_ADD:
1384 int adds;
1385 if (isFloatType(si->dType))
1386 return;
1387 if (si->op != OP_SUB && si->src(0).getImmediate(imm1))
1388 adds = 0;
1389 else if (si->src(1).getImmediate(imm1))
1390 adds = 1;
1391 else
1392 return;
1393 if (si->src(!adds).mod != Modifier(0))
1394 return;
1395 // SHL(ADD(x, y), z) = ADD(SHL(x, z), SHL(y, z))
1396
1397 // This is more operations, but if one of x, y is an immediate, then
1398 // we can get a situation where (a) we can use ISCADD, or (b)
1399 // propagate the add bit into an indirect load.
1400 bld.setPosition(i, false);
1401 i->op = si->op;
1402 i->setSrc(adds, bld.loadImm(NULL, imm1.reg.data.u32 << imm0.reg.data.u32));
1403 i->setSrc(!adds, bld.mkOp2v(OP_SHL, i->dType,
1404 bld.getSSA(i->def(0).getSize(), i->def(0).getFile()),
1405 si->getSrc(!adds),
1406 bld.mkImm(imm0.reg.data.u32)));
1407 break;
1408 default:
1409 return;
1410 }
1411 }
1412 break;
1413
1414 case OP_ABS:
1415 case OP_NEG:
1416 case OP_SAT:
1417 case OP_LG2:
1418 case OP_RCP:
1419 case OP_SQRT:
1420 case OP_RSQ:
1421 case OP_PRESIN:
1422 case OP_SIN:
1423 case OP_COS:
1424 case OP_PREEX2:
1425 case OP_EX2:
1426 unary(i, imm0);
1427 break;
1428 case OP_BFIND: {
1429 int32_t res;
1430 switch (i->dType) {
1431 case TYPE_S32: res = util_last_bit_signed(imm0.reg.data.s32) - 1; break;
1432 case TYPE_U32: res = util_last_bit(imm0.reg.data.u32) - 1; break;
1433 default:
1434 return;
1435 }
1436 if (i->subOp == NV50_IR_SUBOP_BFIND_SAMT && res >= 0)
1437 res = 31 - res;
1438 bld.setPosition(i, false); /* make sure bld is init'ed */
1439 i->setSrc(0, bld.mkImm(res));
1440 i->setSrc(1, NULL);
1441 i->op = OP_MOV;
1442 i->subOp = 0;
1443 break;
1444 }
1445 case OP_POPCNT: {
1446 // Only deal with 1-arg POPCNT here
1447 if (i->srcExists(1))
1448 break;
1449 uint32_t res = util_bitcount(imm0.reg.data.u32);
1450 i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res));
1451 i->setSrc(1, NULL);
1452 i->op = OP_MOV;
1453 break;
1454 }
1455 case OP_CVT: {
1456 Storage res;
1457
1458 // TODO: handle 64-bit values properly
1459 if (typeSizeof(i->dType) == 8 || typeSizeof(i->sType) == 8)
1460 return;
1461
1462 // TODO: handle single byte/word extractions
1463 if (i->subOp)
1464 return;
1465
1466 bld.setPosition(i, true); /* make sure bld is init'ed */
1467
1468 #define CASE(type, dst, fmin, fmax, imin, imax, umin, umax) \
1469 case type: \
1470 switch (i->sType) { \
1471 case TYPE_F64: \
1472 res.data.dst = util_iround(i->saturate ? \
1473 CLAMP(imm0.reg.data.f64, fmin, fmax) : \
1474 imm0.reg.data.f64); \
1475 break; \
1476 case TYPE_F32: \
1477 res.data.dst = util_iround(i->saturate ? \
1478 CLAMP(imm0.reg.data.f32, fmin, fmax) : \
1479 imm0.reg.data.f32); \
1480 break; \
1481 case TYPE_S32: \
1482 res.data.dst = i->saturate ? \
1483 CLAMP(imm0.reg.data.s32, imin, imax) : \
1484 imm0.reg.data.s32; \
1485 break; \
1486 case TYPE_U32: \
1487 res.data.dst = i->saturate ? \
1488 CLAMP(imm0.reg.data.u32, umin, umax) : \
1489 imm0.reg.data.u32; \
1490 break; \
1491 case TYPE_S16: \
1492 res.data.dst = i->saturate ? \
1493 CLAMP(imm0.reg.data.s16, imin, imax) : \
1494 imm0.reg.data.s16; \
1495 break; \
1496 case TYPE_U16: \
1497 res.data.dst = i->saturate ? \
1498 CLAMP(imm0.reg.data.u16, umin, umax) : \
1499 imm0.reg.data.u16; \
1500 break; \
1501 default: return; \
1502 } \
1503 i->setSrc(0, bld.mkImm(res.data.dst)); \
1504 break
1505
1506 switch(i->dType) {
1507 CASE(TYPE_U16, u16, 0, UINT16_MAX, 0, UINT16_MAX, 0, UINT16_MAX);
1508 CASE(TYPE_S16, s16, INT16_MIN, INT16_MAX, INT16_MIN, INT16_MAX, 0, INT16_MAX);
1509 CASE(TYPE_U32, u32, 0, UINT32_MAX, 0, INT32_MAX, 0, UINT32_MAX);
1510 CASE(TYPE_S32, s32, INT32_MIN, INT32_MAX, INT32_MIN, INT32_MAX, 0, INT32_MAX);
1511 case TYPE_F32:
1512 switch (i->sType) {
1513 case TYPE_F64:
1514 res.data.f32 = i->saturate ?
1515 CLAMP(imm0.reg.data.f64, 0.0f, 1.0f) :
1516 imm0.reg.data.f64;
1517 break;
1518 case TYPE_F32:
1519 res.data.f32 = i->saturate ?
1520 CLAMP(imm0.reg.data.f32, 0.0f, 1.0f) :
1521 imm0.reg.data.f32;
1522 break;
1523 case TYPE_U16: res.data.f32 = (float) imm0.reg.data.u16; break;
1524 case TYPE_U32: res.data.f32 = (float) imm0.reg.data.u32; break;
1525 case TYPE_S16: res.data.f32 = (float) imm0.reg.data.s16; break;
1526 case TYPE_S32: res.data.f32 = (float) imm0.reg.data.s32; break;
1527 default:
1528 return;
1529 }
1530 i->setSrc(0, bld.mkImm(res.data.f32));
1531 break;
1532 case TYPE_F64:
1533 switch (i->sType) {
1534 case TYPE_F64:
1535 res.data.f64 = i->saturate ?
1536 CLAMP(imm0.reg.data.f64, 0.0f, 1.0f) :
1537 imm0.reg.data.f64;
1538 break;
1539 case TYPE_F32:
1540 res.data.f64 = i->saturate ?
1541 CLAMP(imm0.reg.data.f32, 0.0f, 1.0f) :
1542 imm0.reg.data.f32;
1543 break;
1544 case TYPE_U16: res.data.f64 = (double) imm0.reg.data.u16; break;
1545 case TYPE_U32: res.data.f64 = (double) imm0.reg.data.u32; break;
1546 case TYPE_S16: res.data.f64 = (double) imm0.reg.data.s16; break;
1547 case TYPE_S32: res.data.f64 = (double) imm0.reg.data.s32; break;
1548 default:
1549 return;
1550 }
1551 i->setSrc(0, bld.mkImm(res.data.f64));
1552 break;
1553 default:
1554 return;
1555 }
1556 #undef CASE
1557
1558 i->setType(i->dType); /* Remove i->sType, which we don't need anymore */
1559 i->op = OP_MOV;
1560 i->saturate = 0;
1561 i->src(0).mod = Modifier(0); /* Clear the already applied modifier */
1562 break;
1563 }
1564 default:
1565 return;
1566 }
1567
1568 // This can get left behind some of the optimizations which simplify
1569 // saturatable values.
1570 if (newi->op == OP_MOV && newi->saturate) {
1571 ImmediateValue tmp;
1572 newi->saturate = 0;
1573 newi->op = OP_SAT;
1574 if (newi->src(0).getImmediate(tmp))
1575 unary(newi, tmp);
1576 }
1577
1578 if (newi->op != op)
1579 foldCount++;
1580 }
1581
1582 // =============================================================================
1583
1584 // Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed.
1585 class ModifierFolding : public Pass
1586 {
1587 private:
1588 virtual bool visit(BasicBlock *);
1589 };
1590
1591 bool
1592 ModifierFolding::visit(BasicBlock *bb)
1593 {
1594 const Target *target = prog->getTarget();
1595
1596 Instruction *i, *next, *mi;
1597 Modifier mod;
1598
1599 for (i = bb->getEntry(); i; i = next) {
1600 next = i->next;
1601
1602 if (0 && i->op == OP_SUB) {
1603 // turn "sub" into "add neg" (do we really want this ?)
1604 i->op = OP_ADD;
1605 i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
1606 }
1607
1608 for (int s = 0; s < 3 && i->srcExists(s); ++s) {
1609 mi = i->getSrc(s)->getInsn();
1610 if (!mi ||
1611 mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8)
1612 continue;
1613 if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) {
1614 if ((i->op != OP_ADD &&
1615 i->op != OP_MUL) ||
1616 (mi->op != OP_ABS &&
1617 mi->op != OP_NEG))
1618 continue;
1619 } else
1620 if (i->sType != mi->dType) {
1621 continue;
1622 }
1623 if ((mod = Modifier(mi->op)) == Modifier(0))
1624 continue;
1625 mod *= mi->src(0).mod;
1626
1627 if ((i->op == OP_ABS) || i->src(s).mod.abs()) {
1628 // abs neg [abs] = abs
1629 mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS));
1630 } else
1631 if ((i->op == OP_NEG) && mod.neg()) {
1632 assert(s == 0);
1633 // neg as both opcode and modifier on same insn is prohibited
1634 // neg neg abs = abs, neg neg = identity
1635 mod = mod & Modifier(~NV50_IR_MOD_NEG);
1636 i->op = mod.getOp();
1637 mod = mod & Modifier(~NV50_IR_MOD_ABS);
1638 if (mod == Modifier(0))
1639 i->op = OP_MOV;
1640 }
1641
1642 if (target->isModSupported(i, s, mod)) {
1643 i->setSrc(s, mi->getSrc(0));
1644 i->src(s).mod *= mod;
1645 }
1646 }
1647
1648 if (i->op == OP_SAT) {
1649 mi = i->getSrc(0)->getInsn();
1650 if (mi &&
1651 mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) {
1652 mi->saturate = 1;
1653 mi->setDef(0, i->getDef(0));
1654 delete_Instruction(prog, i);
1655 }
1656 }
1657 }
1658
1659 return true;
1660 }
1661
1662 // =============================================================================
1663
1664 // MUL + ADD -> MAD/FMA
1665 // MIN/MAX(a, a) -> a, etc.
1666 // SLCT(a, b, const) -> cc(const) ? a : b
1667 // RCP(RCP(a)) -> a
1668 // MUL(MUL(a, b), const) -> MUL_Xconst(a, b)
1669 // EXTBF(RDSV(COMBINED_TID)) -> RDSV(TID)
1670 class AlgebraicOpt : public Pass
1671 {
1672 private:
1673 virtual bool visit(BasicBlock *);
1674
1675 void handleABS(Instruction *);
1676 bool handleADD(Instruction *);
1677 bool tryADDToMADOrSAD(Instruction *, operation toOp);
1678 void handleMINMAX(Instruction *);
1679 void handleRCP(Instruction *);
1680 void handleSLCT(Instruction *);
1681 void handleLOGOP(Instruction *);
1682 void handleCVT_NEG(Instruction *);
1683 void handleCVT_CVT(Instruction *);
1684 void handleCVT_EXTBF(Instruction *);
1685 void handleSUCLAMP(Instruction *);
1686 void handleNEG(Instruction *);
1687 void handleEXTBF_RDSV(Instruction *);
1688
1689 BuildUtil bld;
1690 };
1691
1692 void
1693 AlgebraicOpt::handleABS(Instruction *abs)
1694 {
1695 Instruction *sub = abs->getSrc(0)->getInsn();
1696 DataType ty;
1697 if (!sub ||
1698 !prog->getTarget()->isOpSupported(OP_SAD, abs->dType))
1699 return;
1700 // expect not to have mods yet, if we do, bail
1701 if (sub->src(0).mod || sub->src(1).mod)
1702 return;
1703 // hidden conversion ?
1704 ty = intTypeToSigned(sub->dType);
1705 if (abs->dType != abs->sType || ty != abs->sType)
1706 return;
1707
1708 if ((sub->op != OP_ADD && sub->op != OP_SUB) ||
1709 sub->src(0).getFile() != FILE_GPR || sub->src(0).mod ||
1710 sub->src(1).getFile() != FILE_GPR || sub->src(1).mod)
1711 return;
1712
1713 Value *src0 = sub->getSrc(0);
1714 Value *src1 = sub->getSrc(1);
1715
1716 if (sub->op == OP_ADD) {
1717 Instruction *neg = sub->getSrc(1)->getInsn();
1718 if (neg && neg->op != OP_NEG) {
1719 neg = sub->getSrc(0)->getInsn();
1720 src0 = sub->getSrc(1);
1721 }
1722 if (!neg || neg->op != OP_NEG ||
1723 neg->dType != neg->sType || neg->sType != ty)
1724 return;
1725 src1 = neg->getSrc(0);
1726 }
1727
1728 // found ABS(SUB))
1729 abs->moveSources(1, 2); // move sources >=1 up by 2
1730 abs->op = OP_SAD;
1731 abs->setType(sub->dType);
1732 abs->setSrc(0, src0);
1733 abs->setSrc(1, src1);
1734 bld.setPosition(abs, false);
1735 abs->setSrc(2, bld.loadImm(bld.getSSA(typeSizeof(ty)), 0));
1736 }
1737
1738 bool
1739 AlgebraicOpt::handleADD(Instruction *add)
1740 {
1741 Value *src0 = add->getSrc(0);
1742 Value *src1 = add->getSrc(1);
1743
1744 if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
1745 return false;
1746
1747 bool changed = false;
1748 // we can't optimize to MAD if the add is precise
1749 if (!add->precise && prog->getTarget()->isOpSupported(OP_MAD, add->dType))
1750 changed = tryADDToMADOrSAD(add, OP_MAD);
1751 if (!changed && prog->getTarget()->isOpSupported(OP_SAD, add->dType))
1752 changed = tryADDToMADOrSAD(add, OP_SAD);
1753 return changed;
1754 }
1755
1756 // ADD(SAD(a,b,0), c) -> SAD(a,b,c)
1757 // ADD(MUL(a,b), c) -> MAD(a,b,c)
1758 bool
1759 AlgebraicOpt::tryADDToMADOrSAD(Instruction *add, operation toOp)
1760 {
1761 Value *src0 = add->getSrc(0);
1762 Value *src1 = add->getSrc(1);
1763 Value *src;
1764 int s;
1765 const operation srcOp = toOp == OP_SAD ? OP_SAD : OP_MUL;
1766 const Modifier modBad = Modifier(~((toOp == OP_MAD) ? NV50_IR_MOD_NEG : 0));
1767 Modifier mod[4];
1768
1769 if (src0->refCount() == 1 &&
1770 src0->getUniqueInsn() && src0->getUniqueInsn()->op == srcOp)
1771 s = 0;
1772 else
1773 if (src1->refCount() == 1 &&
1774 src1->getUniqueInsn() && src1->getUniqueInsn()->op == srcOp)
1775 s = 1;
1776 else
1777 return false;
1778
1779 src = add->getSrc(s);
1780
1781 if (src->getUniqueInsn() && src->getUniqueInsn()->bb != add->bb)
1782 return false;
1783
1784 if (src->getInsn()->saturate || src->getInsn()->postFactor ||
1785 src->getInsn()->dnz || src->getInsn()->precise)
1786 return false;
1787
1788 if (toOp == OP_SAD) {
1789 ImmediateValue imm;
1790 if (!src->getInsn()->src(2).getImmediate(imm))
1791 return false;
1792 if (!imm.isInteger(0))
1793 return false;
1794 }
1795
1796 if (typeSizeof(add->dType) != typeSizeof(src->getInsn()->dType) ||
1797 isFloatType(add->dType) != isFloatType(src->getInsn()->dType))
1798 return false;
1799
1800 mod[0] = add->src(0).mod;
1801 mod[1] = add->src(1).mod;
1802 mod[2] = src->getUniqueInsn()->src(0).mod;
1803 mod[3] = src->getUniqueInsn()->src(1).mod;
1804
1805 if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & modBad)
1806 return false;
1807
1808 add->op = toOp;
1809 add->subOp = src->getInsn()->subOp; // potentially mul-high
1810 add->dnz = src->getInsn()->dnz;
1811 add->dType = src->getInsn()->dType; // sign matters for imad hi
1812 add->sType = src->getInsn()->sType;
1813
1814 add->setSrc(2, add->src(s ? 0 : 1));
1815
1816 add->setSrc(0, src->getInsn()->getSrc(0));
1817 add->src(0).mod = mod[2] ^ mod[s];
1818 add->setSrc(1, src->getInsn()->getSrc(1));
1819 add->src(1).mod = mod[3];
1820
1821 return true;
1822 }
1823
1824 void
1825 AlgebraicOpt::handleMINMAX(Instruction *minmax)
1826 {
1827 Value *src0 = minmax->getSrc(0);
1828 Value *src1 = minmax->getSrc(1);
1829
1830 if (src0 != src1 || src0->reg.file != FILE_GPR)
1831 return;
1832 if (minmax->src(0).mod == minmax->src(1).mod) {
1833 if (minmax->def(0).mayReplace(minmax->src(0))) {
1834 minmax->def(0).replace(minmax->src(0), false);
1835 minmax->bb->remove(minmax);
1836 } else {
1837 minmax->op = OP_CVT;
1838 minmax->setSrc(1, NULL);
1839 }
1840 } else {
1841 // TODO:
1842 // min(x, -x) = -abs(x)
1843 // min(x, -abs(x)) = -abs(x)
1844 // min(x, abs(x)) = x
1845 // max(x, -abs(x)) = x
1846 // max(x, abs(x)) = abs(x)
1847 // max(x, -x) = abs(x)
1848 }
1849 }
1850
1851 // rcp(rcp(a)) = a
1852 // rcp(sqrt(a)) = rsq(a)
1853 void
1854 AlgebraicOpt::handleRCP(Instruction *rcp)
1855 {
1856 Instruction *si = rcp->getSrc(0)->getUniqueInsn();
1857
1858 if (!si)
1859 return;
1860
1861 if (si->op == OP_RCP) {
1862 Modifier mod = rcp->src(0).mod * si->src(0).mod;
1863 rcp->op = mod.getOp();
1864 rcp->setSrc(0, si->getSrc(0));
1865 } else if (si->op == OP_SQRT) {
1866 rcp->op = OP_RSQ;
1867 rcp->setSrc(0, si->getSrc(0));
1868 rcp->src(0).mod = rcp->src(0).mod * si->src(0).mod;
1869 }
1870 }
1871
1872 void
1873 AlgebraicOpt::handleSLCT(Instruction *slct)
1874 {
1875 if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) {
1876 if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f))
1877 slct->setSrc(0, slct->getSrc(1));
1878 } else
1879 if (slct->getSrc(0) != slct->getSrc(1)) {
1880 return;
1881 }
1882 slct->op = OP_MOV;
1883 slct->setSrc(1, NULL);
1884 slct->setSrc(2, NULL);
1885 }
1886
1887 void
1888 AlgebraicOpt::handleLOGOP(Instruction *logop)
1889 {
1890 Value *src0 = logop->getSrc(0);
1891 Value *src1 = logop->getSrc(1);
1892
1893 if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
1894 return;
1895
1896 if (src0 == src1) {
1897 if ((logop->op == OP_AND || logop->op == OP_OR) &&
1898 logop->def(0).mayReplace(logop->src(0))) {
1899 logop->def(0).replace(logop->src(0), false);
1900 delete_Instruction(prog, logop);
1901 }
1902 } else {
1903 // try AND(SET, SET) -> SET_AND(SET)
1904 Instruction *set0 = src0->getInsn();
1905 Instruction *set1 = src1->getInsn();
1906
1907 if (!set0 || set0->fixed || !set1 || set1->fixed)
1908 return;
1909 if (set1->op != OP_SET) {
1910 Instruction *xchg = set0;
1911 set0 = set1;
1912 set1 = xchg;
1913 if (set1->op != OP_SET)
1914 return;
1915 }
1916 operation redOp = (logop->op == OP_AND ? OP_SET_AND :
1917 logop->op == OP_XOR ? OP_SET_XOR : OP_SET_OR);
1918 if (!prog->getTarget()->isOpSupported(redOp, set1->sType))
1919 return;
1920 if (set0->op != OP_SET &&
1921 set0->op != OP_SET_AND &&
1922 set0->op != OP_SET_OR &&
1923 set0->op != OP_SET_XOR)
1924 return;
1925 if (set0->getDef(0)->refCount() > 1 &&
1926 set1->getDef(0)->refCount() > 1)
1927 return;
1928 if (set0->getPredicate() || set1->getPredicate())
1929 return;
1930 // check that they don't source each other
1931 for (int s = 0; s < 2; ++s)
1932 if (set0->getSrc(s) == set1->getDef(0) ||
1933 set1->getSrc(s) == set0->getDef(0))
1934 return;
1935
1936 set0 = cloneForward(func, set0);
1937 set1 = cloneShallow(func, set1);
1938 logop->bb->insertAfter(logop, set1);
1939 logop->bb->insertAfter(logop, set0);
1940
1941 set0->dType = TYPE_U8;
1942 set0->getDef(0)->reg.file = FILE_PREDICATE;
1943 set0->getDef(0)->reg.size = 1;
1944 set1->setSrc(2, set0->getDef(0));
1945 set1->op = redOp;
1946 set1->setDef(0, logop->getDef(0));
1947 delete_Instruction(prog, logop);
1948 }
1949 }
1950
1951 // F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0
1952 // nv50:
1953 // F2I(NEG(I2F(ABS(SET))))
1954 void
1955 AlgebraicOpt::handleCVT_NEG(Instruction *cvt)
1956 {
1957 Instruction *insn = cvt->getSrc(0)->getInsn();
1958 if (cvt->sType != TYPE_F32 ||
1959 cvt->dType != TYPE_S32 || cvt->src(0).mod != Modifier(0))
1960 return;
1961 if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32)
1962 return;
1963 if (insn->src(0).mod != Modifier(0))
1964 return;
1965 insn = insn->getSrc(0)->getInsn();
1966
1967 // check for nv50 SET(-1,0) -> SET(1.0f/0.0f) chain and nvc0's f32 SET
1968 if (insn && insn->op == OP_CVT &&
1969 insn->dType == TYPE_F32 &&
1970 insn->sType == TYPE_S32) {
1971 insn = insn->getSrc(0)->getInsn();
1972 if (!insn || insn->op != OP_ABS || insn->sType != TYPE_S32 ||
1973 insn->src(0).mod)
1974 return;
1975 insn = insn->getSrc(0)->getInsn();
1976 if (!insn || insn->op != OP_SET || insn->dType != TYPE_U32)
1977 return;
1978 } else
1979 if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32) {
1980 return;
1981 }
1982
1983 Instruction *bset = cloneShallow(func, insn);
1984 bset->dType = TYPE_U32;
1985 bset->setDef(0, cvt->getDef(0));
1986 cvt->bb->insertAfter(cvt, bset);
1987 delete_Instruction(prog, cvt);
1988 }
1989
1990 // F2I(TRUNC()) and so on can be expressed as a single CVT. If the earlier CVT
1991 // does a type conversion, this becomes trickier as there might be range
1992 // changes/etc. We could handle those in theory as long as the range was being
1993 // reduced or kept the same.
1994 void
1995 AlgebraicOpt::handleCVT_CVT(Instruction *cvt)
1996 {
1997 Instruction *insn = cvt->getSrc(0)->getInsn();
1998 RoundMode rnd = insn->rnd;
1999
2000 if (insn->saturate ||
2001 insn->subOp ||
2002 insn->dType != insn->sType ||
2003 insn->dType != cvt->sType)
2004 return;
2005
2006 switch (insn->op) {
2007 case OP_CEIL:
2008 rnd = ROUND_PI;
2009 break;
2010 case OP_FLOOR:
2011 rnd = ROUND_MI;
2012 break;
2013 case OP_TRUNC:
2014 rnd = ROUND_ZI;
2015 break;
2016 case OP_CVT:
2017 break;
2018 default:
2019 return;
2020 }
2021
2022 if (!isFloatType(cvt->dType) || !isFloatType(insn->sType))
2023 rnd = (RoundMode)(rnd & 3);
2024
2025 cvt->rnd = rnd;
2026 cvt->setSrc(0, insn->getSrc(0));
2027 cvt->src(0).mod *= insn->src(0).mod;
2028 cvt->sType = insn->sType;
2029 }
2030
2031 // Some shaders extract packed bytes out of words and convert them to
2032 // e.g. float. The Fermi+ CVT instruction can extract those directly, as can
2033 // nv50 for word sizes.
2034 //
2035 // CVT(EXTBF(x, byte/word))
2036 // CVT(AND(bytemask, x))
2037 // CVT(AND(bytemask, SHR(x, 8/16/24)))
2038 // CVT(SHR(x, 16/24))
2039 void
2040 AlgebraicOpt::handleCVT_EXTBF(Instruction *cvt)
2041 {
2042 Instruction *insn = cvt->getSrc(0)->getInsn();
2043 ImmediateValue imm;
2044 Value *arg = NULL;
2045 unsigned width, offset;
2046 if ((cvt->sType != TYPE_U32 && cvt->sType != TYPE_S32) || !insn)
2047 return;
2048 if (insn->op == OP_EXTBF && insn->src(1).getImmediate(imm)) {
2049 width = (imm.reg.data.u32 >> 8) & 0xff;
2050 offset = imm.reg.data.u32 & 0xff;
2051 arg = insn->getSrc(0);
2052
2053 if (width != 8 && width != 16)
2054 return;
2055 if (width == 8 && offset & 0x7)
2056 return;
2057 if (width == 16 && offset & 0xf)
2058 return;
2059 } else if (insn->op == OP_AND) {
2060 int s;
2061 if (insn->src(0).getImmediate(imm))
2062 s = 0;
2063 else if (insn->src(1).getImmediate(imm))
2064 s = 1;
2065 else
2066 return;
2067
2068 if (imm.reg.data.u32 == 0xff)
2069 width = 8;
2070 else if (imm.reg.data.u32 == 0xffff)
2071 width = 16;
2072 else
2073 return;
2074
2075 arg = insn->getSrc(!s);
2076 Instruction *shift = arg->getInsn();
2077 offset = 0;
2078 if (shift && shift->op == OP_SHR &&
2079 shift->sType == cvt->sType &&
2080 shift->src(1).getImmediate(imm) &&
2081 ((width == 8 && (imm.reg.data.u32 & 0x7) == 0) ||
2082 (width == 16 && (imm.reg.data.u32 & 0xf) == 0))) {
2083 arg = shift->getSrc(0);
2084 offset = imm.reg.data.u32;
2085 }
2086 // We just AND'd the high bits away, which means this is effectively an
2087 // unsigned value.
2088 cvt->sType = TYPE_U32;
2089 } else if (insn->op == OP_SHR &&
2090 insn->sType == cvt->sType &&
2091 insn->src(1).getImmediate(imm)) {
2092 arg = insn->getSrc(0);
2093 if (imm.reg.data.u32 == 24) {
2094 width = 8;
2095 offset = 24;
2096 } else if (imm.reg.data.u32 == 16) {
2097 width = 16;
2098 offset = 16;
2099 } else {
2100 return;
2101 }
2102 }
2103
2104 if (!arg)
2105 return;
2106
2107 // Irrespective of what came earlier, we can undo a shift on the argument
2108 // by adjusting the offset.
2109 Instruction *shift = arg->getInsn();
2110 if (shift && shift->op == OP_SHL &&
2111 shift->src(1).getImmediate(imm) &&
2112 ((width == 8 && (imm.reg.data.u32 & 0x7) == 0) ||
2113 (width == 16 && (imm.reg.data.u32 & 0xf) == 0)) &&
2114 imm.reg.data.u32 <= offset) {
2115 arg = shift->getSrc(0);
2116 offset -= imm.reg.data.u32;
2117 }
2118
2119 // The unpackSnorm lowering still leaves a few shifts behind, but it's too
2120 // annoying to detect them.
2121
2122 if (width == 8) {
2123 cvt->sType = cvt->sType == TYPE_U32 ? TYPE_U8 : TYPE_S8;
2124 } else {
2125 assert(width == 16);
2126 cvt->sType = cvt->sType == TYPE_U32 ? TYPE_U16 : TYPE_S16;
2127 }
2128 cvt->setSrc(0, arg);
2129 cvt->subOp = offset >> 3;
2130 }
2131
2132 // SUCLAMP dst, (ADD b imm), k, 0 -> SUCLAMP dst, b, k, imm (if imm fits s6)
2133 void
2134 AlgebraicOpt::handleSUCLAMP(Instruction *insn)
2135 {
2136 ImmediateValue imm;
2137 int32_t val = insn->getSrc(2)->asImm()->reg.data.s32;
2138 int s;
2139 Instruction *add;
2140
2141 assert(insn->srcExists(0) && insn->src(0).getFile() == FILE_GPR);
2142
2143 // look for ADD (TODO: only count references by non-SUCLAMP)
2144 if (insn->getSrc(0)->refCount() > 1)
2145 return;
2146 add = insn->getSrc(0)->getInsn();
2147 if (!add || add->op != OP_ADD ||
2148 (add->dType != TYPE_U32 &&
2149 add->dType != TYPE_S32))
2150 return;
2151
2152 // look for immediate
2153 for (s = 0; s < 2; ++s)
2154 if (add->src(s).getImmediate(imm))
2155 break;
2156 if (s >= 2)
2157 return;
2158 s = s ? 0 : 1;
2159 // determine if immediate fits
2160 val += imm.reg.data.s32;
2161 if (val > 31 || val < -32)
2162 return;
2163 // determine if other addend fits
2164 if (add->src(s).getFile() != FILE_GPR || add->src(s).mod != Modifier(0))
2165 return;
2166
2167 bld.setPosition(insn, false); // make sure bld is init'ed
2168 // replace sources
2169 insn->setSrc(2, bld.mkImm(val));
2170 insn->setSrc(0, add->getSrc(s));
2171 }
2172
2173 // NEG(AND(SET, 1)) -> SET
2174 void
2175 AlgebraicOpt::handleNEG(Instruction *i) {
2176 Instruction *src = i->getSrc(0)->getInsn();
2177 ImmediateValue imm;
2178 int b;
2179
2180 if (isFloatType(i->sType) || !src || src->op != OP_AND)
2181 return;
2182
2183 if (src->src(0).getImmediate(imm))
2184 b = 1;
2185 else if (src->src(1).getImmediate(imm))
2186 b = 0;
2187 else
2188 return;
2189
2190 if (!imm.isInteger(1))
2191 return;
2192
2193 Instruction *set = src->getSrc(b)->getInsn();
2194 if ((set->op == OP_SET || set->op == OP_SET_AND ||
2195 set->op == OP_SET_OR || set->op == OP_SET_XOR) &&
2196 !isFloatType(set->dType)) {
2197 i->def(0).replace(set->getDef(0), false);
2198 }
2199 }
2200
2201 // EXTBF(RDSV(COMBINED_TID)) -> RDSV(TID)
2202 void
2203 AlgebraicOpt::handleEXTBF_RDSV(Instruction *i)
2204 {
2205 Instruction *rdsv = i->getSrc(0)->getUniqueInsn();
2206 if (rdsv->op != OP_RDSV ||
2207 rdsv->getSrc(0)->asSym()->reg.data.sv.sv != SV_COMBINED_TID)
2208 return;
2209 // Avoid creating more RDSV instructions
2210 if (rdsv->getDef(0)->refCount() > 1)
2211 return;
2212
2213 ImmediateValue imm;
2214 if (!i->src(1).getImmediate(imm))
2215 return;
2216
2217 int index;
2218 if (imm.isInteger(0x1000))
2219 index = 0;
2220 else
2221 if (imm.isInteger(0x0a10))
2222 index = 1;
2223 else
2224 if (imm.isInteger(0x061a))
2225 index = 2;
2226 else
2227 return;
2228
2229 bld.setPosition(i, false);
2230
2231 i->op = OP_RDSV;
2232 i->setSrc(0, bld.mkSysVal(SV_TID, index));
2233 i->setSrc(1, NULL);
2234 }
2235
2236 bool
2237 AlgebraicOpt::visit(BasicBlock *bb)
2238 {
2239 Instruction *next;
2240 for (Instruction *i = bb->getEntry(); i; i = next) {
2241 next = i->next;
2242 switch (i->op) {
2243 case OP_ABS:
2244 handleABS(i);
2245 break;
2246 case OP_ADD:
2247 handleADD(i);
2248 break;
2249 case OP_RCP:
2250 handleRCP(i);
2251 break;
2252 case OP_MIN:
2253 case OP_MAX:
2254 handleMINMAX(i);
2255 break;
2256 case OP_SLCT:
2257 handleSLCT(i);
2258 break;
2259 case OP_AND:
2260 case OP_OR:
2261 case OP_XOR:
2262 handleLOGOP(i);
2263 break;
2264 case OP_CVT:
2265 handleCVT_NEG(i);
2266 handleCVT_CVT(i);
2267 if (prog->getTarget()->isOpSupported(OP_EXTBF, TYPE_U32))
2268 handleCVT_EXTBF(i);
2269 break;
2270 case OP_SUCLAMP:
2271 handleSUCLAMP(i);
2272 break;
2273 case OP_NEG:
2274 handleNEG(i);
2275 break;
2276 case OP_EXTBF:
2277 handleEXTBF_RDSV(i);
2278 break;
2279 default:
2280 break;
2281 }
2282 }
2283
2284 return true;
2285 }
2286
2287 // =============================================================================
2288
2289 // ADD(SHL(a, b), c) -> SHLADD(a, b, c)
2290 class LateAlgebraicOpt : public Pass
2291 {
2292 private:
2293 virtual bool visit(Instruction *);
2294
2295 void handleADD(Instruction *);
2296 bool tryADDToSHLADD(Instruction *);
2297 };
2298
2299 void
2300 LateAlgebraicOpt::handleADD(Instruction *add)
2301 {
2302 Value *src0 = add->getSrc(0);
2303 Value *src1 = add->getSrc(1);
2304
2305 if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
2306 return;
2307
2308 if (prog->getTarget()->isOpSupported(OP_SHLADD, add->dType))
2309 tryADDToSHLADD(add);
2310 }
2311
2312 // ADD(SHL(a, b), c) -> SHLADD(a, b, c)
2313 bool
2314 LateAlgebraicOpt::tryADDToSHLADD(Instruction *add)
2315 {
2316 Value *src0 = add->getSrc(0);
2317 Value *src1 = add->getSrc(1);
2318 ImmediateValue imm;
2319 Instruction *shl;
2320 Value *src;
2321 int s;
2322
2323 if (add->saturate || add->usesFlags() || typeSizeof(add->dType) == 8
2324 || isFloatType(add->dType))
2325 return false;
2326
2327 if (src0->getUniqueInsn() && src0->getUniqueInsn()->op == OP_SHL)
2328 s = 0;
2329 else
2330 if (src1->getUniqueInsn() && src1->getUniqueInsn()->op == OP_SHL)
2331 s = 1;
2332 else
2333 return false;
2334
2335 src = add->getSrc(s);
2336 shl = src->getUniqueInsn();
2337
2338 if (shl->bb != add->bb || shl->usesFlags() || shl->subOp || shl->src(0).mod)
2339 return false;
2340
2341 if (!shl->src(1).getImmediate(imm))
2342 return false;
2343
2344 add->op = OP_SHLADD;
2345 add->setSrc(2, add->src(!s));
2346 // SHL can't have any modifiers, but the ADD source may have had
2347 // one. Preserve it.
2348 add->setSrc(0, shl->getSrc(0));
2349 if (s == 1)
2350 add->src(0).mod = add->src(1).mod;
2351 add->setSrc(1, new_ImmediateValue(shl->bb->getProgram(), imm.reg.data.u32));
2352 add->src(1).mod = Modifier(0);
2353
2354 return true;
2355 }
2356
2357 bool
2358 LateAlgebraicOpt::visit(Instruction *i)
2359 {
2360 switch (i->op) {
2361 case OP_ADD:
2362 handleADD(i);
2363 break;
2364 default:
2365 break;
2366 }
2367
2368 return true;
2369 }
2370
2371 // =============================================================================
2372
2373 // Split 64-bit MUL and MAD
2374 class Split64BitOpPreRA : public Pass
2375 {
2376 private:
2377 virtual bool visit(BasicBlock *);
2378 void split64MulMad(Function *, Instruction *, DataType);
2379
2380 BuildUtil bld;
2381 };
2382
2383 bool
2384 Split64BitOpPreRA::visit(BasicBlock *bb)
2385 {
2386 Instruction *i, *next;
2387 Modifier mod;
2388
2389 for (i = bb->getEntry(); i; i = next) {
2390 next = i->next;
2391
2392 DataType hTy;
2393 switch (i->dType) {
2394 case TYPE_U64: hTy = TYPE_U32; break;
2395 case TYPE_S64: hTy = TYPE_S32; break;
2396 default:
2397 continue;
2398 }
2399
2400 if (i->op == OP_MAD || i->op == OP_MUL)
2401 split64MulMad(func, i, hTy);
2402 }
2403
2404 return true;
2405 }
2406
2407 void
2408 Split64BitOpPreRA::split64MulMad(Function *fn, Instruction *i, DataType hTy)
2409 {
2410 assert(i->op == OP_MAD || i->op == OP_MUL);
2411 assert(!isFloatType(i->dType) && !isFloatType(i->sType));
2412 assert(typeSizeof(hTy) == 4);
2413
2414 bld.setPosition(i, true);
2415
2416 Value *zero = bld.mkImm(0u);
2417 Value *carry = bld.getSSA(1, FILE_FLAGS);
2418
2419 // We want to compute `d = a * b (+ c)?`, where a, b, c and d are 64-bit
2420 // values (a, b and c might be 32-bit values), using 32-bit operations. This
2421 // gives the following operations:
2422 // * `d.low = low(a.low * b.low) (+ c.low)?`
2423 // * `d.high = low(a.high * b.low) + low(a.low * b.high)
2424 // + high(a.low * b.low) (+ c.high)?`
2425 //
2426 // To compute the high bits, we can split in the following operations:
2427 // * `tmp1 = low(a.high * b.low) (+ c.high)?`
2428 // * `tmp2 = low(a.low * b.high) + tmp1`
2429 // * `d.high = high(a.low * b.low) + tmp2`
2430 //
2431 // mkSplit put lower bits at index 0 and higher bits at index 1
2432
2433 Value *op1[2];
2434 if (i->getSrc(0)->reg.size == 8)
2435 bld.mkSplit(op1, 4, i->getSrc(0));
2436 else {
2437 op1[0] = i->getSrc(0);
2438 op1[1] = zero;
2439 }
2440 Value *op2[2];
2441 if (i->getSrc(1)->reg.size == 8)
2442 bld.mkSplit(op2, 4, i->getSrc(1));
2443 else {
2444 op2[0] = i->getSrc(1);
2445 op2[1] = zero;
2446 }
2447
2448 Value *op3[2] = { NULL, NULL };
2449 if (i->op == OP_MAD) {
2450 if (i->getSrc(2)->reg.size == 8)
2451 bld.mkSplit(op3, 4, i->getSrc(2));
2452 else {
2453 op3[0] = i->getSrc(2);
2454 op3[1] = zero;
2455 }
2456 }
2457
2458 Value *tmpRes1Hi = bld.getSSA();
2459 if (i->op == OP_MAD)
2460 bld.mkOp3(OP_MAD, hTy, tmpRes1Hi, op1[1], op2[0], op3[1]);
2461 else
2462 bld.mkOp2(OP_MUL, hTy, tmpRes1Hi, op1[1], op2[0]);
2463
2464 Value *tmpRes2Hi = bld.mkOp3v(OP_MAD, hTy, bld.getSSA(), op1[0], op2[1], tmpRes1Hi);
2465
2466 Value *def[2] = { bld.getSSA(), bld.getSSA() };
2467
2468 // If it was a MAD, add the carry from the low bits
2469 // It is not needed if it was a MUL, since we added high(a.low * b.low) to
2470 // d.high
2471 if (i->op == OP_MAD)
2472 bld.mkOp3(OP_MAD, hTy, def[0], op1[0], op2[0], op3[0])->setFlagsDef(1, carry);
2473 else
2474 bld.mkOp2(OP_MUL, hTy, def[0], op1[0], op2[0]);
2475
2476 Instruction *hiPart3 = bld.mkOp3(OP_MAD, hTy, def[1], op1[0], op2[0], tmpRes2Hi);
2477 hiPart3->subOp = NV50_IR_SUBOP_MUL_HIGH;
2478 if (i->op == OP_MAD)
2479 hiPart3->setFlagsSrc(3, carry);
2480
2481 bld.mkOp2(OP_MERGE, i->dType, i->getDef(0), def[0], def[1]);
2482
2483 delete_Instruction(fn->getProgram(), i);
2484 }
2485
2486 // =============================================================================
2487
2488 static inline void
2489 updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn)
2490 {
2491 if (offset != ldst->getSrc(0)->reg.data.offset) {
2492 if (ldst->getSrc(0)->refCount() > 1)
2493 ldst->setSrc(0, cloneShallow(fn, ldst->getSrc(0)));
2494 ldst->getSrc(0)->reg.data.offset = offset;
2495 }
2496 }
2497
2498 // Combine loads and stores, forward stores to loads where possible.
2499 class MemoryOpt : public Pass
2500 {
2501 private:
2502 class Record
2503 {
2504 public:
2505 Record *next;
2506 Instruction *insn;
2507 const Value *rel[2];
2508 const Value *base;
2509 int32_t offset;
2510 int8_t fileIndex;
2511 uint8_t size;
2512 bool locked;
2513 Record *prev;
2514
2515 bool overlaps(const Instruction *ldst) const;
2516
2517 inline void link(Record **);
2518 inline void unlink(Record **);
2519 inline void set(const Instruction *ldst);
2520 };
2521
2522 public:
2523 MemoryOpt();
2524
2525 Record *loads[DATA_FILE_COUNT];
2526 Record *stores[DATA_FILE_COUNT];
2527
2528 MemoryPool recordPool;
2529
2530 private:
2531 virtual bool visit(BasicBlock *);
2532 bool runOpt(BasicBlock *);
2533
2534 Record **getList(const Instruction *);
2535
2536 Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const;
2537
2538 // merge @insn into load/store instruction from @rec
2539 bool combineLd(Record *rec, Instruction *ld);
2540 bool combineSt(Record *rec, Instruction *st);
2541
2542 bool replaceLdFromLd(Instruction *ld, Record *ldRec);
2543 bool replaceLdFromSt(Instruction *ld, Record *stRec);
2544 bool replaceStFromSt(Instruction *restrict st, Record *stRec);
2545
2546 void addRecord(Instruction *ldst);
2547 void purgeRecords(Instruction *const st, DataFile);
2548 void lockStores(Instruction *const ld);
2549 void reset();
2550
2551 private:
2552 Record *prevRecord;
2553 };
2554
2555 MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6)
2556 {
2557 for (int i = 0; i < DATA_FILE_COUNT; ++i) {
2558 loads[i] = NULL;
2559 stores[i] = NULL;
2560 }
2561 prevRecord = NULL;
2562 }
2563
2564 void
2565 MemoryOpt::reset()
2566 {
2567 for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) {
2568 Record *it, *next;
2569 for (it = loads[i]; it; it = next) {
2570 next = it->next;
2571 recordPool.release(it);
2572 }
2573 loads[i] = NULL;
2574 for (it = stores[i]; it; it = next) {
2575 next = it->next;
2576 recordPool.release(it);
2577 }
2578 stores[i] = NULL;
2579 }
2580 }
2581
2582 bool
2583 MemoryOpt::combineLd(Record *rec, Instruction *ld)
2584 {
2585 int32_t offRc = rec->offset;
2586 int32_t offLd = ld->getSrc(0)->reg.data.offset;
2587 int sizeRc = rec->size;
2588 int sizeLd = typeSizeof(ld->dType);
2589 int size = sizeRc + sizeLd;
2590 int d, j;
2591
2592 if (!prog->getTarget()->
2593 isAccessSupported(ld->getSrc(0)->reg.file, typeOfSize(size)))
2594 return false;
2595 // no unaligned loads
2596 if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) ||
2597 ((size == 0xc) && (MIN2(offLd, offRc) & 0xf)))
2598 return false;
2599 // for compute indirect loads are not guaranteed to be aligned
2600 if (prog->getType() == Program::TYPE_COMPUTE && rec->rel[0])
2601 return false;
2602
2603 assert(sizeRc + sizeLd <= 16 && offRc != offLd);
2604
2605 // lock any stores that overlap with the load being merged into the
2606 // existing record.
2607 lockStores(ld);
2608
2609 for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j);
2610
2611 if (offLd < offRc) {
2612 int sz;
2613 for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d);
2614 // d: nr of definitions in ld
2615 // j: nr of definitions in rec->insn, move:
2616 for (d = d + j - 1; j > 0; --j, --d)
2617 rec->insn->setDef(d, rec->insn->getDef(j - 1));
2618
2619 if (rec->insn->getSrc(0)->refCount() > 1)
2620 rec->insn->setSrc(0, cloneShallow(func, rec->insn->getSrc(0)));
2621 rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd;
2622
2623 d = 0;
2624 } else {
2625 d = j;
2626 }
2627 // move definitions of @ld to @rec->insn
2628 for (j = 0; sizeLd; ++j, ++d) {
2629 sizeLd -= ld->getDef(j)->reg.size;
2630 rec->insn->setDef(d, ld->getDef(j));
2631 }
2632
2633 rec->size = size;
2634 rec->insn->getSrc(0)->reg.size = size;
2635 rec->insn->setType(typeOfSize(size));
2636
2637 delete_Instruction(prog, ld);
2638
2639 return true;
2640 }
2641
2642 bool
2643 MemoryOpt::combineSt(Record *rec, Instruction *st)
2644 {
2645 int32_t offRc = rec->offset;
2646 int32_t offSt = st->getSrc(0)->reg.data.offset;
2647 int sizeRc = rec->size;
2648 int sizeSt = typeSizeof(st->dType);
2649 int s = sizeSt / 4;
2650 int size = sizeRc + sizeSt;
2651 int j, k;
2652 Value *src[4]; // no modifiers in ValueRef allowed for st
2653 Value *extra[3];
2654
2655 if (!prog->getTarget()->
2656 isAccessSupported(st->getSrc(0)->reg.file, typeOfSize(size)))
2657 return false;
2658 // no unaligned stores
2659 if (size == 8 && MIN2(offRc, offSt) & 0x7)
2660 return false;
2661 // for compute indirect stores are not guaranteed to be aligned
2662 if (prog->getType() == Program::TYPE_COMPUTE && rec->rel[0])
2663 return false;
2664
2665 // remove any existing load/store records for the store being merged into
2666 // the existing record.
2667 purgeRecords(st, DATA_FILE_COUNT);
2668
2669 st->takeExtraSources(0, extra); // save predicate and indirect address
2670
2671 if (offRc < offSt) {
2672 // save values from @st
2673 for (s = 0; sizeSt; ++s) {
2674 sizeSt -= st->getSrc(s + 1)->reg.size;
2675 src[s] = st->getSrc(s + 1);
2676 }
2677 // set record's values as low sources of @st
2678 for (j = 1; sizeRc; ++j) {
2679 sizeRc -= rec->insn->getSrc(j)->reg.size;
2680 st->setSrc(j, rec->insn->getSrc(j));
2681 }
2682 // set saved values as high sources of @st
2683 for (k = j, j = 0; j < s; ++j)
2684 st->setSrc(k++, src[j]);
2685
2686 updateLdStOffset(st, offRc, func);
2687 } else {
2688 for (j = 1; sizeSt; ++j)
2689 sizeSt -= st->getSrc(j)->reg.size;
2690 for (s = 1; sizeRc; ++j, ++s) {
2691 sizeRc -= rec->insn->getSrc(s)->reg.size;
2692 st->setSrc(j, rec->insn->getSrc(s));
2693 }
2694 rec->offset = offSt;
2695 }
2696 st->putExtraSources(0, extra); // restore pointer and predicate
2697
2698 delete_Instruction(prog, rec->insn);
2699 rec->insn = st;
2700 rec->size = size;
2701 rec->insn->getSrc(0)->reg.size = size;
2702 rec->insn->setType(typeOfSize(size));
2703 return true;
2704 }
2705
2706 void
2707 MemoryOpt::Record::set(const Instruction *ldst)
2708 {
2709 const Symbol *mem = ldst->getSrc(0)->asSym();
2710 fileIndex = mem->reg.fileIndex;
2711 rel[0] = ldst->getIndirect(0, 0);
2712 rel[1] = ldst->getIndirect(0, 1);
2713 offset = mem->reg.data.offset;
2714 base = mem->getBase();
2715 size = typeSizeof(ldst->sType);
2716 }
2717
2718 void
2719 MemoryOpt::Record::link(Record **list)
2720 {
2721 next = *list;
2722 if (next)
2723 next->prev = this;
2724 prev = NULL;
2725 *list = this;
2726 }
2727
2728 void
2729 MemoryOpt::Record::unlink(Record **list)
2730 {
2731 if (next)
2732 next->prev = prev;
2733 if (prev)
2734 prev->next = next;
2735 else
2736 *list = next;
2737 }
2738
2739 MemoryOpt::Record **
2740 MemoryOpt::getList(const Instruction *insn)
2741 {
2742 if (insn->op == OP_LOAD || insn->op == OP_VFETCH)
2743 return &loads[insn->src(0).getFile()];
2744 return &stores[insn->src(0).getFile()];
2745 }
2746
2747 void
2748 MemoryOpt::addRecord(Instruction *i)
2749 {
2750 Record **list = getList(i);
2751 Record *it = reinterpret_cast<Record *>(recordPool.allocate());
2752
2753 it->link(list);
2754 it->set(i);
2755 it->insn = i;
2756 it->locked = false;
2757 }
2758
2759 MemoryOpt::Record *
2760 MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const
2761 {
2762 const Symbol *sym = insn->getSrc(0)->asSym();
2763 const int size = typeSizeof(insn->sType);
2764 Record *rec = NULL;
2765 Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file];
2766
2767 for (; it; it = it->next) {
2768 if (it->locked && insn->op != OP_LOAD && insn->op != OP_VFETCH)
2769 continue;
2770 if ((it->offset >> 4) != (sym->reg.data.offset >> 4) ||
2771 it->rel[0] != insn->getIndirect(0, 0) ||
2772 it->fileIndex != sym->reg.fileIndex ||
2773 it->rel[1] != insn->getIndirect(0, 1))
2774 continue;
2775
2776 if (it->offset < sym->reg.data.offset) {
2777 if (it->offset + it->size >= sym->reg.data.offset) {
2778 isAdj = (it->offset + it->size == sym->reg.data.offset);
2779 if (!isAdj)
2780 return it;
2781 if (!(it->offset & 0x7))
2782 rec = it;
2783 }
2784 } else {
2785 isAdj = it->offset != sym->reg.data.offset;
2786 if (size <= it->size && !isAdj)
2787 return it;
2788 else
2789 if (!(sym->reg.data.offset & 0x7))
2790 if (it->offset - size <= sym->reg.data.offset)
2791 rec = it;
2792 }
2793 }
2794 return rec;
2795 }
2796
2797 bool
2798 MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec)
2799 {
2800 Instruction *st = rec->insn;
2801 int32_t offSt = rec->offset;
2802 int32_t offLd = ld->getSrc(0)->reg.data.offset;
2803 int d, s;
2804
2805 for (s = 1; offSt != offLd && st->srcExists(s); ++s)
2806 offSt += st->getSrc(s)->reg.size;
2807 if (offSt != offLd)
2808 return false;
2809
2810 for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) {
2811 if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size)
2812 return false;
2813 if (st->getSrc(s)->reg.file != FILE_GPR)
2814 return false;
2815 ld->def(d).replace(st->src(s), false);
2816 }
2817 ld->bb->remove(ld);
2818 return true;
2819 }
2820
2821 bool
2822 MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec)
2823 {
2824 Instruction *ldR = rec->insn;
2825 int32_t offR = rec->offset;
2826 int32_t offE = ldE->getSrc(0)->reg.data.offset;
2827 int dR, dE;
2828
2829 assert(offR <= offE);
2830 for (dR = 0; offR < offE && ldR->defExists(dR); ++dR)
2831 offR += ldR->getDef(dR)->reg.size;
2832 if (offR != offE)
2833 return false;
2834
2835 for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) {
2836 if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size)
2837 return false;
2838 ldE->def(dE).replace(ldR->getDef(dR), false);
2839 }
2840
2841 delete_Instruction(prog, ldE);
2842 return true;
2843 }
2844
2845 bool
2846 MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec)
2847 {
2848 const Instruction *const ri = rec->insn;
2849 Value *extra[3];
2850
2851 int32_t offS = st->getSrc(0)->reg.data.offset;
2852 int32_t offR = rec->offset;
2853 int32_t endS = offS + typeSizeof(st->dType);
2854 int32_t endR = offR + typeSizeof(ri->dType);
2855
2856 rec->size = MAX2(endS, endR) - MIN2(offS, offR);
2857
2858 st->takeExtraSources(0, extra);
2859
2860 if (offR < offS) {
2861 Value *vals[10];
2862 int s, n;
2863 int k = 0;
2864 // get non-replaced sources of ri
2865 for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s)
2866 vals[k++] = ri->getSrc(s);
2867 n = s;
2868 // get replaced sources of st
2869 for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s)
2870 vals[k++] = st->getSrc(s);
2871 // skip replaced sources of ri
2872 for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s);
2873 // get non-replaced sources after values covered by st
2874 for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s)
2875 vals[k++] = ri->getSrc(s);
2876 assert((unsigned int)k <= ARRAY_SIZE(vals));
2877 for (s = 0; s < k; ++s)
2878 st->setSrc(s + 1, vals[s]);
2879 st->setSrc(0, ri->getSrc(0));
2880 } else
2881 if (endR > endS) {
2882 int j, s;
2883 for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size);
2884 for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size);
2885 for (; offR < endR; offR += ri->getSrc(j++)->reg.size)
2886 st->setSrc(s++, ri->getSrc(j));
2887 }
2888 st->putExtraSources(0, extra);
2889
2890 delete_Instruction(prog, rec->insn);
2891
2892 rec->insn = st;
2893 rec->offset = st->getSrc(0)->reg.data.offset;
2894
2895 st->setType(typeOfSize(rec->size));
2896
2897 return true;
2898 }
2899
2900 bool
2901 MemoryOpt::Record::overlaps(const Instruction *ldst) const
2902 {
2903 Record that;
2904 that.set(ldst);
2905
2906 // This assumes that images/buffers can't overlap. They can.
2907 // TODO: Plumb the restrict logic through, and only skip when it's a
2908 // restrict situation, or there can implicitly be no writes.
2909 if (this->fileIndex != that.fileIndex && this->rel[1] == that.rel[1])
2910 return false;
2911
2912 if (this->rel[0] || that.rel[0])
2913 return this->base == that.base;
2914
2915 return
2916 (this->offset < that.offset + that.size) &&
2917 (this->offset + this->size > that.offset);
2918 }
2919
2920 // We must not eliminate stores that affect the result of @ld if
2921 // we find later stores to the same location, and we may no longer
2922 // merge them with later stores.
2923 // The stored value can, however, still be used to determine the value
2924 // returned by future loads.
2925 void
2926 MemoryOpt::lockStores(Instruction *const ld)
2927 {
2928 for (Record *r = stores[ld->src(0).getFile()]; r; r = r->next)
2929 if (!r->locked && r->overlaps(ld))
2930 r->locked = true;
2931 }
2932
2933 // Prior loads from the location of @st are no longer valid.
2934 // Stores to the location of @st may no longer be used to derive
2935 // the value at it nor be coalesced into later stores.
2936 void
2937 MemoryOpt::purgeRecords(Instruction *const st, DataFile f)
2938 {
2939 if (st)
2940 f = st->src(0).getFile();
2941
2942 for (Record *r = loads[f]; r; r = r->next)
2943 if (!st || r->overlaps(st))
2944 r->unlink(&loads[f]);
2945
2946 for (Record *r = stores[f]; r; r = r->next)
2947 if (!st || r->overlaps(st))
2948 r->unlink(&stores[f]);
2949 }
2950
2951 bool
2952 MemoryOpt::visit(BasicBlock *bb)
2953 {
2954 bool ret = runOpt(bb);
2955 // Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st
2956 // where 96 bit memory operations are forbidden.
2957 if (ret)
2958 ret = runOpt(bb);
2959 return ret;
2960 }
2961
2962 bool
2963 MemoryOpt::runOpt(BasicBlock *bb)
2964 {
2965 Instruction *ldst, *next;
2966 Record *rec;
2967 bool isAdjacent = true;
2968
2969 for (ldst = bb->getEntry(); ldst; ldst = next) {
2970 bool keep = true;
2971 bool isLoad = true;
2972 next = ldst->next;
2973
2974 if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) {
2975 if (ldst->isDead()) {
2976 // might have been produced by earlier optimization
2977 delete_Instruction(prog, ldst);
2978 continue;
2979 }
2980 } else
2981 if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) {
2982 if (typeSizeof(ldst->dType) == 4 &&
2983 ldst->src(1).getFile() == FILE_GPR &&
2984 ldst->getSrc(1)->getInsn()->op == OP_NOP) {
2985 delete_Instruction(prog, ldst);
2986 continue;
2987 }
2988 isLoad = false;
2989 } else {
2990 // TODO: maybe have all fixed ops act as barrier ?
2991 if (ldst->op == OP_CALL ||
2992 ldst->op == OP_BAR ||
2993 ldst->op == OP_MEMBAR) {
2994 purgeRecords(NULL, FILE_MEMORY_LOCAL);
2995 purgeRecords(NULL, FILE_MEMORY_GLOBAL);
2996 purgeRecords(NULL, FILE_MEMORY_SHARED);
2997 purgeRecords(NULL, FILE_SHADER_OUTPUT);
2998 } else
2999 if (ldst->op == OP_ATOM || ldst->op == OP_CCTL) {
3000 if (ldst->src(0).getFile() == FILE_MEMORY_GLOBAL) {
3001 purgeRecords(NULL, FILE_MEMORY_LOCAL);
3002 purgeRecords(NULL, FILE_MEMORY_GLOBAL);
3003 purgeRecords(NULL, FILE_MEMORY_SHARED);
3004 } else {
3005 purgeRecords(NULL, ldst->src(0).getFile());
3006 }
3007 } else
3008 if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) {
3009 purgeRecords(NULL, FILE_SHADER_OUTPUT);
3010 }
3011 continue;
3012 }
3013 if (ldst->getPredicate()) // TODO: handle predicated ld/st
3014 continue;
3015 if (ldst->perPatch) // TODO: create separate per-patch lists
3016 continue;
3017
3018 if (isLoad) {
3019 DataFile file = ldst->src(0).getFile();
3020
3021 // if ld l[]/g[] look for previous store to eliminate the reload
3022 if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) {
3023 // TODO: shared memory ?
3024 rec = findRecord(ldst, false, isAdjacent);
3025 if (rec && !isAdjacent)
3026 keep = !replaceLdFromSt(ldst, rec);
3027 }
3028
3029 // or look for ld from the same location and replace this one
3030 rec = keep ? findRecord(ldst, true, isAdjacent) : NULL;
3031 if (rec) {
3032 if (!isAdjacent)
3033 keep = !replaceLdFromLd(ldst, rec);
3034 else
3035 // or combine a previous load with this one
3036 keep = !combineLd(rec, ldst);
3037 }
3038 if (keep)
3039 lockStores(ldst);
3040 } else {
3041 rec = findRecord(ldst, false, isAdjacent);
3042 if (rec) {
3043 if (!isAdjacent)
3044 keep = !replaceStFromSt(ldst, rec);
3045 else
3046 keep = !combineSt(rec, ldst);
3047 }
3048 if (keep)
3049 purgeRecords(ldst, DATA_FILE_COUNT);
3050 }
3051 if (keep)
3052 addRecord(ldst);
3053 }
3054 reset();
3055
3056 return true;
3057 }
3058
3059 // =============================================================================
3060
3061 // Turn control flow into predicated instructions (after register allocation !).
3062 // TODO:
3063 // Could move this to before register allocation on NVC0 and also handle nested
3064 // constructs.
3065 class FlatteningPass : public Pass
3066 {
3067 private:
3068 virtual bool visit(Function *);
3069 virtual bool visit(BasicBlock *);
3070
3071 bool tryPredicateConditional(BasicBlock *);
3072 void predicateInstructions(BasicBlock *, Value *pred, CondCode cc);
3073 void tryPropagateBranch(BasicBlock *);
3074 inline bool isConstantCondition(Value *pred);
3075 inline bool mayPredicate(const Instruction *, const Value *pred) const;
3076 inline void removeFlow(Instruction *);
3077
3078 uint8_t gpr_unit;
3079 };
3080
3081 bool
3082 FlatteningPass::isConstantCondition(Value *pred)
3083 {
3084 Instruction *insn = pred->getUniqueInsn();
3085 assert(insn);
3086 if (insn->op != OP_SET || insn->srcExists(2))
3087 return false;
3088
3089 for (int s = 0; s < 2 && insn->srcExists(s); ++s) {
3090 Instruction *ld = insn->getSrc(s)->getUniqueInsn();
3091 DataFile file;
3092 if (ld) {
3093 if (ld->op != OP_MOV && ld->op != OP_LOAD)
3094 return false;
3095 if (ld->src(0).isIndirect(0))
3096 return false;
3097 file = ld->src(0).getFile();
3098 } else {
3099 file = insn->src(s).getFile();
3100 // catch $r63 on NVC0 and $r63/$r127 on NV50. Unfortunately maxGPR is
3101 // in register "units", which can vary between targets.
3102 if (file == FILE_GPR) {
3103 Value *v = insn->getSrc(s);
3104 int bytes = v->reg.data.id * MIN2(v->reg.size, 4);
3105 int units = bytes >> gpr_unit;
3106 if (units > prog->maxGPR)
3107 file = FILE_IMMEDIATE;
3108 }
3109 }
3110 if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST)
3111 return false;
3112 }
3113 return true;
3114 }
3115
3116 void
3117 FlatteningPass::removeFlow(Instruction *insn)
3118 {
3119 FlowInstruction *term = insn ? insn->asFlow() : NULL;
3120 if (!term)
3121 return;
3122 Graph::Edge::Type ty = term->bb->cfg.outgoing().getType();
3123
3124 if (term->op == OP_BRA) {
3125 // TODO: this might get more difficult when we get arbitrary BRAs
3126 if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK)
3127 return;
3128 } else
3129 if (term->op != OP_JOIN)
3130 return;
3131
3132 Value *pred = term->getPredicate();
3133
3134 delete_Instruction(prog, term);
3135
3136 if (pred && pred->refCount() == 0) {
3137 Instruction *pSet = pred->getUniqueInsn();
3138 pred->join->reg.data.id = -1; // deallocate
3139 if (pSet->isDead())
3140 delete_Instruction(prog, pSet);
3141 }
3142 }
3143
3144 void
3145 FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc)
3146 {
3147 for (Instruction *i = bb->getEntry(); i; i = i->next) {
3148 if (i->isNop())
3149 continue;
3150 assert(!i->getPredicate());
3151 i->setPredicate(cc, pred);
3152 }
3153 removeFlow(bb->getExit());
3154 }
3155
3156 bool
3157 FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const
3158 {
3159 if (insn->isPseudo())
3160 return true;
3161 // TODO: calls where we don't know which registers are modified
3162
3163 if (!prog->getTarget()->mayPredicate(insn, pred))
3164 return false;
3165 for (int d = 0; insn->defExists(d); ++d)
3166 if (insn->getDef(d)->equals(pred))
3167 return false;
3168 return true;
3169 }
3170
3171 // If we jump to BRA/RET/EXIT, replace the jump with it.
3172 // NOTE: We do not update the CFG anymore here !
3173 //
3174 // TODO: Handle cases where we skip over a branch (maybe do that elsewhere ?):
3175 // BB:0
3176 // @p0 bra BB:2 -> @!p0 bra BB:3 iff (!) BB:2 immediately adjoins BB:1
3177 // BB1:
3178 // bra BB:3
3179 // BB2:
3180 // ...
3181 // BB3:
3182 // ...
3183 void
3184 FlatteningPass::tryPropagateBranch(BasicBlock *bb)
3185 {
3186 for (Instruction *i = bb->getExit(); i && i->op == OP_BRA; i = i->prev) {
3187 BasicBlock *bf = i->asFlow()->target.bb;
3188
3189 if (bf->getInsnCount() != 1)
3190 continue;
3191
3192 FlowInstruction *bra = i->asFlow();
3193 FlowInstruction *rep = bf->getExit()->asFlow();
3194
3195 if (!rep || rep->getPredicate())
3196 continue;
3197 if (rep->op != OP_BRA &&
3198 rep->op != OP_JOIN &&
3199 rep->op != OP_EXIT)
3200 continue;
3201
3202 // TODO: If there are multiple branches to @rep, only the first would
3203 // be replaced, so only remove them after this pass is done ?
3204 // Also, need to check all incident blocks for fall-through exits and
3205 // add the branch there.
3206 bra->op = rep->op;
3207 bra->target.bb = rep->target.bb;
3208 if (bf->cfg.incidentCount() == 1)
3209 bf->remove(rep);
3210 }
3211 }
3212
3213 bool
3214 FlatteningPass::visit(Function *fn)
3215 {
3216 gpr_unit = prog->getTarget()->getFileUnit(FILE_GPR);
3217
3218 return true;
3219 }
3220
3221 bool
3222 FlatteningPass::visit(BasicBlock *bb)
3223 {
3224 if (tryPredicateConditional(bb))
3225 return true;
3226
3227 // try to attach join to previous instruction
3228 if (prog->getTarget()->hasJoin) {
3229 Instruction *insn = bb->getExit();
3230 if (insn && insn->op == OP_JOIN && !insn->getPredicate()) {
3231 insn = insn->prev;
3232 if (insn && !insn->getPredicate() &&
3233 !insn->asFlow() &&
3234 insn->op != OP_DISCARD &&
3235 insn->op != OP_TEXBAR &&
3236 !isTextureOp(insn->op) && // probably just nve4
3237 !isSurfaceOp(insn->op) && // not confirmed
3238 insn->op != OP_LINTERP && // probably just nve4
3239 insn->op != OP_PINTERP && // probably just nve4
3240 ((insn->op != OP_LOAD && insn->op != OP_STORE && insn->op != OP_ATOM) ||
3241 (typeSizeof(insn->dType) <= 4 && !insn->src(0).isIndirect(0))) &&
3242 !insn->isNop()) {
3243 insn->join = 1;
3244 bb->remove(bb->getExit());
3245 return true;
3246 }
3247 }
3248 }
3249
3250 tryPropagateBranch(bb);
3251
3252 return true;
3253 }
3254
3255 bool
3256 FlatteningPass::tryPredicateConditional(BasicBlock *bb)
3257 {
3258 BasicBlock *bL = NULL, *bR = NULL;
3259 unsigned int nL = 0, nR = 0, limit = 12;
3260 Instruction *insn;
3261 unsigned int mask;
3262
3263 mask = bb->initiatesSimpleConditional();
3264 if (!mask)
3265 return false;
3266
3267 assert(bb->getExit());
3268 Value *pred = bb->getExit()->getPredicate();
3269 assert(pred);
3270
3271 if (isConstantCondition(pred))
3272 limit = 4;
3273
3274 Graph::EdgeIterator ei = bb->cfg.outgoing();
3275
3276 if (mask & 1) {
3277 bL = BasicBlock::get(ei.getNode());
3278 for (insn = bL->getEntry(); insn; insn = insn->next, ++nL)
3279 if (!mayPredicate(insn, pred))
3280 return false;
3281 if (nL > limit)
3282 return false; // too long, do a real branch
3283 }
3284 ei.next();
3285
3286 if (mask & 2) {
3287 bR = BasicBlock::get(ei.getNode());
3288 for (insn = bR->getEntry(); insn; insn = insn->next, ++nR)
3289 if (!mayPredicate(insn, pred))
3290 return false;
3291 if (nR > limit)
3292 return false; // too long, do a real branch
3293 }
3294
3295 if (bL)
3296 predicateInstructions(bL, pred, bb->getExit()->cc);
3297 if (bR)
3298 predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc));
3299
3300 if (bb->joinAt) {
3301 bb->remove(bb->joinAt);
3302 bb->joinAt = NULL;
3303 }
3304 removeFlow(bb->getExit()); // delete the branch/join at the fork point
3305
3306 // remove potential join operations at the end of the conditional
3307 if (prog->getTarget()->joinAnterior) {
3308 bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode());
3309 if (bb->getEntry() && bb->getEntry()->op == OP_JOIN)
3310 removeFlow(bb->getEntry());
3311 }
3312
3313 return true;
3314 }
3315
3316 // =============================================================================
3317
3318 // Fold Immediate into MAD; must be done after register allocation due to
3319 // constraint SDST == SSRC2
3320 // TODO:
3321 // Does NVC0+ have other situations where this pass makes sense?
3322 class PostRaLoadPropagation : public Pass
3323 {
3324 private:
3325 virtual bool visit(Instruction *);
3326
3327 void handleMADforNV50(Instruction *);
3328 void handleMADforNVC0(Instruction *);
3329 };
3330
3331 static bool
3332 post_ra_dead(Instruction *i)
3333 {
3334 for (int d = 0; i->defExists(d); ++d)
3335 if (i->getDef(d)->refCount())
3336 return false;
3337 return true;
3338 }
3339
3340 // Fold Immediate into MAD; must be done after register allocation due to
3341 // constraint SDST == SSRC2
3342 void
3343 PostRaLoadPropagation::handleMADforNV50(Instruction *i)
3344 {
3345 if (i->def(0).getFile() != FILE_GPR ||
3346 i->src(0).getFile() != FILE_GPR ||
3347 i->src(1).getFile() != FILE_GPR ||
3348 i->src(2).getFile() != FILE_GPR ||
3349 i->getDef(0)->reg.data.id != i->getSrc(2)->reg.data.id)
3350 return;
3351
3352 if (i->getDef(0)->reg.data.id >= 64 ||
3353 i->getSrc(0)->reg.data.id >= 64)
3354 return;
3355
3356 if (i->flagsSrc >= 0 && i->getSrc(i->flagsSrc)->reg.data.id != 0)
3357 return;
3358
3359 if (i->getPredicate())
3360 return;
3361
3362 Value *vtmp;
3363 Instruction *def = i->getSrc(1)->getInsn();
3364
3365 if (def && def->op == OP_SPLIT && typeSizeof(def->sType) == 4)
3366 def = def->getSrc(0)->getInsn();
3367 if (def && def->op == OP_MOV && def->src(0).getFile() == FILE_IMMEDIATE) {
3368 vtmp = i->getSrc(1);
3369 if (isFloatType(i->sType)) {
3370 i->setSrc(1, def->getSrc(0));
3371 } else {
3372 ImmediateValue val;
3373 // getImmediate() has side-effects on the argument so this *shouldn't*
3374 // be folded into the assert()
3375 MAYBE_UNUSED bool ret = def->src(0).getImmediate(val);
3376 assert(ret);
3377 if (i->getSrc(1)->reg.data.id & 1)
3378 val.reg.data.u32 >>= 16;
3379 val.reg.data.u32 &= 0xffff;
3380 i->setSrc(1, new_ImmediateValue(prog, val.reg.data.u32));
3381 }
3382
3383 /* There's no post-RA dead code elimination, so do it here
3384 * XXX: if we add more code-removing post-RA passes, we might
3385 * want to create a post-RA dead-code elim pass */
3386 if (post_ra_dead(vtmp->getInsn())) {
3387 Value *src = vtmp->getInsn()->getSrc(0);
3388 // Careful -- splits will have already been removed from the
3389 // functions. Don't double-delete.
3390 if (vtmp->getInsn()->bb)
3391 delete_Instruction(prog, vtmp->getInsn());
3392 if (src->getInsn() && post_ra_dead(src->getInsn()))
3393 delete_Instruction(prog, src->getInsn());
3394 }
3395 }
3396 }
3397
3398 void
3399 PostRaLoadPropagation::handleMADforNVC0(Instruction *i)
3400 {
3401 if (i->def(0).getFile() != FILE_GPR ||
3402 i->src(0).getFile() != FILE_GPR ||
3403 i->src(1).getFile() != FILE_GPR ||
3404 i->src(2).getFile() != FILE_GPR ||
3405 i->getDef(0)->reg.data.id != i->getSrc(2)->reg.data.id)
3406 return;
3407
3408 // TODO: gm107 can also do this for S32, maybe other chipsets as well
3409 if (i->dType != TYPE_F32)
3410 return;
3411
3412 if ((i->src(2).mod | Modifier(NV50_IR_MOD_NEG)) != Modifier(NV50_IR_MOD_NEG))
3413 return;
3414
3415 ImmediateValue val;
3416 int s;
3417
3418 if (i->src(0).getImmediate(val))
3419 s = 1;
3420 else if (i->src(1).getImmediate(val))
3421 s = 0;
3422 else
3423 return;
3424
3425 if ((i->src(s).mod | Modifier(NV50_IR_MOD_NEG)) != Modifier(NV50_IR_MOD_NEG))
3426 return;
3427
3428 if (s == 1)
3429 i->swapSources(0, 1);
3430
3431 Instruction *imm = i->getSrc(1)->getInsn();
3432 i->setSrc(1, imm->getSrc(0));
3433 if (post_ra_dead(imm))
3434 delete_Instruction(prog, imm);
3435 }
3436
3437 bool
3438 PostRaLoadPropagation::visit(Instruction *i)
3439 {
3440 switch (i->op) {
3441 case OP_FMA:
3442 case OP_MAD:
3443 if (prog->getTarget()->getChipset() < 0xc0)
3444 handleMADforNV50(i);
3445 else
3446 handleMADforNVC0(i);
3447 break;
3448 default:
3449 break;
3450 }
3451
3452 return true;
3453 }
3454
3455 // =============================================================================
3456
3457 // Common subexpression elimination. Stupid O^2 implementation.
3458 class LocalCSE : public Pass
3459 {
3460 private:
3461 virtual bool visit(BasicBlock *);
3462
3463 inline bool tryReplace(Instruction **, Instruction *);
3464
3465 DLList ops[OP_LAST + 1];
3466 };
3467
3468 class GlobalCSE : public Pass
3469 {
3470 private:
3471 virtual bool visit(BasicBlock *);
3472 };
3473
3474 bool
3475 Instruction::isActionEqual(const Instruction *that) const
3476 {
3477 if (this->op != that->op ||
3478 this->dType != that->dType ||
3479 this->sType != that->sType)
3480 return false;
3481 if (this->cc != that->cc)
3482 return false;
3483
3484 if (this->asTex()) {
3485 if (memcmp(&this->asTex()->tex,
3486 &that->asTex()->tex,
3487 sizeof(this->asTex()->tex)))
3488 return false;
3489 } else
3490 if (this->asCmp()) {
3491 if (this->asCmp()->setCond != that->asCmp()->setCond)
3492 return false;
3493 } else
3494 if (this->asFlow()) {
3495 return false;
3496 } else
3497 if (this->op == OP_PHI && this->bb != that->bb) {
3498 /* TODO: we could probably be a bit smarter here by following the
3499 * control flow, but honestly, it is quite painful to check */
3500 return false;
3501 } else {
3502 if (this->ipa != that->ipa ||
3503 this->lanes != that->lanes ||
3504 this->perPatch != that->perPatch)
3505 return false;
3506 if (this->postFactor != that->postFactor)
3507 return false;
3508 }
3509
3510 if (this->subOp != that->subOp ||
3511 this->saturate != that->saturate ||
3512 this->rnd != that->rnd ||
3513 this->ftz != that->ftz ||
3514 this->dnz != that->dnz ||
3515 this->cache != that->cache ||
3516 this->mask != that->mask)
3517 return false;
3518
3519 return true;
3520 }
3521
3522 bool
3523 Instruction::isResultEqual(const Instruction *that) const
3524 {
3525 unsigned int d, s;
3526
3527 // NOTE: location of discard only affects tex with liveOnly and quadops
3528 if (!this->defExists(0) && this->op != OP_DISCARD)
3529 return false;
3530
3531 if (!isActionEqual(that))
3532 return false;
3533
3534 if (this->predSrc != that->predSrc)
3535 return false;
3536
3537 for (d = 0; this->defExists(d); ++d) {
3538 if (!that->defExists(d) ||
3539 !this->getDef(d)->equals(that->getDef(d), false))
3540 return false;
3541 }
3542 if (that->defExists(d))
3543 return false;
3544
3545 for (s = 0; this->srcExists(s); ++s) {
3546 if (!that->srcExists(s))
3547 return false;
3548 if (this->src(s).mod != that->src(s).mod)
3549 return false;
3550 if (!this->getSrc(s)->equals(that->getSrc(s), true))
3551 return false;
3552 }
3553 if (that->srcExists(s))
3554 return false;
3555
3556 if (op == OP_LOAD || op == OP_VFETCH || op == OP_ATOM) {
3557 switch (src(0).getFile()) {
3558 case FILE_MEMORY_CONST:
3559 case FILE_SHADER_INPUT:
3560 return true;
3561 case FILE_SHADER_OUTPUT:
3562 return bb->getProgram()->getType() == Program::TYPE_TESSELLATION_EVAL;
3563 default:
3564 return false;
3565 }
3566 }
3567
3568 return true;
3569 }
3570
3571 // pull through common expressions from different in-blocks
3572 bool
3573 GlobalCSE::visit(BasicBlock *bb)
3574 {
3575 Instruction *phi, *next, *ik;
3576 int s;
3577
3578 // TODO: maybe do this with OP_UNION, too
3579
3580 for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) {
3581 next = phi->next;
3582 if (phi->getSrc(0)->refCount() > 1)
3583 continue;
3584 ik = phi->getSrc(0)->getInsn();
3585 if (!ik)
3586 continue; // probably a function input
3587 if (ik->defCount(0xff) > 1)
3588 continue; // too painful to check if we can really push this forward
3589 for (s = 1; phi->srcExists(s); ++s) {
3590 if (phi->getSrc(s)->refCount() > 1)
3591 break;
3592 if (!phi->getSrc(s)->getInsn() ||
3593 !phi->getSrc(s)->getInsn()->isResultEqual(ik))
3594 break;
3595 }
3596 if (!phi->srcExists(s)) {
3597 assert(ik->op != OP_PHI);
3598 Instruction *entry = bb->getEntry();
3599 ik->bb->remove(ik);
3600 if (!entry || entry->op != OP_JOIN)
3601 bb->insertHead(ik);
3602 else
3603 bb->insertAfter(entry, ik);
3604 ik->setDef(0, phi->getDef(0));
3605 delete_Instruction(prog, phi);
3606 }
3607 }
3608
3609 return true;
3610 }
3611
3612 bool
3613 LocalCSE::tryReplace(Instruction **ptr, Instruction *i)
3614 {
3615 Instruction *old = *ptr;
3616
3617 // TODO: maybe relax this later (causes trouble with OP_UNION)
3618 if (i->isPredicated())
3619 return false;
3620
3621 if (!old->isResultEqual(i))
3622 return false;
3623
3624 for (int d = 0; old->defExists(d); ++d)
3625 old->def(d).replace(i->getDef(d), false);
3626 delete_Instruction(prog, old);
3627 *ptr = NULL;
3628 return true;
3629 }
3630
3631 bool
3632 LocalCSE::visit(BasicBlock *bb)
3633 {
3634 unsigned int replaced;
3635
3636 do {
3637 Instruction *ir, *next;
3638
3639 replaced = 0;
3640
3641 // will need to know the order of instructions
3642 int serial = 0;
3643 for (ir = bb->getFirst(); ir; ir = ir->next)
3644 ir->serial = serial++;
3645
3646 for (ir = bb->getFirst(); ir; ir = next) {
3647 int s;
3648 Value *src = NULL;
3649
3650 next = ir->next;
3651
3652 if (ir->fixed) {
3653 ops[ir->op].insert(ir);
3654 continue;
3655 }
3656
3657 for (s = 0; ir->srcExists(s); ++s)
3658 if (ir->getSrc(s)->asLValue())
3659 if (!src || ir->getSrc(s)->refCount() < src->refCount())
3660 src = ir->getSrc(s);
3661
3662 if (src) {
3663 for (Value::UseIterator it = src->uses.begin();
3664 it != src->uses.end(); ++it) {
3665 Instruction *ik = (*it)->getInsn();
3666 if (ik && ik->bb == ir->bb && ik->serial < ir->serial)
3667 if (tryReplace(&ir, ik))
3668 break;
3669 }
3670 } else {
3671 DLLIST_FOR_EACH(&ops[ir->op], iter)
3672 {
3673 Instruction *ik = reinterpret_cast<Instruction *>(iter.get());
3674 if (tryReplace(&ir, ik))
3675 break;
3676 }
3677 }
3678
3679 if (ir)
3680 ops[ir->op].insert(ir);
3681 else
3682 ++replaced;
3683 }
3684 for (unsigned int i = 0; i <= OP_LAST; ++i)
3685 ops[i].clear();
3686
3687 } while (replaced);
3688
3689 return true;
3690 }
3691
3692 // =============================================================================
3693
3694 // Remove computations of unused values.
3695 class DeadCodeElim : public Pass
3696 {
3697 public:
3698 bool buryAll(Program *);
3699
3700 private:
3701 virtual bool visit(BasicBlock *);
3702
3703 void checkSplitLoad(Instruction *ld); // for partially dead loads
3704
3705 unsigned int deadCount;
3706 };
3707
3708 bool
3709 DeadCodeElim::buryAll(Program *prog)
3710 {
3711 do {
3712 deadCount = 0;
3713 if (!this->run(prog, false, false))
3714 return false;
3715 } while (deadCount);
3716
3717 return true;
3718 }
3719
3720 bool
3721 DeadCodeElim::visit(BasicBlock *bb)
3722 {
3723 Instruction *prev;
3724
3725 for (Instruction *i = bb->getExit(); i; i = prev) {
3726 prev = i->prev;
3727 if (i->isDead()) {
3728 ++deadCount;
3729 delete_Instruction(prog, i);
3730 } else
3731 if (i->defExists(1) &&
3732 i->subOp == 0 &&
3733 (i->op == OP_VFETCH || i->op == OP_LOAD)) {
3734 checkSplitLoad(i);
3735 } else
3736 if (i->defExists(0) && !i->getDef(0)->refCount()) {
3737 if (i->op == OP_ATOM ||
3738 i->op == OP_SUREDP ||
3739 i->op == OP_SUREDB) {
3740 i->setDef(0, NULL);
3741 if (i->op == OP_ATOM && i->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
3742 i->cache = CACHE_CV;
3743 i->op = OP_STORE;
3744 i->subOp = 0;
3745 }
3746 } else if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LOAD_LOCKED) {
3747 i->setDef(0, i->getDef(1));
3748 i->setDef(1, NULL);
3749 }
3750 }
3751 }
3752 return true;
3753 }
3754
3755 // Each load can go into up to 4 destinations, any of which might potentially
3756 // be dead (i.e. a hole). These can always be split into 2 loads, independent
3757 // of where the holes are. We find the first contiguous region, put it into
3758 // the first load, and then put the second contiguous region into the second
3759 // load. There can be at most 2 contiguous regions.
3760 //
3761 // Note that there are some restrictions, for example it's not possible to do
3762 // a 64-bit load that's not 64-bit aligned, so such a load has to be split
3763 // up. Also hardware doesn't support 96-bit loads, so those also have to be
3764 // split into a 64-bit and 32-bit load.
3765 void
3766 DeadCodeElim::checkSplitLoad(Instruction *ld1)
3767 {
3768 Instruction *ld2 = NULL; // can get at most 2 loads
3769 Value *def1[4];
3770 Value *def2[4];
3771 int32_t addr1, addr2;
3772 int32_t size1, size2;
3773 int d, n1, n2;
3774 uint32_t mask = 0xffffffff;
3775
3776 for (d = 0; ld1->defExists(d); ++d)
3777 if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0)
3778 mask &= ~(1 << d);
3779 if (mask == 0xffffffff)
3780 return;
3781
3782 addr1 = ld1->getSrc(0)->reg.data.offset;
3783 n1 = n2 = 0;
3784 size1 = size2 = 0;
3785
3786 // Compute address/width for first load
3787 for (d = 0; ld1->defExists(d); ++d) {
3788 if (mask & (1 << d)) {
3789 if (size1 && (addr1 & 0x7))
3790 break;
3791 def1[n1] = ld1->getDef(d);
3792 size1 += def1[n1++]->reg.size;
3793 } else
3794 if (!n1) {
3795 addr1 += ld1->getDef(d)->reg.size;
3796 } else {
3797 break;
3798 }
3799 }
3800
3801 // Scale back the size of the first load until it can be loaded. This
3802 // typically happens for TYPE_B96 loads.
3803 while (n1 &&
3804 !prog->getTarget()->isAccessSupported(ld1->getSrc(0)->reg.file,
3805 typeOfSize(size1))) {
3806 size1 -= def1[--n1]->reg.size;
3807 d--;
3808 }
3809
3810 // Compute address/width for second load
3811 for (addr2 = addr1 + size1; ld1->defExists(d); ++d) {
3812 if (mask & (1 << d)) {
3813 assert(!size2 || !(addr2 & 0x7));
3814 def2[n2] = ld1->getDef(d);
3815 size2 += def2[n2++]->reg.size;
3816 } else if (!n2) {
3817 assert(!n2);
3818 addr2 += ld1->getDef(d)->reg.size;
3819 } else {
3820 break;
3821 }
3822 }
3823
3824 // Make sure that we've processed all the values
3825 for (; ld1->defExists(d); ++d)
3826 assert(!(mask & (1 << d)));
3827
3828 updateLdStOffset(ld1, addr1, func);
3829 ld1->setType(typeOfSize(size1));
3830 for (d = 0; d < 4; ++d)
3831 ld1->setDef(d, (d < n1) ? def1[d] : NULL);
3832
3833 if (!n2)
3834 return;
3835
3836 ld2 = cloneShallow(func, ld1);
3837 updateLdStOffset(ld2, addr2, func);
3838 ld2->setType(typeOfSize(size2));
3839 for (d = 0; d < 4; ++d)
3840 ld2->setDef(d, (d < n2) ? def2[d] : NULL);
3841
3842 ld1->bb->insertAfter(ld1, ld2);
3843 }
3844
3845 // =============================================================================
3846
3847 #define RUN_PASS(l, n, f) \
3848 if (level >= (l)) { \
3849 if (dbgFlags & NV50_IR_DEBUG_VERBOSE) \
3850 INFO("PEEPHOLE: %s\n", #n); \
3851 n pass; \
3852 if (!pass.f(this)) \
3853 return false; \
3854 }
3855
3856 bool
3857 Program::optimizeSSA(int level)
3858 {
3859 RUN_PASS(1, DeadCodeElim, buryAll);
3860 RUN_PASS(1, CopyPropagation, run);
3861 RUN_PASS(1, MergeSplits, run);
3862 RUN_PASS(2, GlobalCSE, run);
3863 RUN_PASS(1, LocalCSE, run);
3864 RUN_PASS(2, AlgebraicOpt, run);
3865 RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks
3866 RUN_PASS(1, ConstantFolding, foldAll);
3867 RUN_PASS(1, Split64BitOpPreRA, run);
3868 RUN_PASS(2, LateAlgebraicOpt, run);
3869 RUN_PASS(1, LoadPropagation, run);
3870 RUN_PASS(1, IndirectPropagation, run);
3871 RUN_PASS(2, MemoryOpt, run);
3872 RUN_PASS(2, LocalCSE, run);
3873 RUN_PASS(0, DeadCodeElim, buryAll);
3874
3875 return true;
3876 }
3877
3878 bool
3879 Program::optimizePostRA(int level)
3880 {
3881 RUN_PASS(2, FlatteningPass, run);
3882 RUN_PASS(2, PostRaLoadPropagation, run);
3883
3884 return true;
3885 }
3886
3887 }