projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nvc0/ir: insn can never be null
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_lowering_nvc0.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 #include "codegen/nv50_ir_target_nvc0.h"
27 #include "codegen/nv50_ir_lowering_nvc0.h"
28
29 #include <limits>
30
31 namespace nv50_ir {
32
33 #define QOP_ADD 0
34 #define QOP_SUBR 1
35 #define QOP_SUB 2
36 #define QOP_MOV2 3
37
38 // UL UR LL LR
39 #define QUADOP(q, r, s, t) \
40 ((QOP_##q << 6) | (QOP_##r << 4) | \
41 (QOP_##s << 2) | (QOP_##t << 0))
42
43 void
44 NVC0LegalizeSSA::handleDIV(Instruction *i)
45 {
46 FlowInstruction *call;
47 int builtin;
48 Value *def[2];
49
50 bld.setPosition(i, false);
51 def[0] = bld.mkMovToReg(0, i->getSrc(0))->getDef(0);
52 def[1] = bld.mkMovToReg(1, i->getSrc(1))->getDef(0);
53 switch (i->dType) {
54 case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;
55 case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;
56 default:
57 return;
58 }
59 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
60 bld.mkMov(i->getDef(0), def[(i->op == OP_DIV) ? 0 : 1]);
61 bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);
62 bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);
63
64 call->fixed = 1;
65 call->absolute = call->builtin = 1;
66 call->target.builtin = builtin;
67 delete_Instruction(prog, i);
68 }
69
70 void
71 NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)
72 {
73 // TODO
74 }
75
76 bool
77 NVC0LegalizeSSA::visit(Function *fn)
78 {
79 bld.setProgram(fn->getProgram());
80 return true;
81 }
82
83 bool
84 NVC0LegalizeSSA::visit(BasicBlock *bb)
85 {
86 Instruction *next;
87 for (Instruction *i = bb->getEntry(); i; i = next) {
88 next = i->next;
89 if (i->dType == TYPE_F32)
90 continue;
91 switch (i->op) {
92 case OP_DIV:
93 case OP_MOD:
94 handleDIV(i);
95 break;
96 case OP_RCP:
97 case OP_RSQ:
98 if (i->dType == TYPE_F64)
99 handleRCPRSQ(i);
100 break;
101 default:
102 break;
103 }
104 }
105 return true;
106 }
107
108 NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)
109 : rZero(NULL),
110 carry(NULL),
111 needTexBar(prog->getTarget()->getChipset() >= 0xe0)
112 {
113 }
114
115 bool
116 NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,
117 const Instruction *early) const
118 {
119 if (early->bb == later->bb)
120 return early->serial < later->serial;
121 return later->bb->dominatedBy(early->bb);
122 }
123
124 void
125 NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses,
126 Instruction *usei, const Instruction *insn)
127 {
128 bool add = true;
129 for (std::list<TexUse>::iterator it = uses.begin();
130 it != uses.end();) {
131 if (insnDominatedBy(usei, it->insn)) {
132 add = false;
133 break;
134 }
135 if (insnDominatedBy(it->insn, usei))
136 it = uses.erase(it);
137 else
138 ++it;
139 }
140 if (add)
141 uses.push_back(TexUse(usei, insn));
142 }
143
144 void
145 NVC0LegalizePostRA::findOverwritingDefs(const Instruction *texi,
146 Instruction *insn,
147 const BasicBlock *term,
148 std::list<TexUse> &uses)
149 {
150 while (insn->op == OP_MOV && insn->getDef(0)->equals(insn->getSrc(0)))
151 insn = insn->getSrc(0)->getUniqueInsn();
152
153 if (!insn->bb->reachableBy(texi->bb, term))
154 return;
155
156 switch (insn->op) {
157 /* Values not connected to the tex's definition through any of these should
158 * not be conflicting.
159 */
160 case OP_SPLIT:
161 case OP_MERGE:
162 case OP_PHI:
163 case OP_UNION:
164 /* recurse again */
165 for (int s = 0; insn->srcExists(s); ++s)
166 findOverwritingDefs(texi, insn->getSrc(s)->getUniqueInsn(), term,
167 uses);
168 break;
169 default:
170 // if (!isTextureOp(insn->op)) // TODO: are TEXes always ordered ?
171 addTexUse(uses, insn, texi);
172 break;
173 }
174 }
175
176 void
177 NVC0LegalizePostRA::findFirstUses(
178 const Instruction *texi,
179 const Instruction *insn,
180 std::list<TexUse> &uses,
181 std::tr1::unordered_set<const Instruction *>& visited)
182 {
183 for (int d = 0; insn->defExists(d); ++d) {
184 Value *v = insn->getDef(d);
185 for (Value::UseIterator u = v->uses.begin(); u != v->uses.end(); ++u) {
186 Instruction *usei = (*u)->getInsn();
187
188 // NOTE: In case of a loop that overwrites a value but never uses
189 // it, it can happen that we have a cycle of uses that consists only
190 // of phis and no-op moves and will thus cause an infinite loop here
191 // since these are not considered actual uses.
192 // The most obvious (and perhaps the only) way to prevent this is to
193 // remember which instructions we've already visited.
194
195 if (visited.find(usei) != visited.end())
196 continue;
197
198 visited.insert(usei);
199
200 if (usei->op == OP_PHI || usei->op == OP_UNION) {
201 // need a barrier before WAW cases
202 for (int s = 0; usei->srcExists(s); ++s) {
203 Instruction *defi = usei->getSrc(s)->getUniqueInsn();
204 if (defi && &usei->src(s) != *u)
205 findOverwritingDefs(texi, defi, usei->bb, uses);
206 }
207 }
208
209 if (usei->op == OP_SPLIT ||
210 usei->op == OP_MERGE ||
211 usei->op == OP_PHI ||
212 usei->op == OP_UNION) {
213 // these uses don't manifest in the machine code
214 findFirstUses(texi, usei, uses, visited);
215 } else
216 if (usei->op == OP_MOV && usei->getDef(0)->equals(usei->getSrc(0)) &&
217 usei->subOp != NV50_IR_SUBOP_MOV_FINAL) {
218 findFirstUses(texi, usei, uses, visited);
219 } else {
220 addTexUse(uses, usei, insn);
221 }
222 }
223 }
224 }
225
226 // Texture barriers:
227 // This pass is a bit long and ugly and can probably be optimized.
228 //
229 // 1. obtain a list of TEXes and their outputs' first use(s)
230 // 2. calculate the barrier level of each first use (minimal number of TEXes,
231 // over all paths, between the TEX and the use in question)
232 // 3. for each barrier, if all paths from the source TEX to that barrier
233 // contain a barrier of lesser level, it can be culled
234 bool
235 NVC0LegalizePostRA::insertTextureBarriers(Function *fn)
236 {
237 std::list<TexUse> *uses;
238 std::vector<Instruction *> texes;
239 std::vector<int> bbFirstTex;
240 std::vector<int> bbFirstUse;
241 std::vector<int> texCounts;
242 std::vector<TexUse> useVec;
243 ArrayList insns;
244
245 fn->orderInstructions(insns);
246
247 texCounts.resize(fn->allBBlocks.getSize(), 0);
248 bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());
249 bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());
250
251 // tag BB CFG nodes by their id for later
252 for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {
253 BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get());
254 if (bb)
255 bb->cfg.tag = bb->getId();
256 }
257
258 // gather the first uses for each TEX
259 for (int i = 0; i < insns.getSize(); ++i) {
260 Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i));
261 if (isTextureOp(tex->op)) {
262 texes.push_back(tex);
263 if (!texCounts.at(tex->bb->getId()))
264 bbFirstTex[tex->bb->getId()] = texes.size() - 1;
265 texCounts[tex->bb->getId()]++;
266 }
267 }
268 insns.clear();
269 if (texes.empty())
270 return false;
271 uses = new std::list<TexUse>[texes.size()];
272 if (!uses)
273 return false;
274 for (size_t i = 0; i < texes.size(); ++i) {
275 std::tr1::unordered_set<const Instruction *> visited;
276 findFirstUses(texes[i], texes[i], uses[i], visited);
277 }
278
279 // determine the barrier level at each use
280 for (size_t i = 0; i < texes.size(); ++i) {
281 for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end();
282 ++u) {
283 BasicBlock *tb = texes[i]->bb;
284 BasicBlock *ub = u->insn->bb;
285 if (tb == ub) {
286 u->level = 0;
287 for (size_t j = i + 1; j < texes.size() &&
288 texes[j]->bb == tb && texes[j]->serial < u->insn->serial;
289 ++j)
290 u->level++;
291 } else {
292 u->level = fn->cfg.findLightestPathWeight(&tb->cfg,
293 &ub->cfg, texCounts);
294 if (u->level < 0) {
295 WARN("Failed to find path TEX -> TEXBAR\n");
296 u->level = 0;
297 continue;
298 }
299 // this counted all TEXes in the origin block, correct that
300 u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;
301 // and did not count the TEXes in the destination block, add those
302 for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&
303 texes[j]->bb == ub && texes[j]->serial < u->insn->serial;
304 ++j)
305 u->level++;
306 }
307 assert(u->level >= 0);
308 useVec.push_back(*u);
309 }
310 }
311 delete[] uses;
312 uses = NULL;
313
314 // insert the barriers
315 for (size_t i = 0; i < useVec.size(); ++i) {
316 Instruction *prev = useVec[i].insn->prev;
317 if (useVec[i].level < 0)
318 continue;
319 if (prev && prev->op == OP_TEXBAR) {
320 if (prev->subOp > useVec[i].level)
321 prev->subOp = useVec[i].level;
322 prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));
323 } else {
324 Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);
325 bar->fixed = 1;
326 bar->subOp = useVec[i].level;
327 // make use explicit to ease latency calculation
328 bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));
329 useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);
330 }
331 }
332
333 if (fn->getProgram()->optLevel < 3) {
334 if (uses)
335 delete[] uses;
336 return true;
337 }
338
339 std::vector<Limits> limitT, limitB, limitS; // entry, exit, single
340
341 limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));
342 limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));
343 limitS.resize(fn->allBBlocks.getSize());
344
345 // cull unneeded barriers (should do that earlier, but for simplicity)
346 IteratorRef bi = fn->cfg.iteratorCFG();
347 // first calculate min/max outstanding TEXes for each BB
348 for (bi->reset(); !bi->end(); bi->next()) {
349 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
350 BasicBlock *bb = BasicBlock::get(n);
351 int min = 0;
352 int max = std::numeric_limits<int>::max();
353 for (Instruction *i = bb->getFirst(); i; i = i->next) {
354 if (isTextureOp(i->op)) {
355 min++;
356 if (max < std::numeric_limits<int>::max())
357 max++;
358 } else
359 if (i->op == OP_TEXBAR) {
360 min = MIN2(min, i->subOp);
361 max = MIN2(max, i->subOp);
362 }
363 }
364 // limits when looking at an isolated block
365 limitS[bb->getId()].min = min;
366 limitS[bb->getId()].max = max;
367 }
368 // propagate the min/max values
369 for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {
370 for (bi->reset(); !bi->end(); bi->next()) {
371 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
372 BasicBlock *bb = BasicBlock::get(n);
373 const int bbId = bb->getId();
374 for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {
375 BasicBlock *in = BasicBlock::get(ei.getNode());
376 const int inId = in->getId();
377 limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);
378 limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);
379 }
380 // I just hope this is correct ...
381 if (limitS[bbId].max == std::numeric_limits<int>::max()) {
382 // no barrier
383 limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;
384 limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;
385 } else {
386 // block contained a barrier
387 limitB[bbId].min = MIN2(limitS[bbId].max,
388 limitT[bbId].min + limitS[bbId].min);
389 limitB[bbId].max = MIN2(limitS[bbId].max,
390 limitT[bbId].max + limitS[bbId].min);
391 }
392 }
393 }
394 // finally delete unnecessary barriers
395 for (bi->reset(); !bi->end(); bi->next()) {
396 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
397 BasicBlock *bb = BasicBlock::get(n);
398 Instruction *prev = NULL;
399 Instruction *next;
400 int max = limitT[bb->getId()].max;
401 for (Instruction *i = bb->getFirst(); i; i = next) {
402 next = i->next;
403 if (i->op == OP_TEXBAR) {
404 if (i->subOp >= max) {
405 delete_Instruction(prog, i);
406 i = NULL;
407 } else {
408 max = i->subOp;
409 if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {
410 delete_Instruction(prog, prev);
411 prev = NULL;
412 }
413 }
414 } else
415 if (isTextureOp(i->op)) {
416 max++;
417 }
418 if (i && !i->isNop())
419 prev = i;
420 }
421 }
422 if (uses)
423 delete[] uses;
424 return true;
425 }
426
427 bool
428 NVC0LegalizePostRA::visit(Function *fn)
429 {
430 if (needTexBar)
431 insertTextureBarriers(fn);
432
433 rZero = new_LValue(fn, FILE_GPR);
434 carry = new_LValue(fn, FILE_FLAGS);
435
436 rZero->reg.data.id = prog->getTarget()->getFileSize(FILE_GPR);
437 carry->reg.data.id = 0;
438
439 return true;
440 }
441
442 void
443 NVC0LegalizePostRA::replaceZero(Instruction *i)
444 {
445 for (int s = 0; i->srcExists(s); ++s) {
446 if (s == 2 && i->op == OP_SUCLAMP)
447 continue;
448 ImmediateValue *imm = i->getSrc(s)->asImm();
449 if (imm && imm->reg.data.u64 == 0)
450 i->setSrc(s, rZero);
451 }
452 }
453
454 // replace CONT with BRA for single unconditional continue
455 bool
456 NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)
457 {
458 if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)
459 return false;
460 Graph::EdgeIterator ei = bb->cfg.incident();
461 if (ei.getType() != Graph::Edge::BACK)
462 ei.next();
463 if (ei.getType() != Graph::Edge::BACK)
464 return false;
465 BasicBlock *contBB = BasicBlock::get(ei.getNode());
466
467 if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||
468 contBB->getExit()->getPredicate())
469 return false;
470 contBB->getExit()->op = OP_BRA;
471 bb->remove(bb->getEntry()); // delete PRECONT
472
473 ei.next();
474 assert(ei.end() || ei.getType() != Graph::Edge::BACK);
475 return true;
476 }
477
478 // replace branches to join blocks with join ops
479 void
480 NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)
481 {
482 if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)
483 return;
484 for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
485 BasicBlock *in = BasicBlock::get(ei.getNode());
486 Instruction *exit = in->getExit();
487 if (!exit) {
488 in->insertTail(new FlowInstruction(func, OP_JOIN, bb));
489 // there should always be a terminator instruction
490 WARN("inserted missing terminator in BB:%i\n", in->getId());
491 } else
492 if (exit->op == OP_BRA) {
493 exit->op = OP_JOIN;
494 exit->asFlow()->limit = 1; // must-not-propagate marker
495 }
496 }
497 bb->remove(bb->getEntry());
498 }
499
500 bool
501 NVC0LegalizePostRA::visit(BasicBlock *bb)
502 {
503 Instruction *i, *next;
504
505 // remove pseudo operations and non-fixed no-ops, split 64 bit operations
506 for (i = bb->getFirst(); i; i = next) {
507 next = i->next;
508 if (i->op == OP_EMIT || i->op == OP_RESTART) {
509 if (!i->getDef(0)->refCount())
510 i->setDef(0, NULL);
511 if (i->src(0).getFile() == FILE_IMMEDIATE)
512 i->setSrc(0, rZero); // initial value must be 0
513 replaceZero(i);
514 } else
515 if (i->isNop()) {
516 bb->remove(i);
517 } else {
518 // TODO: Move this to before register allocation for operations that
519 // need the $c register !
520 if (typeSizeof(i->dType) == 8) {
521 Instruction *hi;
522 hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);
523 if (hi)
524 next = hi;
525 }
526
527 if (i->op != OP_MOV && i->op != OP_PFETCH)
528 replaceZero(i);
529 }
530 }
531 if (!bb->getEntry())
532 return true;
533
534 if (!tryReplaceContWithBra(bb))
535 propagateJoin(bb);
536
537 return true;
538 }
539
540 NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())
541 {
542 bld.setProgram(prog);
543 gMemBase = NULL;
544 }
545
546 bool
547 NVC0LoweringPass::visit(Function *fn)
548 {
549 if (prog->getType() == Program::TYPE_GEOMETRY) {
550 assert(!strncmp(fn->getName(), "MAIN", 4));
551 // TODO: when we generate actual functions pass this value along somehow
552 bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);
553 gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();
554 if (fn->cfgExit) {
555 bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);
556 bld.mkMovToReg(0, gpEmitAddress);
557 }
558 }
559 return true;
560 }
561
562 bool
563 NVC0LoweringPass::visit(BasicBlock *bb)
564 {
565 return true;
566 }
567
568 inline Value *
569 NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)
570 {
571 uint8_t b = prog->driver->io.resInfoCBSlot;
572 uint32_t off = prog->driver->io.texBindBase + slot * 4;
573 return bld.
574 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
575 }
576
577 // move array source to first slot, convert to u16, add indirections
578 bool
579 NVC0LoweringPass::handleTEX(TexInstruction *i)
580 {
581 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
582 const int arg = i->tex.target.getArgCount();
583 const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);
584 const int chipset = prog->getTarget()->getChipset();
585
586 // Arguments to the TEX instruction are a little insane. Even though the
587 // encoding is identical between SM20 and SM30, the arguments mean
588 // different things between Fermi and Kepler+. A lot of arguments are
589 // optional based on flags passed to the instruction. This summarizes the
590 // order of things.
591 //
592 // Fermi:
593 // array/indirect
594 // coords
595 // sample
596 // lod bias
597 // depth compare
598 // offsets:
599 // - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)
600 // - other: 4 bits each, single reg
601 //
602 // Kepler+:
603 // indirect handle
604 // array (+ offsets for txd in upper 16 bits)
605 // coords
606 // sample
607 // lod bias
608 // depth compare
609 // offsets (same as fermi, except txd which takes it with array)
610
611 if (chipset >= NVISA_GK104_CHIPSET) {
612 if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
613 // XXX this ignores tsc, and assumes a 1:1 mapping
614 assert(i->tex.rIndirectSrc >= 0);
615 Value *hnd = loadTexHandle(
616 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
617 i->getIndirectR(), bld.mkImm(2)),
618 i->tex.r);
619 i->tex.r = 0xff;
620 i->tex.s = 0x1f;
621 i->setIndirectR(hnd);
622 i->setIndirectS(NULL);
623 } else if (i->tex.r == i->tex.s) {
624 i->tex.r += prog->driver->io.texBindBase / 4;
625 i->tex.s = 0; // only a single cX[] value possible here
626 } else {
627 Value *hnd = bld.getScratch();
628 Value *rHnd = loadTexHandle(NULL, i->tex.r);
629 Value *sHnd = loadTexHandle(NULL, i->tex.s);
630
631 bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);
632
633 i->tex.r = 0; // not used for indirect tex
634 i->tex.s = 0;
635 i->setIndirectR(hnd);
636 }
637 if (i->tex.target.isArray()) {
638 LValue *layer = new_LValue(func, FILE_GPR);
639 Value *src = i->getSrc(lyr);
640 const int sat = (i->op == OP_TXF) ? 1 : 0;
641 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
642 bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;
643 for (int s = dim; s >= 1; --s)
644 i->setSrc(s, i->getSrc(s - 1));
645 i->setSrc(0, layer);
646 }
647 // Move the indirect reference to the first place
648 if (i->tex.rIndirectSrc >= 0) {
649 Value *hnd = i->getIndirectR();
650
651 i->setIndirectR(NULL);
652 i->moveSources(0, 1);
653 i->setSrc(0, hnd);
654 i->tex.rIndirectSrc = 0;
655 i->tex.sIndirectSrc = -1;
656 }
657 } else
658 // (nvc0) generate and move the tsc/tic/array source to the front
659 if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
660 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
661
662 Value *ticRel = i->getIndirectR();
663 Value *tscRel = i->getIndirectS();
664
665 if (ticRel) {
666 i->setSrc(i->tex.rIndirectSrc, NULL);
667 if (i->tex.r)
668 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
669 ticRel, bld.mkImm(i->tex.r));
670 }
671 if (tscRel) {
672 i->setSrc(i->tex.sIndirectSrc, NULL);
673 if (i->tex.s)
674 tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
675 tscRel, bld.mkImm(i->tex.s));
676 }
677
678 Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;
679 for (int s = dim; s >= 1; --s)
680 i->setSrc(s, i->getSrc(s - 1));
681 i->setSrc(0, arrayIndex);
682
683 if (arrayIndex) {
684 int sat = (i->op == OP_TXF) ? 1 : 0;
685 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
686 bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;
687 } else {
688 bld.loadImm(src, 0);
689 }
690
691 if (ticRel)
692 bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);
693 if (tscRel)
694 bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);
695
696 i->setSrc(0, src);
697 }
698
699 // For nvc0, the sample id has to be in the second operand, as the offset
700 // does. Right now we don't know how to pass both in, and this case can't
701 // happen with OpenGL. On nve0, the sample id is part of the texture
702 // coordinate argument.
703 assert(chipset >= NVISA_GK104_CHIPSET ||
704 !i->tex.useOffsets || !i->tex.target.isMS());
705
706 // offset is between lod and dc
707 if (i->tex.useOffsets) {
708 int n, c;
709 int s = i->srcCount(0xff, true);
710 if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {
711 if (i->tex.target.isShadow())
712 s--;
713 if (i->srcExists(s)) // move potential predicate out of the way
714 i->moveSources(s, 1);
715 if (i->tex.useOffsets == 4 && i->srcExists(s + 1))
716 i->moveSources(s + 1, 1);
717 }
718 if (i->op == OP_TXG) {
719 // Either there is 1 offset, which goes into the 2 low bytes of the
720 // first source, or there are 4 offsets, which go into 2 sources (8
721 // values, 1 byte each).
722 Value *offs[2] = {NULL, NULL};
723 for (n = 0; n < i->tex.useOffsets; n++) {
724 for (c = 0; c < 2; ++c) {
725 if ((n % 2) == 0 && c == 0)
726 offs[n / 2] = i->offset[n][c].get();
727 else
728 bld.mkOp3(OP_INSBF, TYPE_U32,
729 offs[n / 2],
730 i->offset[n][c].get(),
731 bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),
732 offs[n / 2]);
733 }
734 }
735 i->setSrc(s, offs[0]);
736 if (offs[1])
737 i->setSrc(s + 1, offs[1]);
738 } else {
739 unsigned imm = 0;
740 assert(i->tex.useOffsets == 1);
741 for (c = 0; c < 3; ++c) {
742 ImmediateValue val;
743 assert(i->offset[0][c].getImmediate(val));
744 imm |= (val.reg.data.u32 & 0xf) << (c * 4);
745 }
746 if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {
747 // The offset goes into the upper 16 bits of the array index. So
748 // create it if it's not already there, and INSBF it if it already
749 // is.
750 s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;
751 if (i->tex.target.isArray()) {
752 bld.mkOp3(OP_INSBF, TYPE_U32, i->getSrc(0),
753 bld.loadImm(NULL, imm), bld.mkImm(0xc10),
754 i->getSrc(s));
755 } else {
756 i->moveSources(s, 1);
757 i->setSrc(s, bld.loadImm(NULL, imm << 16));
758 }
759 } else {
760 i->setSrc(s, bld.loadImm(NULL, imm));
761 }
762 }
763 }
764
765 if (chipset >= NVISA_GK104_CHIPSET) {
766 //
767 // If TEX requires more than 4 sources, the 2nd register tuple must be
768 // aligned to 4, even if it consists of just a single 4-byte register.
769 //
770 // XXX HACK: We insert 0 sources to avoid the 5 or 6 regs case.
771 //
772 int s = i->srcCount(0xff, true);
773 if (s > 4 && s < 7) {
774 if (i->srcExists(s)) // move potential predicate out of the way
775 i->moveSources(s, 7 - s);
776 while (s < 7)
777 i->setSrc(s++, bld.loadImm(NULL, 0));
778 }
779 }
780
781 return true;
782 }
783
784 bool
785 NVC0LoweringPass::handleManualTXD(TexInstruction *i)
786 {
787 static const uint8_t qOps[4][2] =
788 {
789 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0
790 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1
791 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2
792 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3
793 };
794 Value *def[4][4];
795 Value *crd[3];
796 Instruction *tex;
797 Value *zero = bld.loadImm(bld.getSSA(), 0);
798 int l, c;
799 const int dim = i->tex.target.getDim();
800 const int array = i->tex.target.isArray();
801
802 i->op = OP_TEX; // no need to clone dPdx/dPdy later
803
804 for (c = 0; c < dim; ++c)
805 crd[c] = bld.getScratch();
806
807 bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
808 for (l = 0; l < 4; ++l) {
809 // mov coordinates from lane l to all lanes
810 for (c = 0; c < dim; ++c)
811 bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);
812 // add dPdx from lane l to lanes dx
813 for (c = 0; c < dim; ++c)
814 bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);
815 // add dPdy from lane l to lanes dy
816 for (c = 0; c < dim; ++c)
817 bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);
818 // texture
819 bld.insert(tex = cloneForward(func, i));
820 for (c = 0; c < dim; ++c)
821 tex->setSrc(c + array, crd[c]);
822 // save results
823 for (c = 0; i->defExists(c); ++c) {
824 Instruction *mov;
825 def[c][l] = bld.getSSA();
826 mov = bld.mkMov(def[c][l], tex->getDef(c));
827 mov->fixed = 1;
828 mov->lanes = 1 << l;
829 }
830 }
831 bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
832
833 for (c = 0; i->defExists(c); ++c) {
834 Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
835 for (l = 0; l < 4; ++l)
836 u->setSrc(l, def[c][l]);
837 }
838
839 i->bb->remove(i);
840 return true;
841 }
842
843 bool
844 NVC0LoweringPass::handleTXD(TexInstruction *txd)
845 {
846 int dim = txd->tex.target.getDim();
847 unsigned arg = txd->tex.target.getArgCount();
848 unsigned expected_args = arg;
849 const int chipset = prog->getTarget()->getChipset();
850
851 if (chipset >= NVISA_GK104_CHIPSET) {
852 if (!txd->tex.target.isArray() && txd->tex.useOffsets)
853 expected_args++;
854 if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)
855 expected_args++;
856 } else {
857 if (txd->tex.useOffsets)
858 expected_args++;
859 if (!txd->tex.target.isArray() && (
860 txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))
861 expected_args++;
862 }
863
864 if (expected_args > 4 ||
865 dim > 2 ||
866 txd->tex.target.isShadow() ||
867 txd->tex.target.isCube())
868 txd->op = OP_TEX;
869
870 handleTEX(txd);
871 while (txd->srcExists(arg))
872 ++arg;
873
874 txd->tex.derivAll = true;
875 if (txd->op == OP_TEX)
876 return handleManualTXD(txd);
877
878 assert(arg == expected_args);
879 for (int c = 0; c < dim; ++c) {
880 txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);
881 txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);
882 txd->dPdx[c].set(NULL);
883 txd->dPdy[c].set(NULL);
884 }
885 return true;
886 }
887
888 bool
889 NVC0LoweringPass::handleTXQ(TexInstruction *txq)
890 {
891 // TODO: indirect resource/sampler index
892 return true;
893 }
894
895 bool
896 NVC0LoweringPass::handleTXLQ(TexInstruction *i)
897 {
898 /* The outputs are inverted compared to what the TGSI instruction
899 * expects. Take that into account in the mask.
900 */
901 assert((i->tex.mask & ~3) == 0);
902 if (i->tex.mask == 1)
903 i->tex.mask = 2;
904 else if (i->tex.mask == 2)
905 i->tex.mask = 1;
906 handleTEX(i);
907 bld.setPosition(i, true);
908
909 /* The returned values are not quite what we want:
910 * (a) convert from s16/u16 to f32
911 * (b) multiply by 1/256
912 */
913 for (int def = 0; def < 2; ++def) {
914 if (!i->defExists(def))
915 continue;
916 enum DataType type = TYPE_S16;
917 if (i->tex.mask == 2 || def > 0)
918 type = TYPE_U16;
919 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));
920 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),
921 i->getDef(def), bld.loadImm(NULL, 1.0f / 256));
922 }
923 if (i->tex.mask == 3) {
924 LValue *t = new_LValue(func, FILE_GPR);
925 bld.mkMov(t, i->getDef(0));
926 bld.mkMov(i->getDef(0), i->getDef(1));
927 bld.mkMov(i->getDef(1), t);
928 }
929 return true;
930 }
931
932
933 bool
934 NVC0LoweringPass::handleATOM(Instruction *atom)
935 {
936 SVSemantic sv;
937
938 switch (atom->src(0).getFile()) {
939 case FILE_MEMORY_LOCAL:
940 sv = SV_LBASE;
941 break;
942 case FILE_MEMORY_SHARED:
943 sv = SV_SBASE;
944 break;
945 default:
946 assert(atom->src(0).getFile() == FILE_MEMORY_GLOBAL);
947 return true;
948 }
949 Value *base =
950 bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));
951 Value *ptr = atom->getIndirect(0, 0);
952
953 atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));
954 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
955 if (ptr)
956 base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);
957 atom->setIndirect(0, 0, base);
958
959 return true;
960 }
961
962 bool
963 NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)
964 {
965 if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&
966 cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)
967 return false;
968 bld.setPosition(cas, true);
969
970 if (needCctl) {
971 Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));
972 cctl->setIndirect(0, 0, cas->getIndirect(0, 0));
973 cctl->fixed = 1;
974 cctl->subOp = NV50_IR_SUBOP_CCTL_IV;
975 if (cas->isPredicated())
976 cctl->setPredicate(cas->cc, cas->getPredicate());
977 }
978
979 if (cas->defExists(0) && cas->subOp == NV50_IR_SUBOP_ATOM_CAS) {
980 // CAS is crazy. It's 2nd source is a double reg, and the 3rd source
981 // should be set to the high part of the double reg or bad things will
982 // happen elsewhere in the universe.
983 // Also, it sometimes returns the new value instead of the old one
984 // under mysterious circumstances.
985 Value *dreg = bld.getSSA(8);
986 bld.setPosition(cas, false);
987 bld.mkOp2(OP_MERGE, TYPE_U64, dreg, cas->getSrc(1), cas->getSrc(2));
988 cas->setSrc(1, dreg);
989 }
990
991 return true;
992 }
993
994 inline Value *
995 NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off)
996 {
997 uint8_t b = prog->driver->io.resInfoCBSlot;
998 off += prog->driver->io.suInfoBase;
999 return bld.
1000 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1001 }
1002
1003 inline Value *
1004 NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)
1005 {
1006 uint8_t b = prog->driver->io.msInfoCBSlot;
1007 off += prog->driver->io.msInfoBase;
1008 return bld.
1009 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1010 }
1011
1012 /* On nvc0, surface info is obtained via the surface binding points passed
1013 * to the SULD/SUST instructions.
1014 * On nve4, surface info is stored in c[] and is used by various special
1015 * instructions, e.g. for clamping coordiantes or generating an address.
1016 * They couldn't just have added an equivalent to TIC now, couldn't they ?
1017 */
1018 #define NVE4_SU_INFO_ADDR 0x00
1019 #define NVE4_SU_INFO_FMT 0x04
1020 #define NVE4_SU_INFO_DIM_X 0x08
1021 #define NVE4_SU_INFO_PITCH 0x0c
1022 #define NVE4_SU_INFO_DIM_Y 0x10
1023 #define NVE4_SU_INFO_ARRAY 0x14
1024 #define NVE4_SU_INFO_DIM_Z 0x18
1025 #define NVE4_SU_INFO_UNK1C 0x1c
1026 #define NVE4_SU_INFO_WIDTH 0x20
1027 #define NVE4_SU_INFO_HEIGHT 0x24
1028 #define NVE4_SU_INFO_DEPTH 0x28
1029 #define NVE4_SU_INFO_TARGET 0x2c
1030 #define NVE4_SU_INFO_CALL 0x30
1031 #define NVE4_SU_INFO_RAW_X 0x34
1032 #define NVE4_SU_INFO_MS_X 0x38
1033 #define NVE4_SU_INFO_MS_Y 0x3c
1034
1035 #define NVE4_SU_INFO__STRIDE 0x40
1036
1037 #define NVE4_SU_INFO_DIM(i) (0x08 + (i) * 8)
1038 #define NVE4_SU_INFO_SIZE(i) (0x20 + (i) * 4)
1039 #define NVE4_SU_INFO_MS(i) (0x38 + (i) * 4)
1040
1041 static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)
1042 {
1043 switch (su->tex.target.getEnum()) {
1044 case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);
1045 case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1046 case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1047 case TEX_TARGET_1D_ARRAY: return (c == 1) ?
1048 NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :
1049 NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1050 case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1051 case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1052 case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1053 case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1054 case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1055 case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1056 case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1057 default:
1058 assert(0);
1059 return 0;
1060 }
1061 }
1062
1063 void
1064 NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex)
1065 {
1066 const uint16_t base = tex->tex.r * NVE4_SU_INFO__STRIDE;
1067 const int arg = tex->tex.target.getArgCount();
1068
1069 if (tex->tex.target == TEX_TARGET_2D_MS)
1070 tex->tex.target = TEX_TARGET_2D;
1071 else
1072 if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY)
1073 tex->tex.target = TEX_TARGET_2D_ARRAY;
1074 else
1075 return;
1076
1077 Value *x = tex->getSrc(0);
1078 Value *y = tex->getSrc(1);
1079 Value *s = tex->getSrc(arg - 1);
1080
1081 Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA();
1082
1083 Value *ms_x = loadResInfo32(NULL, base + NVE4_SU_INFO_MS(0));
1084 Value *ms_y = loadResInfo32(NULL, base + NVE4_SU_INFO_MS(1));
1085
1086 bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x);
1087 bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y);
1088
1089 s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7));
1090 s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3));
1091
1092 Value *dx = loadMsInfo32(ts, 0x0);
1093 Value *dy = loadMsInfo32(ts, 0x4);
1094
1095 bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx);
1096 bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy);
1097
1098 tex->setSrc(0, tx);
1099 tex->setSrc(1, ty);
1100 tex->moveSources(arg, -1);
1101 }
1102
1103 // Sets 64-bit "generic address", predicate and format sources for SULD/SUST.
1104 // They're computed from the coordinates using the surface info in c[] space.
1105 void
1106 NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su)
1107 {
1108 Instruction *insn;
1109 const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP;
1110 const bool raw =
1111 su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB;
1112 const int idx = su->tex.r;
1113 const int dim = su->tex.target.getDim();
1114 const int arg = dim + (su->tex.target.isArray() ? 1 : 0);
1115 const uint16_t base = idx * NVE4_SU_INFO__STRIDE;
1116 int c;
1117 Value *zero = bld.mkImm(0);
1118 Value *p1 = NULL;
1119 Value *v;
1120 Value *src[3];
1121 Value *bf, *eau, *off;
1122 Value *addr, *pred;
1123
1124 off = bld.getScratch(4);
1125 bf = bld.getScratch(4);
1126 addr = bld.getSSA(8);
1127 pred = bld.getScratch(1, FILE_PREDICATE);
1128
1129 bld.setPosition(su, false);
1130
1131 adjustCoordinatesMS(su);
1132
1133 // calculate clamped coordinates
1134 for (c = 0; c < arg; ++c) {
1135 src[c] = bld.getScratch();
1136 if (c == 0 && raw)
1137 v = loadResInfo32(NULL, base + NVE4_SU_INFO_RAW_X);
1138 else
1139 v = loadResInfo32(NULL, base + NVE4_SU_INFO_DIM(c));
1140 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero)
1141 ->subOp = getSuClampSubOp(su, c);
1142 }
1143 for (; c < 3; ++c)
1144 src[c] = zero;
1145
1146 // set predicate output
1147 if (su->tex.target == TEX_TARGET_BUFFER) {
1148 src[0]->getInsn()->setFlagsDef(1, pred);
1149 } else
1150 if (su->tex.target.isArray()) {
1151 p1 = bld.getSSA(1, FILE_PREDICATE);
1152 src[dim]->getInsn()->setFlagsDef(1, p1);
1153 }
1154
1155 // calculate pixel offset
1156 if (dim == 1) {
1157 if (su->tex.target != TEX_TARGET_BUFFER)
1158 bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff));
1159 } else
1160 if (dim == 3) {
1161 v = loadResInfo32(NULL, base + NVE4_SU_INFO_UNK1C);
1162 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1])
1163 ->subOp = NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1164
1165 v = loadResInfo32(NULL, base + NVE4_SU_INFO_PITCH);
1166 bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0])
1167 ->subOp = NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l
1168 } else {
1169 assert(dim == 2);
1170 v = loadResInfo32(NULL, base + NVE4_SU_INFO_PITCH);
1171 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[1], v, src[0])
1172 ->subOp = su->tex.target.isArray() ?
1173 NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1174 }
1175
1176 // calculate effective address part 1
1177 if (su->tex.target == TEX_TARGET_BUFFER) {
1178 if (raw) {
1179 bf = src[0];
1180 } else {
1181 v = loadResInfo32(NULL, base + NVE4_SU_INFO_FMT);
1182 bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero)
1183 ->subOp = NV50_IR_SUBOP_V1(7,6,8|2);
1184 }
1185 } else {
1186 Value *y = src[1];
1187 Value *z = src[2];
1188 uint16_t subOp = 0;
1189
1190 switch (dim) {
1191 case 1:
1192 y = zero;
1193 z = zero;
1194 break;
1195 case 2:
1196 z = off;
1197 if (!su->tex.target.isArray()) {
1198 z = loadResInfo32(NULL, base + NVE4_SU_INFO_UNK1C);
1199 subOp = NV50_IR_SUBOP_SUBFM_3D;
1200 }
1201 break;
1202 default:
1203 subOp = NV50_IR_SUBOP_SUBFM_3D;
1204 assert(dim == 3);
1205 break;
1206 }
1207 insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z);
1208 insn->subOp = subOp;
1209 insn->setFlagsDef(1, pred);
1210 }
1211
1212 // part 2
1213 v = loadResInfo32(NULL, base + NVE4_SU_INFO_ADDR);
1214
1215 if (su->tex.target == TEX_TARGET_BUFFER) {
1216 eau = v;
1217 } else {
1218 eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v);
1219 }
1220 // add array layer offset
1221 if (su->tex.target.isArray()) {
1222 v = loadResInfo32(NULL, base + NVE4_SU_INFO_ARRAY);
1223 if (dim == 1)
1224 bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau)
1225 ->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32
1226 else
1227 bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau)
1228 ->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32
1229 // combine predicates
1230 assert(p1);
1231 bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1);
1232 }
1233
1234 if (atom) {
1235 Value *lo = bf;
1236 if (su->tex.target == TEX_TARGET_BUFFER) {
1237 lo = zero;
1238 bld.mkMov(off, bf);
1239 }
1240 // bf == g[] address & 0xff
1241 // eau == g[] address >> 8
1242 bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau);
1243 bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau);
1244 } else
1245 if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) {
1246 // Convert from u32 to u8 address format, which is what the library code
1247 // doing SULDP currently uses.
1248 // XXX: can SUEAU do this ?
1249 // XXX: does it matter that we don't mask high bytes in bf ?
1250 // Grrr.
1251 bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8));
1252 bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off);
1253 }
1254
1255 bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau);
1256
1257 if (atom && su->tex.target == TEX_TARGET_BUFFER)
1258 bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off);
1259
1260 // let's just set it 0 for raw access and hope it works
1261 v = raw ?
1262 bld.mkImm(0) : loadResInfo32(NULL, base + NVE4_SU_INFO_FMT);
1263
1264 // get rid of old coordinate sources, make space for fmt info and predicate
1265 su->moveSources(arg, 3 - arg);
1266 // set 64 bit address and 32-bit format sources
1267 su->setSrc(0, addr);
1268 su->setSrc(1, v);
1269 su->setSrc(2, pred);
1270 }
1271
1272 void
1273 NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su)
1274 {
1275 processSurfaceCoordsNVE4(su);
1276
1277 // Who do we hate more ? The person who decided that nvc0's SULD doesn't
1278 // have to support conversion or the person who decided that, in OpenCL,
1279 // you don't have to specify the format here like you do in OpenGL ?
1280
1281 if (su->op == OP_SULDP) {
1282 // We don't patch shaders. Ever.
1283 // You get an indirect call to our library blob here.
1284 // But at least it's uniform.
1285 FlowInstruction *call;
1286 LValue *p[3];
1287 LValue *r[5];
1288 uint16_t base = su->tex.r * NVE4_SU_INFO__STRIDE + NVE4_SU_INFO_CALL;
1289
1290 for (int i = 0; i < 4; ++i)
1291 (r[i] = bld.getScratch(4, FILE_GPR))->reg.data.id = i;
1292 for (int i = 0; i < 3; ++i)
1293 (p[i] = bld.getScratch(1, FILE_PREDICATE))->reg.data.id = i;
1294 (r[4] = bld.getScratch(8, FILE_GPR))->reg.data.id = 4;
1295
1296 bld.mkMov(p[1], bld.mkImm((su->cache == CACHE_CA) ? 1 : 0), TYPE_U8);
1297 bld.mkMov(p[2], bld.mkImm((su->cache == CACHE_CG) ? 1 : 0), TYPE_U8);
1298 bld.mkMov(p[0], su->getSrc(2), TYPE_U8);
1299 bld.mkMov(r[4], su->getSrc(0), TYPE_U64);
1300 bld.mkMov(r[2], su->getSrc(1), TYPE_U32);
1301
1302 call = bld.mkFlow(OP_CALL, NULL, su->cc, su->getPredicate());
1303
1304 call->indirect = 1;
1305 call->absolute = 1;
1306 call->setSrc(0, bld.mkSymbol(FILE_MEMORY_CONST,
1307 prog->driver->io.resInfoCBSlot, TYPE_U32,
1308 prog->driver->io.suInfoBase + base));
1309 call->setSrc(1, r[2]);
1310 call->setSrc(2, r[4]);
1311 for (int i = 0; i < 3; ++i)
1312 call->setSrc(3 + i, p[i]);
1313 for (int i = 0; i < 4; ++i) {
1314 call->setDef(i, r[i]);
1315 bld.mkMov(su->getDef(i), r[i]);
1316 }
1317 call->setDef(4, p[1]);
1318 delete_Instruction(bld.getProgram(), su);
1319 }
1320
1321 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
1322 // FIXME: for out of bounds access, destination value will be undefined !
1323 Value *pred = su->getSrc(2);
1324 CondCode cc = CC_NOT_P;
1325 if (su->getPredicate()) {
1326 pred = bld.getScratch(1, FILE_PREDICATE);
1327 cc = su->cc;
1328 if (cc == CC_NOT_P) {
1329 bld.mkOp2(OP_OR, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1330 } else {
1331 bld.mkOp2(OP_AND, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1332 pred->getInsn()->src(1).mod = Modifier(NV50_IR_MOD_NOT);
1333 }
1334 }
1335 Instruction *red = bld.mkOp(OP_ATOM, su->dType, su->getDef(0));
1336 red->subOp = su->subOp;
1337 if (!gMemBase)
1338 gMemBase = bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0);
1339 red->setSrc(0, gMemBase);
1340 red->setSrc(1, su->getSrc(3));
1341 if (su->subOp == NV50_IR_SUBOP_ATOM_CAS)
1342 red->setSrc(2, su->getSrc(4));
1343 red->setIndirect(0, 0, su->getSrc(0));
1344 red->setPredicate(cc, pred);
1345 delete_Instruction(bld.getProgram(), su);
1346 handleCasExch(red, true);
1347 } else {
1348 su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8;
1349 }
1350 }
1351
1352 bool
1353 NVC0LoweringPass::handleWRSV(Instruction *i)
1354 {
1355 Instruction *st;
1356 Symbol *sym;
1357 uint32_t addr;
1358
1359 // must replace, $sreg are not writeable
1360 addr = targ->getSVAddress(FILE_SHADER_OUTPUT, i->getSrc(0)->asSym());
1361 if (addr >= 0x400)
1362 return false;
1363 sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
1364
1365 st = bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0),
1366 i->getSrc(1));
1367 st->perPatch = i->perPatch;
1368
1369 bld.getBB()->remove(i);
1370 return true;
1371 }
1372
1373 void
1374 NVC0LoweringPass::readTessCoord(LValue *dst, int c)
1375 {
1376 Value *laneid = bld.getSSA();
1377 Value *x, *y;
1378
1379 bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0));
1380
1381 if (c == 0) {
1382 x = dst;
1383 y = NULL;
1384 } else
1385 if (c == 1) {
1386 x = NULL;
1387 y = dst;
1388 } else {
1389 assert(c == 2);
1390 x = bld.getSSA();
1391 y = bld.getSSA();
1392 }
1393 if (x)
1394 bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid);
1395 if (y)
1396 bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid);
1397
1398 if (c == 2) {
1399 bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y);
1400 bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst);
1401 }
1402 }
1403
1404 bool
1405 NVC0LoweringPass::handleRDSV(Instruction *i)
1406 {
1407 Symbol *sym = i->getSrc(0)->asSym();
1408 const SVSemantic sv = sym->reg.data.sv.sv;
1409 Value *vtx = NULL;
1410 Instruction *ld;
1411 uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
1412
1413 if (addr >= 0x400) {
1414 // mov $sreg
1415 if (sym->reg.data.sv.index == 3) {
1416 // TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID
1417 i->op = OP_MOV;
1418 i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0));
1419 }
1420 return true;
1421 }
1422
1423 switch (sv) {
1424 case SV_POSITION:
1425 assert(prog->getType() == Program::TYPE_FRAGMENT);
1426 if (i->srcExists(1)) {
1427 // Pass offset through to the interpolation logic
1428 ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET,
1429 i->getDef(0), addr, NULL);
1430 ld->setSrc(1, i->getSrc(1));
1431 } else {
1432 bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
1433 }
1434 break;
1435 case SV_FACE:
1436 {
1437 Value *face = i->getDef(0);
1438 bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL);
1439 if (i->dType == TYPE_F32) {
1440 bld.mkOp2(OP_AND, TYPE_U32, face, face, bld.mkImm(0x80000000));
1441 bld.mkOp2(OP_XOR, TYPE_U32, face, face, bld.mkImm(0xbf800000));
1442 }
1443 }
1444 break;
1445 case SV_TESS_COORD:
1446 assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL);
1447 readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index);
1448 break;
1449 case SV_NTID:
1450 case SV_NCTAID:
1451 case SV_GRIDID:
1452 assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise
1453 if (sym->reg.data.sv.index == 3) {
1454 i->op = OP_MOV;
1455 i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1));
1456 return true;
1457 }
1458 addr += prog->driver->prop.cp.gridInfoBase;
1459 bld.mkLoad(TYPE_U32, i->getDef(0),
1460 bld.mkSymbol(FILE_MEMORY_CONST, 0, TYPE_U32, addr), NULL);
1461 break;
1462 case SV_SAMPLE_INDEX:
1463 // TODO: Properly pass source as an address in the PIX address space
1464 // (which can be of the form [r0+offset]). But this is currently
1465 // unnecessary.
1466 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1467 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
1468 break;
1469 case SV_SAMPLE_POS: {
1470 Value *off = new_LValue(func, FILE_GPR);
1471 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1472 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
1473 bld.mkOp2(OP_SHL, TYPE_U32, off, i->getDef(0), bld.mkImm(3));
1474 bld.mkLoad(TYPE_F32,
1475 i->getDef(0),
1476 bld.mkSymbol(
1477 FILE_MEMORY_CONST, prog->driver->io.resInfoCBSlot,
1478 TYPE_U32, prog->driver->io.sampleInfoBase +
1479 4 * sym->reg.data.sv.index),
1480 off);
1481 break;
1482 }
1483 case SV_SAMPLE_MASK:
1484 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1485 ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK;
1486 break;
1487 default:
1488 if (prog->getType() == Program::TYPE_TESSELLATION_EVAL)
1489 vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0));
1490 ld = bld.mkFetch(i->getDef(0), i->dType,
1491 FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx);
1492 ld->perPatch = i->perPatch;
1493 break;
1494 }
1495 bld.getBB()->remove(i);
1496 return true;
1497 }
1498
1499 bool
1500 NVC0LoweringPass::handleDIV(Instruction *i)
1501 {
1502 if (!isFloatType(i->dType))
1503 return true;
1504 bld.setPosition(i, false);
1505 Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(), i->getSrc(1));
1506 i->op = OP_MUL;
1507 i->setSrc(1, rcp->getDef(0));
1508 return true;
1509 }
1510
1511 bool
1512 NVC0LoweringPass::handleMOD(Instruction *i)
1513 {
1514 if (i->dType != TYPE_F32)
1515 return true;
1516 LValue *value = bld.getScratch();
1517 bld.mkOp1(OP_RCP, TYPE_F32, value, i->getSrc(1));
1518 bld.mkOp2(OP_MUL, TYPE_F32, value, i->getSrc(0), value);
1519 bld.mkOp1(OP_TRUNC, TYPE_F32, value, value);
1520 bld.mkOp2(OP_MUL, TYPE_F32, value, i->getSrc(1), value);
1521 i->op = OP_SUB;
1522 i->setSrc(1, value);
1523 return true;
1524 }
1525
1526 bool
1527 NVC0LoweringPass::handleSQRT(Instruction *i)
1528 {
1529 Instruction *rsq = bld.mkOp1(OP_RSQ, TYPE_F32,
1530 bld.getSSA(), i->getSrc(0));
1531 i->op = OP_MUL;
1532 i->setSrc(1, rsq->getDef(0));
1533
1534 return true;
1535 }
1536
1537 bool
1538 NVC0LoweringPass::handlePOW(Instruction *i)
1539 {
1540 LValue *val = bld.getScratch();
1541
1542 bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
1543 bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
1544 bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
1545
1546 i->op = OP_EX2;
1547 i->setSrc(0, val);
1548 i->setSrc(1, NULL);
1549
1550 return true;
1551 }
1552
1553 bool
1554 NVC0LoweringPass::handleEXPORT(Instruction *i)
1555 {
1556 if (prog->getType() == Program::TYPE_FRAGMENT) {
1557 int id = i->getSrc(0)->reg.data.offset / 4;
1558
1559 if (i->src(0).isIndirect(0)) // TODO, ugly
1560 return false;
1561 i->op = OP_MOV;
1562 i->subOp = NV50_IR_SUBOP_MOV_FINAL;
1563 i->src(0).set(i->src(1));
1564 i->setSrc(1, NULL);
1565 i->setDef(0, new_LValue(func, FILE_GPR));
1566 i->getDef(0)->reg.data.id = id;
1567
1568 prog->maxGPR = MAX2(prog->maxGPR, id);
1569 } else
1570 if (prog->getType() == Program::TYPE_GEOMETRY) {
1571 i->setIndirect(0, 1, gpEmitAddress);
1572 }
1573 return true;
1574 }
1575
1576 bool
1577 NVC0LoweringPass::handleOUT(Instruction *i)
1578 {
1579 Instruction *prev = i->prev;
1580 ImmediateValue stream, prevStream;
1581
1582 // Only merge if the stream ids match. Also, note that the previous
1583 // instruction would have already been lowered, so we take arg1 from it.
1584 if (i->op == OP_RESTART && prev && prev->op == OP_EMIT &&
1585 i->src(0).getImmediate(stream) &&
1586 prev->src(1).getImmediate(prevStream) &&
1587 stream.reg.data.u32 == prevStream.reg.data.u32) {
1588 i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART;
1589 delete_Instruction(prog, i);
1590 } else {
1591 assert(gpEmitAddress);
1592 i->setDef(0, gpEmitAddress);
1593 i->setSrc(1, i->getSrc(0));
1594 i->setSrc(0, gpEmitAddress);
1595 }
1596 return true;
1597 }
1598
1599 // Generate a binary predicate if an instruction is predicated by
1600 // e.g. an f32 value.
1601 void
1602 NVC0LoweringPass::checkPredicate(Instruction *insn)
1603 {
1604 Value *pred = insn->getPredicate();
1605 Value *pdst;
1606
1607 if (!pred || pred->reg.file == FILE_PREDICATE)
1608 return;
1609 pdst = new_LValue(func, FILE_PREDICATE);
1610
1611 // CAUTION: don't use pdst->getInsn, the definition might not be unique,
1612 // delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass
1613
1614 bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred);
1615
1616 insn->setPredicate(insn->cc, pdst);
1617 }
1618
1619 //
1620 // - add quadop dance for texturing
1621 // - put FP outputs in GPRs
1622 // - convert instruction sequences
1623 //
1624 bool
1625 NVC0LoweringPass::visit(Instruction *i)
1626 {
1627 bld.setPosition(i, false);
1628
1629 if (i->cc != CC_ALWAYS)
1630 checkPredicate(i);
1631
1632 switch (i->op) {
1633 case OP_TEX:
1634 case OP_TXB:
1635 case OP_TXL:
1636 case OP_TXF:
1637 case OP_TXG:
1638 return handleTEX(i->asTex());
1639 case OP_TXD:
1640 return handleTXD(i->asTex());
1641 case OP_TXLQ:
1642 return handleTXLQ(i->asTex());
1643 case OP_TXQ:
1644 return handleTXQ(i->asTex());
1645 case OP_EX2:
1646 bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
1647 i->setSrc(0, i->getDef(0));
1648 break;
1649 case OP_POW:
1650 return handlePOW(i);
1651 case OP_DIV:
1652 return handleDIV(i);
1653 case OP_MOD:
1654 return handleMOD(i);
1655 case OP_SQRT:
1656 return handleSQRT(i);
1657 case OP_EXPORT:
1658 return handleEXPORT(i);
1659 case OP_EMIT:
1660 case OP_RESTART:
1661 return handleOUT(i);
1662 case OP_RDSV:
1663 return handleRDSV(i);
1664 case OP_WRSV:
1665 return handleWRSV(i);
1666 case OP_LOAD:
1667 if (i->src(0).getFile() == FILE_SHADER_INPUT) {
1668 if (prog->getType() == Program::TYPE_COMPUTE) {
1669 i->getSrc(0)->reg.file = FILE_MEMORY_CONST;
1670 i->getSrc(0)->reg.fileIndex = 0;
1671 } else
1672 if (prog->getType() == Program::TYPE_GEOMETRY &&
1673 i->src(0).isIndirect(0)) {
1674 // XXX: this assumes vec4 units
1675 Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1676 i->getIndirect(0, 0), bld.mkImm(4));
1677 i->setIndirect(0, 0, ptr);
1678 } else {
1679 i->op = OP_VFETCH;
1680 assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP
1681 }
1682 } else if (i->src(0).getFile() == FILE_MEMORY_CONST) {
1683 if (i->src(0).isIndirect(1)) {
1684 Value *ptr;
1685 if (i->src(0).isIndirect(0))
1686 ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(),
1687 i->getIndirect(0, 1), bld.mkImm(0x1010),
1688 i->getIndirect(0, 0));
1689 else
1690 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1691 i->getIndirect(0, 1), bld.mkImm(16));
1692 i->setIndirect(0, 1, NULL);
1693 i->setIndirect(0, 0, ptr);
1694 i->subOp = NV50_IR_SUBOP_LDC_IS;
1695 }
1696 }
1697 break;
1698 case OP_ATOM:
1699 {
1700 const bool cctl = i->src(0).getFile() == FILE_MEMORY_GLOBAL;
1701 handleATOM(i);
1702 handleCasExch(i, cctl);
1703 }
1704 break;
1705 case OP_SULDB:
1706 case OP_SULDP:
1707 case OP_SUSTB:
1708 case OP_SUSTP:
1709 case OP_SUREDB:
1710 case OP_SUREDP:
1711 if (targ->getChipset() >= NVISA_GK104_CHIPSET)
1712 handleSurfaceOpNVE4(i->asTex());
1713 break;
1714 default:
1715 break;
1716 }
1717 return true;
1718 }
1719
1720 bool
1721 TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const
1722 {
1723 if (stage == CG_STAGE_PRE_SSA) {
1724 NVC0LoweringPass pass(prog);
1725 return pass.run(prog, false, true);
1726 } else
1727 if (stage == CG_STAGE_POST_RA) {
1728 NVC0LegalizePostRA pass(prog);
1729 return pass.run(prog, false, true);
1730 } else
1731 if (stage == CG_STAGE_SSA) {
1732 NVC0LegalizeSSA pass;
1733 return pass.run(prog, false, true);
1734 }
1735 return false;
1736 }
1737
1738 } // namespace nv50_ir