projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nvc0/ir: describe the tex arguments for fermi/kepler
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_lowering_nvc0.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 #include "codegen/nv50_ir_target_nvc0.h"
27 #include "codegen/nv50_ir_lowering_nvc0.h"
28
29 #include <limits>
30
31 namespace nv50_ir {
32
33 #define QOP_ADD 0
34 #define QOP_SUBR 1
35 #define QOP_SUB 2
36 #define QOP_MOV2 3
37
38 // UL UR LL LR
39 #define QUADOP(q, r, s, t) \
40 ((QOP_##q << 6) | (QOP_##r << 4) | \
41 (QOP_##s << 2) | (QOP_##t << 0))
42
43 void
44 NVC0LegalizeSSA::handleDIV(Instruction *i)
45 {
46 FlowInstruction *call;
47 int builtin;
48 Value *def[2];
49
50 bld.setPosition(i, false);
51 def[0] = bld.mkMovToReg(0, i->getSrc(0))->getDef(0);
52 def[1] = bld.mkMovToReg(1, i->getSrc(1))->getDef(0);
53 switch (i->dType) {
54 case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;
55 case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;
56 default:
57 return;
58 }
59 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
60 bld.mkMov(i->getDef(0), def[(i->op == OP_DIV) ? 0 : 1]);
61 bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);
62 bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);
63
64 call->fixed = 1;
65 call->absolute = call->builtin = 1;
66 call->target.builtin = builtin;
67 delete_Instruction(prog, i);
68 }
69
70 void
71 NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)
72 {
73 // TODO
74 }
75
76 bool
77 NVC0LegalizeSSA::visit(Function *fn)
78 {
79 bld.setProgram(fn->getProgram());
80 return true;
81 }
82
83 bool
84 NVC0LegalizeSSA::visit(BasicBlock *bb)
85 {
86 Instruction *next;
87 for (Instruction *i = bb->getEntry(); i; i = next) {
88 next = i->next;
89 if (i->dType == TYPE_F32)
90 continue;
91 switch (i->op) {
92 case OP_DIV:
93 case OP_MOD:
94 handleDIV(i);
95 break;
96 case OP_RCP:
97 case OP_RSQ:
98 if (i->dType == TYPE_F64)
99 handleRCPRSQ(i);
100 break;
101 default:
102 break;
103 }
104 }
105 return true;
106 }
107
108 NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)
109 : rZero(NULL),
110 carry(NULL),
111 needTexBar(prog->getTarget()->getChipset() >= 0xe0)
112 {
113 }
114
115 bool
116 NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,
117 const Instruction *early) const
118 {
119 if (early->bb == later->bb)
120 return early->serial < later->serial;
121 return later->bb->dominatedBy(early->bb);
122 }
123
124 void
125 NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses,
126 Instruction *usei, const Instruction *insn)
127 {
128 bool add = true;
129 for (std::list<TexUse>::iterator it = uses.begin();
130 it != uses.end();) {
131 if (insnDominatedBy(usei, it->insn)) {
132 add = false;
133 break;
134 }
135 if (insnDominatedBy(it->insn, usei))
136 it = uses.erase(it);
137 else
138 ++it;
139 }
140 if (add)
141 uses.push_back(TexUse(usei, insn));
142 }
143
144 void
145 NVC0LegalizePostRA::findOverwritingDefs(const Instruction *texi,
146 Instruction *insn,
147 const BasicBlock *term,
148 std::list<TexUse> &uses)
149 {
150 while (insn->op == OP_MOV && insn->getDef(0)->equals(insn->getSrc(0)))
151 insn = insn->getSrc(0)->getUniqueInsn();
152
153 if (!insn || !insn->bb->reachableBy(texi->bb, term))
154 return;
155
156 switch (insn->op) {
157 /* Values not connected to the tex's definition through any of these should
158 * not be conflicting.
159 */
160 case OP_SPLIT:
161 case OP_MERGE:
162 case OP_PHI:
163 case OP_UNION:
164 /* recurse again */
165 for (int s = 0; insn->srcExists(s); ++s)
166 findOverwritingDefs(texi, insn->getSrc(s)->getUniqueInsn(), term,
167 uses);
168 break;
169 default:
170 // if (!isTextureOp(insn->op)) // TODO: are TEXes always ordered ?
171 addTexUse(uses, insn, texi);
172 break;
173 }
174 }
175
176 void
177 NVC0LegalizePostRA::findFirstUses(const Instruction *texi,
178 const Instruction *insn,
179 std::list<TexUse> &uses)
180 {
181 for (int d = 0; insn->defExists(d); ++d) {
182 Value *v = insn->getDef(d);
183 for (Value::UseIterator u = v->uses.begin(); u != v->uses.end(); ++u) {
184 Instruction *usei = (*u)->getInsn();
185
186 if (usei->op == OP_PHI || usei->op == OP_UNION) {
187 // need a barrier before WAW cases
188 for (int s = 0; usei->srcExists(s); ++s) {
189 Instruction *defi = usei->getSrc(s)->getUniqueInsn();
190 if (defi && &usei->src(s) != *u)
191 findOverwritingDefs(texi, defi, usei->bb, uses);
192 }
193 }
194
195 if (usei->op == OP_SPLIT ||
196 usei->op == OP_MERGE ||
197 usei->op == OP_PHI ||
198 usei->op == OP_UNION) {
199 // these uses don't manifest in the machine code
200 findFirstUses(texi, usei, uses);
201 } else
202 if (usei->op == OP_MOV && usei->getDef(0)->equals(usei->getSrc(0)) &&
203 usei->subOp != NV50_IR_SUBOP_MOV_FINAL) {
204 findFirstUses(texi, usei, uses);
205 } else {
206 addTexUse(uses, usei, insn);
207 }
208 }
209 }
210 }
211
212 // Texture barriers:
213 // This pass is a bit long and ugly and can probably be optimized.
214 //
215 // 1. obtain a list of TEXes and their outputs' first use(s)
216 // 2. calculate the barrier level of each first use (minimal number of TEXes,
217 // over all paths, between the TEX and the use in question)
218 // 3. for each barrier, if all paths from the source TEX to that barrier
219 // contain a barrier of lesser level, it can be culled
220 bool
221 NVC0LegalizePostRA::insertTextureBarriers(Function *fn)
222 {
223 std::list<TexUse> *uses;
224 std::vector<Instruction *> texes;
225 std::vector<int> bbFirstTex;
226 std::vector<int> bbFirstUse;
227 std::vector<int> texCounts;
228 std::vector<TexUse> useVec;
229 ArrayList insns;
230
231 fn->orderInstructions(insns);
232
233 texCounts.resize(fn->allBBlocks.getSize(), 0);
234 bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());
235 bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());
236
237 // tag BB CFG nodes by their id for later
238 for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {
239 BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get());
240 if (bb)
241 bb->cfg.tag = bb->getId();
242 }
243
244 // gather the first uses for each TEX
245 for (int i = 0; i < insns.getSize(); ++i) {
246 Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i));
247 if (isTextureOp(tex->op)) {
248 texes.push_back(tex);
249 if (!texCounts.at(tex->bb->getId()))
250 bbFirstTex[tex->bb->getId()] = texes.size() - 1;
251 texCounts[tex->bb->getId()]++;
252 }
253 }
254 insns.clear();
255 if (texes.empty())
256 return false;
257 uses = new std::list<TexUse>[texes.size()];
258 if (!uses)
259 return false;
260 for (size_t i = 0; i < texes.size(); ++i)
261 findFirstUses(texes[i], texes[i], uses[i]);
262
263 // determine the barrier level at each use
264 for (size_t i = 0; i < texes.size(); ++i) {
265 for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end();
266 ++u) {
267 BasicBlock *tb = texes[i]->bb;
268 BasicBlock *ub = u->insn->bb;
269 if (tb == ub) {
270 u->level = 0;
271 for (size_t j = i + 1; j < texes.size() &&
272 texes[j]->bb == tb && texes[j]->serial < u->insn->serial;
273 ++j)
274 u->level++;
275 } else {
276 u->level = fn->cfg.findLightestPathWeight(&tb->cfg,
277 &ub->cfg, texCounts);
278 if (u->level < 0) {
279 WARN("Failed to find path TEX -> TEXBAR\n");
280 u->level = 0;
281 continue;
282 }
283 // this counted all TEXes in the origin block, correct that
284 u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;
285 // and did not count the TEXes in the destination block, add those
286 for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&
287 texes[j]->bb == ub && texes[j]->serial < u->insn->serial;
288 ++j)
289 u->level++;
290 }
291 assert(u->level >= 0);
292 useVec.push_back(*u);
293 }
294 }
295 delete[] uses;
296 uses = NULL;
297
298 // insert the barriers
299 for (size_t i = 0; i < useVec.size(); ++i) {
300 Instruction *prev = useVec[i].insn->prev;
301 if (useVec[i].level < 0)
302 continue;
303 if (prev && prev->op == OP_TEXBAR) {
304 if (prev->subOp > useVec[i].level)
305 prev->subOp = useVec[i].level;
306 prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));
307 } else {
308 Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);
309 bar->fixed = 1;
310 bar->subOp = useVec[i].level;
311 // make use explicit to ease latency calculation
312 bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));
313 useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);
314 }
315 }
316
317 if (fn->getProgram()->optLevel < 3) {
318 if (uses)
319 delete[] uses;
320 return true;
321 }
322
323 std::vector<Limits> limitT, limitB, limitS; // entry, exit, single
324
325 limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));
326 limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));
327 limitS.resize(fn->allBBlocks.getSize());
328
329 // cull unneeded barriers (should do that earlier, but for simplicity)
330 IteratorRef bi = fn->cfg.iteratorCFG();
331 // first calculate min/max outstanding TEXes for each BB
332 for (bi->reset(); !bi->end(); bi->next()) {
333 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
334 BasicBlock *bb = BasicBlock::get(n);
335 int min = 0;
336 int max = std::numeric_limits<int>::max();
337 for (Instruction *i = bb->getFirst(); i; i = i->next) {
338 if (isTextureOp(i->op)) {
339 min++;
340 if (max < std::numeric_limits<int>::max())
341 max++;
342 } else
343 if (i->op == OP_TEXBAR) {
344 min = MIN2(min, i->subOp);
345 max = MIN2(max, i->subOp);
346 }
347 }
348 // limits when looking at an isolated block
349 limitS[bb->getId()].min = min;
350 limitS[bb->getId()].max = max;
351 }
352 // propagate the min/max values
353 for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {
354 for (bi->reset(); !bi->end(); bi->next()) {
355 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
356 BasicBlock *bb = BasicBlock::get(n);
357 const int bbId = bb->getId();
358 for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {
359 BasicBlock *in = BasicBlock::get(ei.getNode());
360 const int inId = in->getId();
361 limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);
362 limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);
363 }
364 // I just hope this is correct ...
365 if (limitS[bbId].max == std::numeric_limits<int>::max()) {
366 // no barrier
367 limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;
368 limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;
369 } else {
370 // block contained a barrier
371 limitB[bbId].min = MIN2(limitS[bbId].max,
372 limitT[bbId].min + limitS[bbId].min);
373 limitB[bbId].max = MIN2(limitS[bbId].max,
374 limitT[bbId].max + limitS[bbId].min);
375 }
376 }
377 }
378 // finally delete unnecessary barriers
379 for (bi->reset(); !bi->end(); bi->next()) {
380 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
381 BasicBlock *bb = BasicBlock::get(n);
382 Instruction *prev = NULL;
383 Instruction *next;
384 int max = limitT[bb->getId()].max;
385 for (Instruction *i = bb->getFirst(); i; i = next) {
386 next = i->next;
387 if (i->op == OP_TEXBAR) {
388 if (i->subOp >= max) {
389 delete_Instruction(prog, i);
390 i = NULL;
391 } else {
392 max = i->subOp;
393 if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {
394 delete_Instruction(prog, prev);
395 prev = NULL;
396 }
397 }
398 } else
399 if (isTextureOp(i->op)) {
400 max++;
401 }
402 if (i && !i->isNop())
403 prev = i;
404 }
405 }
406 if (uses)
407 delete[] uses;
408 return true;
409 }
410
411 bool
412 NVC0LegalizePostRA::visit(Function *fn)
413 {
414 if (needTexBar)
415 insertTextureBarriers(fn);
416
417 rZero = new_LValue(fn, FILE_GPR);
418 carry = new_LValue(fn, FILE_FLAGS);
419
420 rZero->reg.data.id = prog->getTarget()->getFileSize(FILE_GPR);
421 carry->reg.data.id = 0;
422
423 return true;
424 }
425
426 void
427 NVC0LegalizePostRA::replaceZero(Instruction *i)
428 {
429 for (int s = 0; i->srcExists(s); ++s) {
430 if (s == 2 && i->op == OP_SUCLAMP)
431 continue;
432 ImmediateValue *imm = i->getSrc(s)->asImm();
433 if (imm && imm->reg.data.u64 == 0)
434 i->setSrc(s, rZero);
435 }
436 }
437
438 // replace CONT with BRA for single unconditional continue
439 bool
440 NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)
441 {
442 if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)
443 return false;
444 Graph::EdgeIterator ei = bb->cfg.incident();
445 if (ei.getType() != Graph::Edge::BACK)
446 ei.next();
447 if (ei.getType() != Graph::Edge::BACK)
448 return false;
449 BasicBlock *contBB = BasicBlock::get(ei.getNode());
450
451 if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||
452 contBB->getExit()->getPredicate())
453 return false;
454 contBB->getExit()->op = OP_BRA;
455 bb->remove(bb->getEntry()); // delete PRECONT
456
457 ei.next();
458 assert(ei.end() || ei.getType() != Graph::Edge::BACK);
459 return true;
460 }
461
462 // replace branches to join blocks with join ops
463 void
464 NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)
465 {
466 if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)
467 return;
468 for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
469 BasicBlock *in = BasicBlock::get(ei.getNode());
470 Instruction *exit = in->getExit();
471 if (!exit) {
472 in->insertTail(new FlowInstruction(func, OP_JOIN, bb));
473 // there should always be a terminator instruction
474 WARN("inserted missing terminator in BB:%i\n", in->getId());
475 } else
476 if (exit->op == OP_BRA) {
477 exit->op = OP_JOIN;
478 exit->asFlow()->limit = 1; // must-not-propagate marker
479 }
480 }
481 bb->remove(bb->getEntry());
482 }
483
484 bool
485 NVC0LegalizePostRA::visit(BasicBlock *bb)
486 {
487 Instruction *i, *next;
488
489 // remove pseudo operations and non-fixed no-ops, split 64 bit operations
490 for (i = bb->getFirst(); i; i = next) {
491 next = i->next;
492 if (i->op == OP_EMIT || i->op == OP_RESTART) {
493 if (!i->getDef(0)->refCount())
494 i->setDef(0, NULL);
495 if (i->src(0).getFile() == FILE_IMMEDIATE)
496 i->setSrc(0, rZero); // initial value must be 0
497 replaceZero(i);
498 } else
499 if (i->isNop()) {
500 bb->remove(i);
501 } else {
502 // TODO: Move this to before register allocation for operations that
503 // need the $c register !
504 if (typeSizeof(i->dType) == 8) {
505 Instruction *hi;
506 hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);
507 if (hi)
508 next = hi;
509 }
510
511 if (i->op != OP_MOV && i->op != OP_PFETCH)
512 replaceZero(i);
513 }
514 }
515 if (!bb->getEntry())
516 return true;
517
518 if (!tryReplaceContWithBra(bb))
519 propagateJoin(bb);
520
521 return true;
522 }
523
524 NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())
525 {
526 bld.setProgram(prog);
527 gMemBase = NULL;
528 }
529
530 bool
531 NVC0LoweringPass::visit(Function *fn)
532 {
533 if (prog->getType() == Program::TYPE_GEOMETRY) {
534 assert(!strncmp(fn->getName(), "MAIN", 4));
535 // TODO: when we generate actual functions pass this value along somehow
536 bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);
537 gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();
538 if (fn->cfgExit) {
539 bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);
540 bld.mkMovToReg(0, gpEmitAddress);
541 }
542 }
543 return true;
544 }
545
546 bool
547 NVC0LoweringPass::visit(BasicBlock *bb)
548 {
549 return true;
550 }
551
552 inline Value *
553 NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)
554 {
555 uint8_t b = prog->driver->io.resInfoCBSlot;
556 uint32_t off = prog->driver->io.texBindBase + slot * 4;
557 return bld.
558 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
559 }
560
561 // move array source to first slot, convert to u16, add indirections
562 bool
563 NVC0LoweringPass::handleTEX(TexInstruction *i)
564 {
565 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
566 const int arg = i->tex.target.getArgCount();
567 const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);
568 const int chipset = prog->getTarget()->getChipset();
569
570 // Arguments to the TEX instruction are a little insane. Even though the
571 // encoding is identical between SM20 and SM30, the arguments mean
572 // different things between Fermi and Kepler+. A lot of arguments are
573 // optional based on flags passed to the instruction. This summarizes the
574 // order of things.
575 //
576 // Fermi:
577 // array/indirect
578 // coords
579 // sample
580 // lod bias
581 // depth compare
582 // offsets:
583 // - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)
584 // - other: 4 bits each, single reg
585 //
586 // Kepler+:
587 // indirect handle
588 // array (+ offsets for txd in upper 16 bits)
589 // coords
590 // sample
591 // lod bias
592 // depth compare
593 // offsets (same as fermi, except txd which takes it with array)
594
595 if (chipset >= NVISA_GK104_CHIPSET) {
596 if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
597 // XXX this ignores tsc, and assumes a 1:1 mapping
598 assert(i->tex.rIndirectSrc >= 0);
599 Value *hnd = loadTexHandle(
600 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
601 i->getIndirectR(), bld.mkImm(2)),
602 i->tex.r);
603 i->tex.r = 0xff;
604 i->tex.s = 0x1f;
605 i->setIndirectR(hnd);
606 i->setIndirectS(NULL);
607 } else if (i->tex.r == i->tex.s) {
608 i->tex.r += prog->driver->io.texBindBase / 4;
609 i->tex.s = 0; // only a single cX[] value possible here
610 } else {
611 Value *hnd = bld.getScratch();
612 Value *rHnd = loadTexHandle(NULL, i->tex.r);
613 Value *sHnd = loadTexHandle(NULL, i->tex.s);
614
615 bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);
616
617 i->tex.r = 0; // not used for indirect tex
618 i->tex.s = 0;
619 i->setIndirectR(hnd);
620 }
621 if (i->tex.target.isArray()) {
622 LValue *layer = new_LValue(func, FILE_GPR);
623 Value *src = i->getSrc(lyr);
624 const int sat = (i->op == OP_TXF) ? 1 : 0;
625 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
626 bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;
627 for (int s = dim; s >= 1; --s)
628 i->setSrc(s, i->getSrc(s - 1));
629 i->setSrc(0, layer);
630 }
631 // Move the indirect reference to the first place
632 if (i->tex.rIndirectSrc >= 0) {
633 Value *hnd = i->getIndirectR();
634
635 i->setIndirectR(NULL);
636 i->moveSources(0, 1);
637 i->setSrc(0, hnd);
638 i->tex.rIndirectSrc = 0;
639 i->tex.sIndirectSrc = -1;
640 }
641 } else
642 // (nvc0) generate and move the tsc/tic/array source to the front
643 if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
644 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
645
646 Value *ticRel = i->getIndirectR();
647 Value *tscRel = i->getIndirectS();
648
649 if (ticRel) {
650 i->setSrc(i->tex.rIndirectSrc, NULL);
651 if (i->tex.r)
652 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
653 ticRel, bld.mkImm(i->tex.r));
654 }
655 if (tscRel) {
656 i->setSrc(i->tex.sIndirectSrc, NULL);
657 if (i->tex.s)
658 tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
659 tscRel, bld.mkImm(i->tex.s));
660 }
661
662 Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;
663 for (int s = dim; s >= 1; --s)
664 i->setSrc(s, i->getSrc(s - 1));
665 i->setSrc(0, arrayIndex);
666
667 if (arrayIndex) {
668 int sat = (i->op == OP_TXF) ? 1 : 0;
669 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
670 bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;
671 } else {
672 bld.loadImm(src, 0);
673 }
674
675 if (ticRel)
676 bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);
677 if (tscRel)
678 bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);
679
680 i->setSrc(0, src);
681 }
682
683 // For nvc0, the sample id has to be in the second operand, as the offset
684 // does. Right now we don't know how to pass both in, and this case can't
685 // happen with OpenGL. On nve0, the sample id is part of the texture
686 // coordinate argument.
687 assert(chipset >= NVISA_GK104_CHIPSET ||
688 !i->tex.useOffsets || !i->tex.target.isMS());
689
690 // offset is between lod and dc
691 if (i->tex.useOffsets) {
692 int n, c;
693 int s = i->srcCount(0xff, true);
694 if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {
695 if (i->tex.target.isShadow())
696 s--;
697 if (i->srcExists(s)) // move potential predicate out of the way
698 i->moveSources(s, 1);
699 if (i->tex.useOffsets == 4 && i->srcExists(s + 1))
700 i->moveSources(s + 1, 1);
701 }
702 if (i->op == OP_TXG) {
703 // Either there is 1 offset, which goes into the 2 low bytes of the
704 // first source, or there are 4 offsets, which go into 2 sources (8
705 // values, 1 byte each).
706 Value *offs[2] = {NULL, NULL};
707 for (n = 0; n < i->tex.useOffsets; n++) {
708 for (c = 0; c < 2; ++c) {
709 if ((n % 2) == 0 && c == 0)
710 offs[n / 2] = i->offset[n][c].get();
711 else
712 bld.mkOp3(OP_INSBF, TYPE_U32,
713 offs[n / 2],
714 i->offset[n][c].get(),
715 bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),
716 offs[n / 2]);
717 }
718 }
719 i->setSrc(s, offs[0]);
720 if (offs[1])
721 i->setSrc(s + 1, offs[1]);
722 } else {
723 unsigned imm = 0;
724 assert(i->tex.useOffsets == 1);
725 for (c = 0; c < 3; ++c) {
726 ImmediateValue val;
727 assert(i->offset[0][c].getImmediate(val));
728 imm |= (val.reg.data.u32 & 0xf) << (c * 4);
729 }
730 if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {
731 // The offset goes into the upper 16 bits of the array index. So
732 // create it if it's not already there, and INSBF it if it already
733 // is.
734 s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;
735 if (i->tex.target.isArray()) {
736 bld.mkOp3(OP_INSBF, TYPE_U32, i->getSrc(0),
737 bld.loadImm(NULL, imm), bld.mkImm(0xc10),
738 i->getSrc(s));
739 } else {
740 i->moveSources(s, 1);
741 i->setSrc(s, bld.loadImm(NULL, imm << 16));
742 }
743 } else {
744 i->setSrc(s, bld.loadImm(NULL, imm));
745 }
746 }
747 }
748
749 if (chipset >= NVISA_GK104_CHIPSET) {
750 //
751 // If TEX requires more than 4 sources, the 2nd register tuple must be
752 // aligned to 4, even if it consists of just a single 4-byte register.
753 //
754 // XXX HACK: We insert 0 sources to avoid the 5 or 6 regs case.
755 //
756 int s = i->srcCount(0xff, true);
757 if (s > 4 && s < 7) {
758 if (i->srcExists(s)) // move potential predicate out of the way
759 i->moveSources(s, 7 - s);
760 while (s < 7)
761 i->setSrc(s++, bld.loadImm(NULL, 0));
762 }
763 }
764
765 return true;
766 }
767
768 bool
769 NVC0LoweringPass::handleManualTXD(TexInstruction *i)
770 {
771 static const uint8_t qOps[4][2] =
772 {
773 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0
774 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1
775 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2
776 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3
777 };
778 Value *def[4][4];
779 Value *crd[3];
780 Instruction *tex;
781 Value *zero = bld.loadImm(bld.getSSA(), 0);
782 int l, c;
783 const int dim = i->tex.target.getDim();
784 const int array = i->tex.target.isArray();
785
786 i->op = OP_TEX; // no need to clone dPdx/dPdy later
787
788 for (c = 0; c < dim; ++c)
789 crd[c] = bld.getScratch();
790
791 bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
792 for (l = 0; l < 4; ++l) {
793 // mov coordinates from lane l to all lanes
794 for (c = 0; c < dim; ++c)
795 bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);
796 // add dPdx from lane l to lanes dx
797 for (c = 0; c < dim; ++c)
798 bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);
799 // add dPdy from lane l to lanes dy
800 for (c = 0; c < dim; ++c)
801 bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);
802 // texture
803 bld.insert(tex = cloneForward(func, i));
804 for (c = 0; c < dim; ++c)
805 tex->setSrc(c + array, crd[c]);
806 // save results
807 for (c = 0; i->defExists(c); ++c) {
808 Instruction *mov;
809 def[c][l] = bld.getSSA();
810 mov = bld.mkMov(def[c][l], tex->getDef(c));
811 mov->fixed = 1;
812 mov->lanes = 1 << l;
813 }
814 }
815 bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
816
817 for (c = 0; i->defExists(c); ++c) {
818 Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
819 for (l = 0; l < 4; ++l)
820 u->setSrc(l, def[c][l]);
821 }
822
823 i->bb->remove(i);
824 return true;
825 }
826
827 bool
828 NVC0LoweringPass::handleTXD(TexInstruction *txd)
829 {
830 int dim = txd->tex.target.getDim();
831 unsigned arg = txd->tex.target.getArgCount();
832 unsigned expected_args = arg;
833 const int chipset = prog->getTarget()->getChipset();
834
835 if (chipset >= NVISA_GK104_CHIPSET) {
836 if (!txd->tex.target.isArray() && txd->tex.useOffsets)
837 expected_args++;
838 if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)
839 expected_args++;
840 } else {
841 if (txd->tex.useOffsets)
842 expected_args++;
843 if (!txd->tex.target.isArray() && (
844 txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))
845 expected_args++;
846 }
847
848 if (expected_args > 4 ||
849 dim > 2 ||
850 txd->tex.target.isShadow() ||
851 txd->tex.target.isCube())
852 txd->op = OP_TEX;
853
854 handleTEX(txd);
855 while (txd->srcExists(arg))
856 ++arg;
857
858 txd->tex.derivAll = true;
859 if (txd->op == OP_TEX)
860 return handleManualTXD(txd);
861
862 assert(arg == expected_args);
863 for (int c = 0; c < dim; ++c) {
864 txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);
865 txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);
866 txd->dPdx[c].set(NULL);
867 txd->dPdy[c].set(NULL);
868 }
869 return true;
870 }
871
872 bool
873 NVC0LoweringPass::handleTXQ(TexInstruction *txq)
874 {
875 // TODO: indirect resource/sampler index
876 return true;
877 }
878
879 bool
880 NVC0LoweringPass::handleTXLQ(TexInstruction *i)
881 {
882 /* The outputs are inverted compared to what the TGSI instruction
883 * expects. Take that into account in the mask.
884 */
885 assert((i->tex.mask & ~3) == 0);
886 if (i->tex.mask == 1)
887 i->tex.mask = 2;
888 else if (i->tex.mask == 2)
889 i->tex.mask = 1;
890 handleTEX(i);
891 bld.setPosition(i, true);
892
893 /* The returned values are not quite what we want:
894 * (a) convert from s16/u16 to f32
895 * (b) multiply by 1/256
896 */
897 for (int def = 0; def < 2; ++def) {
898 if (!i->defExists(def))
899 continue;
900 enum DataType type = TYPE_S16;
901 if (i->tex.mask == 2 || def > 0)
902 type = TYPE_U16;
903 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));
904 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),
905 i->getDef(def), bld.loadImm(NULL, 1.0f / 256));
906 }
907 if (i->tex.mask == 3) {
908 LValue *t = new_LValue(func, FILE_GPR);
909 bld.mkMov(t, i->getDef(0));
910 bld.mkMov(i->getDef(0), i->getDef(1));
911 bld.mkMov(i->getDef(1), t);
912 }
913 return true;
914 }
915
916
917 bool
918 NVC0LoweringPass::handleATOM(Instruction *atom)
919 {
920 SVSemantic sv;
921
922 switch (atom->src(0).getFile()) {
923 case FILE_MEMORY_LOCAL:
924 sv = SV_LBASE;
925 break;
926 case FILE_MEMORY_SHARED:
927 sv = SV_SBASE;
928 break;
929 default:
930 assert(atom->src(0).getFile() == FILE_MEMORY_GLOBAL);
931 return true;
932 }
933 Value *base =
934 bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));
935 Value *ptr = atom->getIndirect(0, 0);
936
937 atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));
938 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
939 if (ptr)
940 base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);
941 atom->setIndirect(0, 0, base);
942
943 return true;
944 }
945
946 bool
947 NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)
948 {
949 if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&
950 cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)
951 return false;
952 bld.setPosition(cas, true);
953
954 if (needCctl) {
955 Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));
956 cctl->setIndirect(0, 0, cas->getIndirect(0, 0));
957 cctl->fixed = 1;
958 cctl->subOp = NV50_IR_SUBOP_CCTL_IV;
959 if (cas->isPredicated())
960 cctl->setPredicate(cas->cc, cas->getPredicate());
961 }
962
963 if (cas->defExists(0) && cas->subOp == NV50_IR_SUBOP_ATOM_CAS) {
964 // CAS is crazy. It's 2nd source is a double reg, and the 3rd source
965 // should be set to the high part of the double reg or bad things will
966 // happen elsewhere in the universe.
967 // Also, it sometimes returns the new value instead of the old one
968 // under mysterious circumstances.
969 Value *dreg = bld.getSSA(8);
970 bld.setPosition(cas, false);
971 bld.mkOp2(OP_MERGE, TYPE_U64, dreg, cas->getSrc(1), cas->getSrc(2));
972 cas->setSrc(1, dreg);
973 }
974
975 return true;
976 }
977
978 inline Value *
979 NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off)
980 {
981 uint8_t b = prog->driver->io.resInfoCBSlot;
982 off += prog->driver->io.suInfoBase;
983 return bld.
984 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
985 }
986
987 inline Value *
988 NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)
989 {
990 uint8_t b = prog->driver->io.msInfoCBSlot;
991 off += prog->driver->io.msInfoBase;
992 return bld.
993 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
994 }
995
996 /* On nvc0, surface info is obtained via the surface binding points passed
997 * to the SULD/SUST instructions.
998 * On nve4, surface info is stored in c[] and is used by various special
999 * instructions, e.g. for clamping coordiantes or generating an address.
1000 * They couldn't just have added an equivalent to TIC now, couldn't they ?
1001 */
1002 #define NVE4_SU_INFO_ADDR 0x00
1003 #define NVE4_SU_INFO_FMT 0x04
1004 #define NVE4_SU_INFO_DIM_X 0x08
1005 #define NVE4_SU_INFO_PITCH 0x0c
1006 #define NVE4_SU_INFO_DIM_Y 0x10
1007 #define NVE4_SU_INFO_ARRAY 0x14
1008 #define NVE4_SU_INFO_DIM_Z 0x18
1009 #define NVE4_SU_INFO_UNK1C 0x1c
1010 #define NVE4_SU_INFO_WIDTH 0x20
1011 #define NVE4_SU_INFO_HEIGHT 0x24
1012 #define NVE4_SU_INFO_DEPTH 0x28
1013 #define NVE4_SU_INFO_TARGET 0x2c
1014 #define NVE4_SU_INFO_CALL 0x30
1015 #define NVE4_SU_INFO_RAW_X 0x34
1016 #define NVE4_SU_INFO_MS_X 0x38
1017 #define NVE4_SU_INFO_MS_Y 0x3c
1018
1019 #define NVE4_SU_INFO__STRIDE 0x40
1020
1021 #define NVE4_SU_INFO_DIM(i) (0x08 + (i) * 8)
1022 #define NVE4_SU_INFO_SIZE(i) (0x20 + (i) * 4)
1023 #define NVE4_SU_INFO_MS(i) (0x38 + (i) * 4)
1024
1025 static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)
1026 {
1027 switch (su->tex.target.getEnum()) {
1028 case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);
1029 case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1030 case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1031 case TEX_TARGET_1D_ARRAY: return (c == 1) ?
1032 NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :
1033 NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1034 case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1035 case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1036 case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1037 case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1038 case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1039 case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1040 case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1041 default:
1042 assert(0);
1043 return 0;
1044 }
1045 }
1046
1047 void
1048 NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex)
1049 {
1050 const uint16_t base = tex->tex.r * NVE4_SU_INFO__STRIDE;
1051 const int arg = tex->tex.target.getArgCount();
1052
1053 if (tex->tex.target == TEX_TARGET_2D_MS)
1054 tex->tex.target = TEX_TARGET_2D;
1055 else
1056 if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY)
1057 tex->tex.target = TEX_TARGET_2D_ARRAY;
1058 else
1059 return;
1060
1061 Value *x = tex->getSrc(0);
1062 Value *y = tex->getSrc(1);
1063 Value *s = tex->getSrc(arg - 1);
1064
1065 Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA();
1066
1067 Value *ms_x = loadResInfo32(NULL, base + NVE4_SU_INFO_MS(0));
1068 Value *ms_y = loadResInfo32(NULL, base + NVE4_SU_INFO_MS(1));
1069
1070 bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x);
1071 bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y);
1072
1073 s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7));
1074 s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3));
1075
1076 Value *dx = loadMsInfo32(ts, 0x0);
1077 Value *dy = loadMsInfo32(ts, 0x4);
1078
1079 bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx);
1080 bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy);
1081
1082 tex->setSrc(0, tx);
1083 tex->setSrc(1, ty);
1084 tex->moveSources(arg, -1);
1085 }
1086
1087 // Sets 64-bit "generic address", predicate and format sources for SULD/SUST.
1088 // They're computed from the coordinates using the surface info in c[] space.
1089 void
1090 NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su)
1091 {
1092 Instruction *insn;
1093 const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP;
1094 const bool raw =
1095 su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB;
1096 const int idx = su->tex.r;
1097 const int dim = su->tex.target.getDim();
1098 const int arg = dim + (su->tex.target.isArray() ? 1 : 0);
1099 const uint16_t base = idx * NVE4_SU_INFO__STRIDE;
1100 int c;
1101 Value *zero = bld.mkImm(0);
1102 Value *p1 = NULL;
1103 Value *v;
1104 Value *src[3];
1105 Value *bf, *eau, *off;
1106 Value *addr, *pred;
1107
1108 off = bld.getScratch(4);
1109 bf = bld.getScratch(4);
1110 addr = bld.getSSA(8);
1111 pred = bld.getScratch(1, FILE_PREDICATE);
1112
1113 bld.setPosition(su, false);
1114
1115 adjustCoordinatesMS(su);
1116
1117 // calculate clamped coordinates
1118 for (c = 0; c < arg; ++c) {
1119 src[c] = bld.getScratch();
1120 if (c == 0 && raw)
1121 v = loadResInfo32(NULL, base + NVE4_SU_INFO_RAW_X);
1122 else
1123 v = loadResInfo32(NULL, base + NVE4_SU_INFO_DIM(c));
1124 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero)
1125 ->subOp = getSuClampSubOp(su, c);
1126 }
1127 for (; c < 3; ++c)
1128 src[c] = zero;
1129
1130 // set predicate output
1131 if (su->tex.target == TEX_TARGET_BUFFER) {
1132 src[0]->getInsn()->setFlagsDef(1, pred);
1133 } else
1134 if (su->tex.target.isArray()) {
1135 p1 = bld.getSSA(1, FILE_PREDICATE);
1136 src[dim]->getInsn()->setFlagsDef(1, p1);
1137 }
1138
1139 // calculate pixel offset
1140 if (dim == 1) {
1141 if (su->tex.target != TEX_TARGET_BUFFER)
1142 bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff));
1143 } else
1144 if (dim == 3) {
1145 v = loadResInfo32(NULL, base + NVE4_SU_INFO_UNK1C);
1146 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1])
1147 ->subOp = NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1148
1149 v = loadResInfo32(NULL, base + NVE4_SU_INFO_PITCH);
1150 bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0])
1151 ->subOp = NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l
1152 } else {
1153 assert(dim == 2);
1154 v = loadResInfo32(NULL, base + NVE4_SU_INFO_PITCH);
1155 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[1], v, src[0])
1156 ->subOp = su->tex.target.isArray() ?
1157 NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1158 }
1159
1160 // calculate effective address part 1
1161 if (su->tex.target == TEX_TARGET_BUFFER) {
1162 if (raw) {
1163 bf = src[0];
1164 } else {
1165 v = loadResInfo32(NULL, base + NVE4_SU_INFO_FMT);
1166 bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero)
1167 ->subOp = NV50_IR_SUBOP_V1(7,6,8|2);
1168 }
1169 } else {
1170 Value *y = src[1];
1171 Value *z = src[2];
1172 uint16_t subOp = 0;
1173
1174 switch (dim) {
1175 case 1:
1176 y = zero;
1177 z = zero;
1178 break;
1179 case 2:
1180 z = off;
1181 if (!su->tex.target.isArray()) {
1182 z = loadResInfo32(NULL, base + NVE4_SU_INFO_UNK1C);
1183 subOp = NV50_IR_SUBOP_SUBFM_3D;
1184 }
1185 break;
1186 default:
1187 subOp = NV50_IR_SUBOP_SUBFM_3D;
1188 assert(dim == 3);
1189 break;
1190 }
1191 insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z);
1192 insn->subOp = subOp;
1193 insn->setFlagsDef(1, pred);
1194 }
1195
1196 // part 2
1197 v = loadResInfo32(NULL, base + NVE4_SU_INFO_ADDR);
1198
1199 if (su->tex.target == TEX_TARGET_BUFFER) {
1200 eau = v;
1201 } else {
1202 eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v);
1203 }
1204 // add array layer offset
1205 if (su->tex.target.isArray()) {
1206 v = loadResInfo32(NULL, base + NVE4_SU_INFO_ARRAY);
1207 if (dim == 1)
1208 bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau)
1209 ->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32
1210 else
1211 bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau)
1212 ->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32
1213 // combine predicates
1214 assert(p1);
1215 bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1);
1216 }
1217
1218 if (atom) {
1219 Value *lo = bf;
1220 if (su->tex.target == TEX_TARGET_BUFFER) {
1221 lo = zero;
1222 bld.mkMov(off, bf);
1223 }
1224 // bf == g[] address & 0xff
1225 // eau == g[] address >> 8
1226 bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau);
1227 bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau);
1228 } else
1229 if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) {
1230 // Convert from u32 to u8 address format, which is what the library code
1231 // doing SULDP currently uses.
1232 // XXX: can SUEAU do this ?
1233 // XXX: does it matter that we don't mask high bytes in bf ?
1234 // Grrr.
1235 bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8));
1236 bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off);
1237 }
1238
1239 bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau);
1240
1241 if (atom && su->tex.target == TEX_TARGET_BUFFER)
1242 bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off);
1243
1244 // let's just set it 0 for raw access and hope it works
1245 v = raw ?
1246 bld.mkImm(0) : loadResInfo32(NULL, base + NVE4_SU_INFO_FMT);
1247
1248 // get rid of old coordinate sources, make space for fmt info and predicate
1249 su->moveSources(arg, 3 - arg);
1250 // set 64 bit address and 32-bit format sources
1251 su->setSrc(0, addr);
1252 su->setSrc(1, v);
1253 su->setSrc(2, pred);
1254 }
1255
1256 void
1257 NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su)
1258 {
1259 processSurfaceCoordsNVE4(su);
1260
1261 // Who do we hate more ? The person who decided that nvc0's SULD doesn't
1262 // have to support conversion or the person who decided that, in OpenCL,
1263 // you don't have to specify the format here like you do in OpenGL ?
1264
1265 if (su->op == OP_SULDP) {
1266 // We don't patch shaders. Ever.
1267 // You get an indirect call to our library blob here.
1268 // But at least it's uniform.
1269 FlowInstruction *call;
1270 LValue *p[3];
1271 LValue *r[5];
1272 uint16_t base = su->tex.r * NVE4_SU_INFO__STRIDE + NVE4_SU_INFO_CALL;
1273
1274 for (int i = 0; i < 4; ++i)
1275 (r[i] = bld.getScratch(4, FILE_GPR))->reg.data.id = i;
1276 for (int i = 0; i < 3; ++i)
1277 (p[i] = bld.getScratch(1, FILE_PREDICATE))->reg.data.id = i;
1278 (r[4] = bld.getScratch(8, FILE_GPR))->reg.data.id = 4;
1279
1280 bld.mkMov(p[1], bld.mkImm((su->cache == CACHE_CA) ? 1 : 0), TYPE_U8);
1281 bld.mkMov(p[2], bld.mkImm((su->cache == CACHE_CG) ? 1 : 0), TYPE_U8);
1282 bld.mkMov(p[0], su->getSrc(2), TYPE_U8);
1283 bld.mkMov(r[4], su->getSrc(0), TYPE_U64);
1284 bld.mkMov(r[2], su->getSrc(1), TYPE_U32);
1285
1286 call = bld.mkFlow(OP_CALL, NULL, su->cc, su->getPredicate());
1287
1288 call->indirect = 1;
1289 call->absolute = 1;
1290 call->setSrc(0, bld.mkSymbol(FILE_MEMORY_CONST,
1291 prog->driver->io.resInfoCBSlot, TYPE_U32,
1292 prog->driver->io.suInfoBase + base));
1293 call->setSrc(1, r[2]);
1294 call->setSrc(2, r[4]);
1295 for (int i = 0; i < 3; ++i)
1296 call->setSrc(3 + i, p[i]);
1297 for (int i = 0; i < 4; ++i) {
1298 call->setDef(i, r[i]);
1299 bld.mkMov(su->getDef(i), r[i]);
1300 }
1301 call->setDef(4, p[1]);
1302 delete_Instruction(bld.getProgram(), su);
1303 }
1304
1305 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
1306 // FIXME: for out of bounds access, destination value will be undefined !
1307 Value *pred = su->getSrc(2);
1308 CondCode cc = CC_NOT_P;
1309 if (su->getPredicate()) {
1310 pred = bld.getScratch(1, FILE_PREDICATE);
1311 cc = su->cc;
1312 if (cc == CC_NOT_P) {
1313 bld.mkOp2(OP_OR, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1314 } else {
1315 bld.mkOp2(OP_AND, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1316 pred->getInsn()->src(1).mod = Modifier(NV50_IR_MOD_NOT);
1317 }
1318 }
1319 Instruction *red = bld.mkOp(OP_ATOM, su->dType, su->getDef(0));
1320 red->subOp = su->subOp;
1321 if (!gMemBase)
1322 gMemBase = bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0);
1323 red->setSrc(0, gMemBase);
1324 red->setSrc(1, su->getSrc(3));
1325 if (su->subOp == NV50_IR_SUBOP_ATOM_CAS)
1326 red->setSrc(2, su->getSrc(4));
1327 red->setIndirect(0, 0, su->getSrc(0));
1328 red->setPredicate(cc, pred);
1329 delete_Instruction(bld.getProgram(), su);
1330 handleCasExch(red, true);
1331 } else {
1332 su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8;
1333 }
1334 }
1335
1336 bool
1337 NVC0LoweringPass::handleWRSV(Instruction *i)
1338 {
1339 Instruction *st;
1340 Symbol *sym;
1341 uint32_t addr;
1342
1343 // must replace, $sreg are not writeable
1344 addr = targ->getSVAddress(FILE_SHADER_OUTPUT, i->getSrc(0)->asSym());
1345 if (addr >= 0x400)
1346 return false;
1347 sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
1348
1349 st = bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0),
1350 i->getSrc(1));
1351 st->perPatch = i->perPatch;
1352
1353 bld.getBB()->remove(i);
1354 return true;
1355 }
1356
1357 void
1358 NVC0LoweringPass::readTessCoord(LValue *dst, int c)
1359 {
1360 Value *laneid = bld.getSSA();
1361 Value *x, *y;
1362
1363 bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0));
1364
1365 if (c == 0) {
1366 x = dst;
1367 y = NULL;
1368 } else
1369 if (c == 1) {
1370 x = NULL;
1371 y = dst;
1372 } else {
1373 assert(c == 2);
1374 x = bld.getSSA();
1375 y = bld.getSSA();
1376 }
1377 if (x)
1378 bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid);
1379 if (y)
1380 bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid);
1381
1382 if (c == 2) {
1383 bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y);
1384 bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst);
1385 }
1386 }
1387
1388 bool
1389 NVC0LoweringPass::handleRDSV(Instruction *i)
1390 {
1391 Symbol *sym = i->getSrc(0)->asSym();
1392 const SVSemantic sv = sym->reg.data.sv.sv;
1393 Value *vtx = NULL;
1394 Instruction *ld;
1395 uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
1396
1397 if (addr >= 0x400) {
1398 // mov $sreg
1399 if (sym->reg.data.sv.index == 3) {
1400 // TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID
1401 i->op = OP_MOV;
1402 i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0));
1403 }
1404 return true;
1405 }
1406
1407 switch (sv) {
1408 case SV_POSITION:
1409 assert(prog->getType() == Program::TYPE_FRAGMENT);
1410 if (i->srcExists(1)) {
1411 // Pass offset through to the interpolation logic
1412 ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET,
1413 i->getDef(0), addr, NULL);
1414 ld->setSrc(1, i->getSrc(1));
1415 } else {
1416 bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
1417 }
1418 break;
1419 case SV_FACE:
1420 {
1421 Value *face = i->getDef(0);
1422 bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL);
1423 if (i->dType == TYPE_F32) {
1424 bld.mkOp2(OP_AND, TYPE_U32, face, face, bld.mkImm(0x80000000));
1425 bld.mkOp2(OP_XOR, TYPE_U32, face, face, bld.mkImm(0xbf800000));
1426 }
1427 }
1428 break;
1429 case SV_TESS_COORD:
1430 assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL);
1431 readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index);
1432 break;
1433 case SV_NTID:
1434 case SV_NCTAID:
1435 case SV_GRIDID:
1436 assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise
1437 if (sym->reg.data.sv.index == 3) {
1438 i->op = OP_MOV;
1439 i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1));
1440 return true;
1441 }
1442 addr += prog->driver->prop.cp.gridInfoBase;
1443 bld.mkLoad(TYPE_U32, i->getDef(0),
1444 bld.mkSymbol(FILE_MEMORY_CONST, 0, TYPE_U32, addr), NULL);
1445 break;
1446 case SV_SAMPLE_INDEX:
1447 // TODO: Properly pass source as an address in the PIX address space
1448 // (which can be of the form [r0+offset]). But this is currently
1449 // unnecessary.
1450 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1451 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
1452 break;
1453 case SV_SAMPLE_POS: {
1454 Value *off = new_LValue(func, FILE_GPR);
1455 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1456 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
1457 bld.mkOp2(OP_SHL, TYPE_U32, off, i->getDef(0), bld.mkImm(3));
1458 bld.mkLoad(TYPE_F32,
1459 i->getDef(0),
1460 bld.mkSymbol(
1461 FILE_MEMORY_CONST, prog->driver->io.resInfoCBSlot,
1462 TYPE_U32, prog->driver->io.sampleInfoBase +
1463 4 * sym->reg.data.sv.index),
1464 off);
1465 break;
1466 }
1467 case SV_SAMPLE_MASK:
1468 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
1469 ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK;
1470 break;
1471 default:
1472 if (prog->getType() == Program::TYPE_TESSELLATION_EVAL)
1473 vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0));
1474 ld = bld.mkFetch(i->getDef(0), i->dType,
1475 FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx);
1476 ld->perPatch = i->perPatch;
1477 break;
1478 }
1479 bld.getBB()->remove(i);
1480 return true;
1481 }
1482
1483 bool
1484 NVC0LoweringPass::handleDIV(Instruction *i)
1485 {
1486 if (!isFloatType(i->dType))
1487 return true;
1488 bld.setPosition(i, false);
1489 Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(), i->getSrc(1));
1490 i->op = OP_MUL;
1491 i->setSrc(1, rcp->getDef(0));
1492 return true;
1493 }
1494
1495 bool
1496 NVC0LoweringPass::handleMOD(Instruction *i)
1497 {
1498 if (i->dType != TYPE_F32)
1499 return true;
1500 LValue *value = bld.getScratch();
1501 bld.mkOp1(OP_RCP, TYPE_F32, value, i->getSrc(1));
1502 bld.mkOp2(OP_MUL, TYPE_F32, value, i->getSrc(0), value);
1503 bld.mkOp1(OP_TRUNC, TYPE_F32, value, value);
1504 bld.mkOp2(OP_MUL, TYPE_F32, value, i->getSrc(1), value);
1505 i->op = OP_SUB;
1506 i->setSrc(1, value);
1507 return true;
1508 }
1509
1510 bool
1511 NVC0LoweringPass::handleSQRT(Instruction *i)
1512 {
1513 Instruction *rsq = bld.mkOp1(OP_RSQ, TYPE_F32,
1514 bld.getSSA(), i->getSrc(0));
1515 i->op = OP_MUL;
1516 i->setSrc(1, rsq->getDef(0));
1517
1518 return true;
1519 }
1520
1521 bool
1522 NVC0LoweringPass::handlePOW(Instruction *i)
1523 {
1524 LValue *val = bld.getScratch();
1525
1526 bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
1527 bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
1528 bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
1529
1530 i->op = OP_EX2;
1531 i->setSrc(0, val);
1532 i->setSrc(1, NULL);
1533
1534 return true;
1535 }
1536
1537 bool
1538 NVC0LoweringPass::handleEXPORT(Instruction *i)
1539 {
1540 if (prog->getType() == Program::TYPE_FRAGMENT) {
1541 int id = i->getSrc(0)->reg.data.offset / 4;
1542
1543 if (i->src(0).isIndirect(0)) // TODO, ugly
1544 return false;
1545 i->op = OP_MOV;
1546 i->subOp = NV50_IR_SUBOP_MOV_FINAL;
1547 i->src(0).set(i->src(1));
1548 i->setSrc(1, NULL);
1549 i->setDef(0, new_LValue(func, FILE_GPR));
1550 i->getDef(0)->reg.data.id = id;
1551
1552 prog->maxGPR = MAX2(prog->maxGPR, id);
1553 } else
1554 if (prog->getType() == Program::TYPE_GEOMETRY) {
1555 i->setIndirect(0, 1, gpEmitAddress);
1556 }
1557 return true;
1558 }
1559
1560 bool
1561 NVC0LoweringPass::handleOUT(Instruction *i)
1562 {
1563 Instruction *prev = i->prev;
1564 ImmediateValue stream, prevStream;
1565
1566 // Only merge if the stream ids match. Also, note that the previous
1567 // instruction would have already been lowered, so we take arg1 from it.
1568 if (i->op == OP_RESTART && prev && prev->op == OP_EMIT &&
1569 i->src(0).getImmediate(stream) &&
1570 prev->src(1).getImmediate(prevStream) &&
1571 stream.reg.data.u32 == prevStream.reg.data.u32) {
1572 i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART;
1573 delete_Instruction(prog, i);
1574 } else {
1575 assert(gpEmitAddress);
1576 i->setDef(0, gpEmitAddress);
1577 i->setSrc(1, i->getSrc(0));
1578 i->setSrc(0, gpEmitAddress);
1579 }
1580 return true;
1581 }
1582
1583 // Generate a binary predicate if an instruction is predicated by
1584 // e.g. an f32 value.
1585 void
1586 NVC0LoweringPass::checkPredicate(Instruction *insn)
1587 {
1588 Value *pred = insn->getPredicate();
1589 Value *pdst;
1590
1591 if (!pred || pred->reg.file == FILE_PREDICATE)
1592 return;
1593 pdst = new_LValue(func, FILE_PREDICATE);
1594
1595 // CAUTION: don't use pdst->getInsn, the definition might not be unique,
1596 // delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass
1597
1598 bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred);
1599
1600 insn->setPredicate(insn->cc, pdst);
1601 }
1602
1603 //
1604 // - add quadop dance for texturing
1605 // - put FP outputs in GPRs
1606 // - convert instruction sequences
1607 //
1608 bool
1609 NVC0LoweringPass::visit(Instruction *i)
1610 {
1611 bld.setPosition(i, false);
1612
1613 if (i->cc != CC_ALWAYS)
1614 checkPredicate(i);
1615
1616 switch (i->op) {
1617 case OP_TEX:
1618 case OP_TXB:
1619 case OP_TXL:
1620 case OP_TXF:
1621 case OP_TXG:
1622 return handleTEX(i->asTex());
1623 case OP_TXD:
1624 return handleTXD(i->asTex());
1625 case OP_TXLQ:
1626 return handleTXLQ(i->asTex());
1627 case OP_TXQ:
1628 return handleTXQ(i->asTex());
1629 case OP_EX2:
1630 bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
1631 i->setSrc(0, i->getDef(0));
1632 break;
1633 case OP_POW:
1634 return handlePOW(i);
1635 case OP_DIV:
1636 return handleDIV(i);
1637 case OP_MOD:
1638 return handleMOD(i);
1639 case OP_SQRT:
1640 return handleSQRT(i);
1641 case OP_EXPORT:
1642 return handleEXPORT(i);
1643 case OP_EMIT:
1644 case OP_RESTART:
1645 return handleOUT(i);
1646 case OP_RDSV:
1647 return handleRDSV(i);
1648 case OP_WRSV:
1649 return handleWRSV(i);
1650 case OP_LOAD:
1651 if (i->src(0).getFile() == FILE_SHADER_INPUT) {
1652 if (prog->getType() == Program::TYPE_COMPUTE) {
1653 i->getSrc(0)->reg.file = FILE_MEMORY_CONST;
1654 i->getSrc(0)->reg.fileIndex = 0;
1655 } else
1656 if (prog->getType() == Program::TYPE_GEOMETRY &&
1657 i->src(0).isIndirect(0)) {
1658 // XXX: this assumes vec4 units
1659 Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1660 i->getIndirect(0, 0), bld.mkImm(4));
1661 i->setIndirect(0, 0, ptr);
1662 } else {
1663 i->op = OP_VFETCH;
1664 assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP
1665 }
1666 } else if (i->src(0).getFile() == FILE_MEMORY_CONST) {
1667 if (i->src(0).isIndirect(1)) {
1668 Value *ptr;
1669 if (i->src(0).isIndirect(0))
1670 ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(),
1671 i->getIndirect(0, 1), bld.mkImm(0x1010),
1672 i->getIndirect(0, 0));
1673 else
1674 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1675 i->getIndirect(0, 1), bld.mkImm(16));
1676 i->setIndirect(0, 1, NULL);
1677 i->setIndirect(0, 0, ptr);
1678 i->subOp = NV50_IR_SUBOP_LDC_IS;
1679 }
1680 }
1681 break;
1682 case OP_ATOM:
1683 {
1684 const bool cctl = i->src(0).getFile() == FILE_MEMORY_GLOBAL;
1685 handleATOM(i);
1686 handleCasExch(i, cctl);
1687 }
1688 break;
1689 case OP_SULDB:
1690 case OP_SULDP:
1691 case OP_SUSTB:
1692 case OP_SUSTP:
1693 case OP_SUREDB:
1694 case OP_SUREDP:
1695 if (targ->getChipset() >= NVISA_GK104_CHIPSET)
1696 handleSurfaceOpNVE4(i->asTex());
1697 break;
1698 default:
1699 break;
1700 }
1701 return true;
1702 }
1703
1704 bool
1705 TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const
1706 {
1707 if (stage == CG_STAGE_PRE_SSA) {
1708 NVC0LoweringPass pass(prog);
1709 return pass.run(prog, false, true);
1710 } else
1711 if (stage == CG_STAGE_POST_RA) {
1712 NVC0LegalizePostRA pass(prog);
1713 return pass.run(prog, false, true);
1714 } else
1715 if (stage == CG_STAGE_SSA) {
1716 NVC0LegalizeSSA pass;
1717 return pass.run(prog, false, true);
1718 }
1719 return false;
1720 }
1721
1722 } // namespace nv50_ir