projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nvc0: add support for BGRA8 images
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_lowering_nvc0.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 #include "codegen/nv50_ir_target_nvc0.h"
27 #include "codegen/nv50_ir_lowering_nvc0.h"
28
29 #include <limits>
30
31 namespace nv50_ir {
32
33 #define QOP_ADD 0
34 #define QOP_SUBR 1
35 #define QOP_SUB 2
36 #define QOP_MOV2 3
37
38 // UL UR LL LR
39 #define QUADOP(q, r, s, t) \
40 ((QOP_##q << 6) | (QOP_##r << 4) | \
41 (QOP_##s << 2) | (QOP_##t << 0))
42
43 void
44 NVC0LegalizeSSA::handleDIV(Instruction *i)
45 {
46 FlowInstruction *call;
47 int builtin;
48 Value *def[2];
49
50 bld.setPosition(i, false);
51 def[0] = bld.mkMovToReg(0, i->getSrc(0))->getDef(0);
52 def[1] = bld.mkMovToReg(1, i->getSrc(1))->getDef(0);
53 switch (i->dType) {
54 case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;
55 case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;
56 default:
57 return;
58 }
59 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
60 bld.mkMov(i->getDef(0), def[(i->op == OP_DIV) ? 0 : 1]);
61 bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);
62 bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);
63
64 call->fixed = 1;
65 call->absolute = call->builtin = 1;
66 call->target.builtin = builtin;
67 delete_Instruction(prog, i);
68 }
69
70 void
71 NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)
72 {
73 assert(i->dType == TYPE_F64);
74 // There are instructions that will compute the high 32 bits of the 64-bit
75 // float. We will just stick 0 in the bottom 32 bits.
76
77 bld.setPosition(i, false);
78
79 // 1. Take the source and it up.
80 Value *src[2], *dst[2], *def = i->getDef(0);
81 bld.mkSplit(src, 4, i->getSrc(0));
82
83 // 2. We don't care about the low 32 bits of the destination. Stick a 0 in.
84 dst[0] = bld.loadImm(NULL, 0);
85 dst[1] = bld.getSSA();
86
87 // 3. The new version of the instruction takes the high 32 bits of the
88 // source and outputs the high 32 bits of the destination.
89 i->setSrc(0, src[1]);
90 i->setDef(0, dst[1]);
91 i->setType(TYPE_F32);
92 i->subOp = NV50_IR_SUBOP_RCPRSQ_64H;
93
94 // 4. Recombine the two dst pieces back into the original destination.
95 bld.setPosition(i, true);
96 bld.mkOp2(OP_MERGE, TYPE_U64, def, dst[0], dst[1]);
97 }
98
99 void
100 NVC0LegalizeSSA::handleFTZ(Instruction *i)
101 {
102 // Only want to flush float inputs
103 assert(i->sType == TYPE_F32);
104
105 // If we're already flushing denorms (and NaN's) to zero, no need for this.
106 if (i->dnz)
107 return;
108
109 // Only certain classes of operations can flush
110 OpClass cls = prog->getTarget()->getOpClass(i->op);
111 if (cls != OPCLASS_ARITH && cls != OPCLASS_COMPARE &&
112 cls != OPCLASS_CONVERT)
113 return;
114
115 i->ftz = true;
116 }
117
118 bool
119 NVC0LegalizeSSA::visit(Function *fn)
120 {
121 bld.setProgram(fn->getProgram());
122 return true;
123 }
124
125 bool
126 NVC0LegalizeSSA::visit(BasicBlock *bb)
127 {
128 Instruction *next;
129 for (Instruction *i = bb->getEntry(); i; i = next) {
130 next = i->next;
131 if (i->sType == TYPE_F32) {
132 if (prog->getType() != Program::TYPE_COMPUTE)
133 handleFTZ(i);
134 continue;
135 }
136 switch (i->op) {
137 case OP_DIV:
138 case OP_MOD:
139 handleDIV(i);
140 break;
141 case OP_RCP:
142 case OP_RSQ:
143 if (i->dType == TYPE_F64)
144 handleRCPRSQ(i);
145 break;
146 default:
147 break;
148 }
149 }
150 return true;
151 }
152
153 NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)
154 : rZero(NULL),
155 carry(NULL),
156 pOne(NULL),
157 needTexBar(prog->getTarget()->getChipset() >= 0xe0)
158 {
159 }
160
161 bool
162 NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,
163 const Instruction *early) const
164 {
165 if (early->bb == later->bb)
166 return early->serial < later->serial;
167 return later->bb->dominatedBy(early->bb);
168 }
169
170 void
171 NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses,
172 Instruction *usei, const Instruction *texi)
173 {
174 bool add = true;
175 bool dominated = insnDominatedBy(usei, texi);
176 // Uses before the tex have to all be included. Just because an earlier
177 // instruction dominates another instruction doesn't mean that there's no
178 // way to get from the tex to the later instruction. For example you could
179 // have nested loops, with the tex in the inner loop, and uses before it in
180 // both loops - even though the outer loop's instruction would dominate the
181 // inner's, we still want a texbar before the inner loop's instruction.
182 //
183 // However we can still use the eliding logic between uses dominated by the
184 // tex instruction, as that is unambiguously correct.
185 if (dominated) {
186 for (std::list<TexUse>::iterator it = uses.begin(); it != uses.end();) {
187 if (it->after) {
188 if (insnDominatedBy(usei, it->insn)) {
189 add = false;
190 break;
191 }
192 if (insnDominatedBy(it->insn, usei)) {
193 it = uses.erase(it);
194 continue;
195 }
196 }
197 ++it;
198 }
199 }
200 if (add)
201 uses.push_back(TexUse(usei, texi, dominated));
202 }
203
204 // While it might be tempting to use the an algorithm that just looks at tex
205 // uses, not all texture results are guaranteed to be used on all paths. In
206 // the case where along some control flow path a texture result is never used,
207 // we might reuse that register for something else, creating a
208 // write-after-write hazard. So we have to manually look through all
209 // instructions looking for ones that reference the registers in question.
210 void
211 NVC0LegalizePostRA::findFirstUses(
212 Instruction *texi, std::list<TexUse> &uses)
213 {
214 int minGPR = texi->def(0).rep()->reg.data.id;
215 int maxGPR = minGPR + texi->def(0).rep()->reg.size / 4 - 1;
216
217 unordered_set<const BasicBlock *> visited;
218 findFirstUsesBB(minGPR, maxGPR, texi->next, texi, uses, visited);
219 }
220
221 void
222 NVC0LegalizePostRA::findFirstUsesBB(
223 int minGPR, int maxGPR, Instruction *start,
224 const Instruction *texi, std::list<TexUse> &uses,
225 unordered_set<const BasicBlock *> &visited)
226 {
227 const BasicBlock *bb = start->bb;
228
229 // We don't process the whole bb the first time around. This is correct,
230 // however we might be in a loop and hit this BB again, and need to process
231 // the full thing. So only mark a bb as visited if we processed it from the
232 // beginning.
233 if (start == bb->getEntry()) {
234 if (visited.find(bb) != visited.end())
235 return;
236 visited.insert(bb);
237 }
238
239 for (Instruction *insn = start; insn != bb->getExit(); insn = insn->next) {
240 if (insn->isNop())
241 continue;
242
243 for (int d = 0; insn->defExists(d); ++d) {
244 const Value *def = insn->def(d).rep();
245 if (insn->def(d).getFile() != FILE_GPR ||
246 def->reg.data.id + def->reg.size / 4 - 1 < minGPR ||
247 def->reg.data.id > maxGPR)
248 continue;
249 addTexUse(uses, insn, texi);
250 return;
251 }
252
253 for (int s = 0; insn->srcExists(s); ++s) {
254 const Value *src = insn->src(s).rep();
255 if (insn->src(s).getFile() != FILE_GPR ||
256 src->reg.data.id + src->reg.size / 4 - 1 < minGPR ||
257 src->reg.data.id > maxGPR)
258 continue;
259 addTexUse(uses, insn, texi);
260 return;
261 }
262 }
263
264 for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
265 findFirstUsesBB(minGPR, maxGPR, BasicBlock::get(ei.getNode())->getEntry(),
266 texi, uses, visited);
267 }
268 }
269
270 // Texture barriers:
271 // This pass is a bit long and ugly and can probably be optimized.
272 //
273 // 1. obtain a list of TEXes and their outputs' first use(s)
274 // 2. calculate the barrier level of each first use (minimal number of TEXes,
275 // over all paths, between the TEX and the use in question)
276 // 3. for each barrier, if all paths from the source TEX to that barrier
277 // contain a barrier of lesser level, it can be culled
278 bool
279 NVC0LegalizePostRA::insertTextureBarriers(Function *fn)
280 {
281 std::list<TexUse> *uses;
282 std::vector<Instruction *> texes;
283 std::vector<int> bbFirstTex;
284 std::vector<int> bbFirstUse;
285 std::vector<int> texCounts;
286 std::vector<TexUse> useVec;
287 ArrayList insns;
288
289 fn->orderInstructions(insns);
290
291 texCounts.resize(fn->allBBlocks.getSize(), 0);
292 bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());
293 bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());
294
295 // tag BB CFG nodes by their id for later
296 for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {
297 BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get());
298 if (bb)
299 bb->cfg.tag = bb->getId();
300 }
301
302 // gather the first uses for each TEX
303 for (int i = 0; i < insns.getSize(); ++i) {
304 Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i));
305 if (isTextureOp(tex->op)) {
306 texes.push_back(tex);
307 if (!texCounts.at(tex->bb->getId()))
308 bbFirstTex[tex->bb->getId()] = texes.size() - 1;
309 texCounts[tex->bb->getId()]++;
310 }
311 }
312 insns.clear();
313 if (texes.empty())
314 return false;
315 uses = new std::list<TexUse>[texes.size()];
316 if (!uses)
317 return false;
318 for (size_t i = 0; i < texes.size(); ++i) {
319 findFirstUses(texes[i], uses[i]);
320 }
321
322 // determine the barrier level at each use
323 for (size_t i = 0; i < texes.size(); ++i) {
324 for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end();
325 ++u) {
326 BasicBlock *tb = texes[i]->bb;
327 BasicBlock *ub = u->insn->bb;
328 if (tb == ub) {
329 u->level = 0;
330 for (size_t j = i + 1; j < texes.size() &&
331 texes[j]->bb == tb && texes[j]->serial < u->insn->serial;
332 ++j)
333 u->level++;
334 } else {
335 u->level = fn->cfg.findLightestPathWeight(&tb->cfg,
336 &ub->cfg, texCounts);
337 if (u->level < 0) {
338 WARN("Failed to find path TEX -> TEXBAR\n");
339 u->level = 0;
340 continue;
341 }
342 // this counted all TEXes in the origin block, correct that
343 u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;
344 // and did not count the TEXes in the destination block, add those
345 for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&
346 texes[j]->bb == ub && texes[j]->serial < u->insn->serial;
347 ++j)
348 u->level++;
349 }
350 assert(u->level >= 0);
351 useVec.push_back(*u);
352 }
353 }
354 delete[] uses;
355
356 // insert the barriers
357 for (size_t i = 0; i < useVec.size(); ++i) {
358 Instruction *prev = useVec[i].insn->prev;
359 if (useVec[i].level < 0)
360 continue;
361 if (prev && prev->op == OP_TEXBAR) {
362 if (prev->subOp > useVec[i].level)
363 prev->subOp = useVec[i].level;
364 prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));
365 } else {
366 Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);
367 bar->fixed = 1;
368 bar->subOp = useVec[i].level;
369 // make use explicit to ease latency calculation
370 bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));
371 useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);
372 }
373 }
374
375 if (fn->getProgram()->optLevel < 3)
376 return true;
377
378 std::vector<Limits> limitT, limitB, limitS; // entry, exit, single
379
380 limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));
381 limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));
382 limitS.resize(fn->allBBlocks.getSize());
383
384 // cull unneeded barriers (should do that earlier, but for simplicity)
385 IteratorRef bi = fn->cfg.iteratorCFG();
386 // first calculate min/max outstanding TEXes for each BB
387 for (bi->reset(); !bi->end(); bi->next()) {
388 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
389 BasicBlock *bb = BasicBlock::get(n);
390 int min = 0;
391 int max = std::numeric_limits<int>::max();
392 for (Instruction *i = bb->getFirst(); i; i = i->next) {
393 if (isTextureOp(i->op)) {
394 min++;
395 if (max < std::numeric_limits<int>::max())
396 max++;
397 } else
398 if (i->op == OP_TEXBAR) {
399 min = MIN2(min, i->subOp);
400 max = MIN2(max, i->subOp);
401 }
402 }
403 // limits when looking at an isolated block
404 limitS[bb->getId()].min = min;
405 limitS[bb->getId()].max = max;
406 }
407 // propagate the min/max values
408 for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {
409 for (bi->reset(); !bi->end(); bi->next()) {
410 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
411 BasicBlock *bb = BasicBlock::get(n);
412 const int bbId = bb->getId();
413 for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {
414 BasicBlock *in = BasicBlock::get(ei.getNode());
415 const int inId = in->getId();
416 limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);
417 limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);
418 }
419 // I just hope this is correct ...
420 if (limitS[bbId].max == std::numeric_limits<int>::max()) {
421 // no barrier
422 limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;
423 limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;
424 } else {
425 // block contained a barrier
426 limitB[bbId].min = MIN2(limitS[bbId].max,
427 limitT[bbId].min + limitS[bbId].min);
428 limitB[bbId].max = MIN2(limitS[bbId].max,
429 limitT[bbId].max + limitS[bbId].min);
430 }
431 }
432 }
433 // finally delete unnecessary barriers
434 for (bi->reset(); !bi->end(); bi->next()) {
435 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
436 BasicBlock *bb = BasicBlock::get(n);
437 Instruction *prev = NULL;
438 Instruction *next;
439 int max = limitT[bb->getId()].max;
440 for (Instruction *i = bb->getFirst(); i; i = next) {
441 next = i->next;
442 if (i->op == OP_TEXBAR) {
443 if (i->subOp >= max) {
444 delete_Instruction(prog, i);
445 i = NULL;
446 } else {
447 max = i->subOp;
448 if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {
449 delete_Instruction(prog, prev);
450 prev = NULL;
451 }
452 }
453 } else
454 if (isTextureOp(i->op)) {
455 max++;
456 }
457 if (i && !i->isNop())
458 prev = i;
459 }
460 }
461 return true;
462 }
463
464 bool
465 NVC0LegalizePostRA::visit(Function *fn)
466 {
467 if (needTexBar)
468 insertTextureBarriers(fn);
469
470 rZero = new_LValue(fn, FILE_GPR);
471 pOne = new_LValue(fn, FILE_PREDICATE);
472 carry = new_LValue(fn, FILE_FLAGS);
473
474 rZero->reg.data.id = (prog->getTarget()->getChipset() >= NVISA_GK20A_CHIPSET) ? 255 : 63;
475 carry->reg.data.id = 0;
476 pOne->reg.data.id = 7;
477
478 return true;
479 }
480
481 void
482 NVC0LegalizePostRA::replaceZero(Instruction *i)
483 {
484 for (int s = 0; i->srcExists(s); ++s) {
485 if (s == 2 && i->op == OP_SUCLAMP)
486 continue;
487 ImmediateValue *imm = i->getSrc(s)->asImm();
488 if (imm) {
489 if (i->op == OP_SELP && s == 2) {
490 i->setSrc(s, pOne);
491 if (imm->reg.data.u64 == 0)
492 i->src(s).mod = i->src(s).mod ^ Modifier(NV50_IR_MOD_NOT);
493 } else if (imm->reg.data.u64 == 0) {
494 i->setSrc(s, rZero);
495 }
496 }
497 }
498 }
499
500 // replace CONT with BRA for single unconditional continue
501 bool
502 NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)
503 {
504 if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)
505 return false;
506 Graph::EdgeIterator ei = bb->cfg.incident();
507 if (ei.getType() != Graph::Edge::BACK)
508 ei.next();
509 if (ei.getType() != Graph::Edge::BACK)
510 return false;
511 BasicBlock *contBB = BasicBlock::get(ei.getNode());
512
513 if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||
514 contBB->getExit()->getPredicate())
515 return false;
516 contBB->getExit()->op = OP_BRA;
517 bb->remove(bb->getEntry()); // delete PRECONT
518
519 ei.next();
520 assert(ei.end() || ei.getType() != Graph::Edge::BACK);
521 return true;
522 }
523
524 // replace branches to join blocks with join ops
525 void
526 NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)
527 {
528 if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)
529 return;
530 for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
531 BasicBlock *in = BasicBlock::get(ei.getNode());
532 Instruction *exit = in->getExit();
533 if (!exit) {
534 in->insertTail(new FlowInstruction(func, OP_JOIN, bb));
535 // there should always be a terminator instruction
536 WARN("inserted missing terminator in BB:%i\n", in->getId());
537 } else
538 if (exit->op == OP_BRA) {
539 exit->op = OP_JOIN;
540 exit->asFlow()->limit = 1; // must-not-propagate marker
541 }
542 }
543 bb->remove(bb->getEntry());
544 }
545
546 bool
547 NVC0LegalizePostRA::visit(BasicBlock *bb)
548 {
549 Instruction *i, *next;
550
551 // remove pseudo operations and non-fixed no-ops, split 64 bit operations
552 for (i = bb->getFirst(); i; i = next) {
553 next = i->next;
554 if (i->op == OP_EMIT || i->op == OP_RESTART) {
555 if (!i->getDef(0)->refCount())
556 i->setDef(0, NULL);
557 if (i->src(0).getFile() == FILE_IMMEDIATE)
558 i->setSrc(0, rZero); // initial value must be 0
559 replaceZero(i);
560 } else
561 if (i->isNop()) {
562 bb->remove(i);
563 } else
564 if (i->op == OP_BAR && i->subOp == NV50_IR_SUBOP_BAR_SYNC &&
565 prog->getType() != Program::TYPE_COMPUTE) {
566 // It seems like barriers are never required for tessellation since
567 // the warp size is 32, and there are always at most 32 tcs threads.
568 bb->remove(i);
569 } else
570 if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LDC_IS) {
571 int offset = i->src(0).get()->reg.data.offset;
572 if (abs(offset) > 0x10000)
573 i->src(0).get()->reg.fileIndex += offset >> 16;
574 i->src(0).get()->reg.data.offset = (int)(short)offset;
575 } else {
576 // TODO: Move this to before register allocation for operations that
577 // need the $c register !
578 if (typeSizeof(i->dType) == 8) {
579 Instruction *hi;
580 hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);
581 if (hi)
582 next = hi;
583 }
584
585 if (i->op != OP_MOV && i->op != OP_PFETCH)
586 replaceZero(i);
587 }
588 }
589 if (!bb->getEntry())
590 return true;
591
592 if (!tryReplaceContWithBra(bb))
593 propagateJoin(bb);
594
595 return true;
596 }
597
598 NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())
599 {
600 bld.setProgram(prog);
601 gMemBase = NULL;
602 }
603
604 bool
605 NVC0LoweringPass::visit(Function *fn)
606 {
607 if (prog->getType() == Program::TYPE_GEOMETRY) {
608 assert(!strncmp(fn->getName(), "MAIN", 4));
609 // TODO: when we generate actual functions pass this value along somehow
610 bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);
611 gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();
612 if (fn->cfgExit) {
613 bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);
614 bld.mkMovToReg(0, gpEmitAddress);
615 }
616 }
617 return true;
618 }
619
620 bool
621 NVC0LoweringPass::visit(BasicBlock *bb)
622 {
623 return true;
624 }
625
626 inline Value *
627 NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)
628 {
629 uint8_t b = prog->driver->io.auxCBSlot;
630 uint32_t off = prog->driver->io.texBindBase + slot * 4;
631
632 if (ptr)
633 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(2));
634
635 return bld.
636 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
637 }
638
639 // move array source to first slot, convert to u16, add indirections
640 bool
641 NVC0LoweringPass::handleTEX(TexInstruction *i)
642 {
643 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
644 const int arg = i->tex.target.getArgCount();
645 const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);
646 const int chipset = prog->getTarget()->getChipset();
647
648 /* Only normalize in the non-explicit derivatives case. For explicit
649 * derivatives, this is handled in handleManualTXD.
650 */
651 if (i->tex.target.isCube() && i->dPdx[0].get() == NULL) {
652 Value *src[3], *val;
653 int c;
654 for (c = 0; c < 3; ++c)
655 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), i->getSrc(c));
656 val = bld.getScratch();
657 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
658 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
659 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
660 for (c = 0; c < 3; ++c) {
661 i->setSrc(c, bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(),
662 i->getSrc(c), val));
663 }
664 }
665
666 // Arguments to the TEX instruction are a little insane. Even though the
667 // encoding is identical between SM20 and SM30, the arguments mean
668 // different things between Fermi and Kepler+. A lot of arguments are
669 // optional based on flags passed to the instruction. This summarizes the
670 // order of things.
671 //
672 // Fermi:
673 // array/indirect
674 // coords
675 // sample
676 // lod bias
677 // depth compare
678 // offsets:
679 // - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)
680 // - other: 4 bits each, single reg
681 //
682 // Kepler+:
683 // indirect handle
684 // array (+ offsets for txd in upper 16 bits)
685 // coords
686 // sample
687 // lod bias
688 // depth compare
689 // offsets (same as fermi, except txd which takes it with array)
690 //
691 // Maxwell (tex):
692 // array
693 // coords
694 // indirect handle
695 // sample
696 // lod bias
697 // depth compare
698 // offsets
699 //
700 // Maxwell (txd):
701 // indirect handle
702 // coords
703 // array + offsets
704 // derivatives
705
706 if (chipset >= NVISA_GK104_CHIPSET) {
707 if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
708 // XXX this ignores tsc, and assumes a 1:1 mapping
709 assert(i->tex.rIndirectSrc >= 0);
710 Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r);
711 i->tex.r = 0xff;
712 i->tex.s = 0x1f;
713 i->setIndirectR(hnd);
714 i->setIndirectS(NULL);
715 } else if (i->tex.r == i->tex.s || i->op == OP_TXF) {
716 i->tex.r += prog->driver->io.texBindBase / 4;
717 i->tex.s = 0; // only a single cX[] value possible here
718 } else {
719 Value *hnd = bld.getScratch();
720 Value *rHnd = loadTexHandle(NULL, i->tex.r);
721 Value *sHnd = loadTexHandle(NULL, i->tex.s);
722
723 bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);
724
725 i->tex.r = 0; // not used for indirect tex
726 i->tex.s = 0;
727 i->setIndirectR(hnd);
728 }
729 if (i->tex.target.isArray()) {
730 LValue *layer = new_LValue(func, FILE_GPR);
731 Value *src = i->getSrc(lyr);
732 const int sat = (i->op == OP_TXF) ? 1 : 0;
733 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
734 bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;
735 if (i->op != OP_TXD || chipset < NVISA_GM107_CHIPSET) {
736 for (int s = dim; s >= 1; --s)
737 i->setSrc(s, i->getSrc(s - 1));
738 i->setSrc(0, layer);
739 } else {
740 i->setSrc(dim, layer);
741 }
742 }
743 // Move the indirect reference to the first place
744 if (i->tex.rIndirectSrc >= 0 && (
745 i->op == OP_TXD || chipset < NVISA_GM107_CHIPSET)) {
746 Value *hnd = i->getIndirectR();
747
748 i->setIndirectR(NULL);
749 i->moveSources(0, 1);
750 i->setSrc(0, hnd);
751 i->tex.rIndirectSrc = 0;
752 i->tex.sIndirectSrc = -1;
753 }
754 } else
755 // (nvc0) generate and move the tsc/tic/array source to the front
756 if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
757 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
758
759 Value *ticRel = i->getIndirectR();
760 Value *tscRel = i->getIndirectS();
761
762 if (ticRel) {
763 i->setSrc(i->tex.rIndirectSrc, NULL);
764 if (i->tex.r)
765 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
766 ticRel, bld.mkImm(i->tex.r));
767 }
768 if (tscRel) {
769 i->setSrc(i->tex.sIndirectSrc, NULL);
770 if (i->tex.s)
771 tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
772 tscRel, bld.mkImm(i->tex.s));
773 }
774
775 Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;
776 if (arrayIndex) {
777 for (int s = dim; s >= 1; --s)
778 i->setSrc(s, i->getSrc(s - 1));
779 i->setSrc(0, arrayIndex);
780 } else {
781 i->moveSources(0, 1);
782 }
783
784 if (arrayIndex) {
785 int sat = (i->op == OP_TXF) ? 1 : 0;
786 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
787 bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;
788 } else {
789 bld.loadImm(src, 0);
790 }
791
792 if (ticRel)
793 bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);
794 if (tscRel)
795 bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);
796
797 i->setSrc(0, src);
798 }
799
800 // For nvc0, the sample id has to be in the second operand, as the offset
801 // does. Right now we don't know how to pass both in, and this case can't
802 // happen with OpenGL. On nve0, the sample id is part of the texture
803 // coordinate argument.
804 assert(chipset >= NVISA_GK104_CHIPSET ||
805 !i->tex.useOffsets || !i->tex.target.isMS());
806
807 // offset is between lod and dc
808 if (i->tex.useOffsets) {
809 int n, c;
810 int s = i->srcCount(0xff, true);
811 if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {
812 if (i->tex.target.isShadow())
813 s--;
814 if (i->srcExists(s)) // move potential predicate out of the way
815 i->moveSources(s, 1);
816 if (i->tex.useOffsets == 4 && i->srcExists(s + 1))
817 i->moveSources(s + 1, 1);
818 }
819 if (i->op == OP_TXG) {
820 // Either there is 1 offset, which goes into the 2 low bytes of the
821 // first source, or there are 4 offsets, which go into 2 sources (8
822 // values, 1 byte each).
823 Value *offs[2] = {NULL, NULL};
824 for (n = 0; n < i->tex.useOffsets; n++) {
825 for (c = 0; c < 2; ++c) {
826 if ((n % 2) == 0 && c == 0)
827 offs[n / 2] = i->offset[n][c].get();
828 else
829 bld.mkOp3(OP_INSBF, TYPE_U32,
830 offs[n / 2],
831 i->offset[n][c].get(),
832 bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),
833 offs[n / 2]);
834 }
835 }
836 i->setSrc(s, offs[0]);
837 if (offs[1])
838 i->setSrc(s + 1, offs[1]);
839 } else {
840 unsigned imm = 0;
841 assert(i->tex.useOffsets == 1);
842 for (c = 0; c < 3; ++c) {
843 ImmediateValue val;
844 if (!i->offset[0][c].getImmediate(val))
845 assert(!"non-immediate offset passed to non-TXG");
846 imm |= (val.reg.data.u32 & 0xf) << (c * 4);
847 }
848 if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {
849 // The offset goes into the upper 16 bits of the array index. So
850 // create it if it's not already there, and INSBF it if it already
851 // is.
852 s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;
853 if (chipset >= NVISA_GM107_CHIPSET)
854 s += dim;
855 if (i->tex.target.isArray()) {
856 bld.mkOp3(OP_INSBF, TYPE_U32, i->getSrc(s),
857 bld.loadImm(NULL, imm), bld.mkImm(0xc10),
858 i->getSrc(s));
859 } else {
860 i->moveSources(s, 1);
861 i->setSrc(s, bld.loadImm(NULL, imm << 16));
862 }
863 } else {
864 i->setSrc(s, bld.loadImm(NULL, imm));
865 }
866 }
867 }
868
869 if (chipset >= NVISA_GK104_CHIPSET) {
870 //
871 // If TEX requires more than 4 sources, the 2nd register tuple must be
872 // aligned to 4, even if it consists of just a single 4-byte register.
873 //
874 // XXX HACK: We insert 0 sources to avoid the 5 or 6 regs case.
875 //
876 int s = i->srcCount(0xff, true);
877 if (s > 4 && s < 7) {
878 if (i->srcExists(s)) // move potential predicate out of the way
879 i->moveSources(s, 7 - s);
880 while (s < 7)
881 i->setSrc(s++, bld.loadImm(NULL, 0));
882 }
883 }
884
885 return true;
886 }
887
888 bool
889 NVC0LoweringPass::handleManualTXD(TexInstruction *i)
890 {
891 static const uint8_t qOps[4][2] =
892 {
893 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0
894 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1
895 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2
896 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3
897 };
898 Value *def[4][4];
899 Value *crd[3];
900 Instruction *tex;
901 Value *zero = bld.loadImm(bld.getSSA(), 0);
902 int l, c;
903 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
904
905 // This function is invoked after handleTEX lowering, so we have to expect
906 // the arguments in the order that the hw wants them. For Fermi, array and
907 // indirect are both in the leading arg, while for Kepler, array and
908 // indirect are separate (and both precede the coordinates). Maxwell is
909 // handled in a separate function.
910 unsigned array;
911 if (targ->getChipset() < NVISA_GK104_CHIPSET)
912 array = i->tex.target.isArray() || i->tex.rIndirectSrc >= 0;
913 else
914 array = i->tex.target.isArray() + (i->tex.rIndirectSrc >= 0);
915
916 i->op = OP_TEX; // no need to clone dPdx/dPdy later
917
918 for (c = 0; c < dim; ++c)
919 crd[c] = bld.getScratch();
920
921 bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
922 for (l = 0; l < 4; ++l) {
923 Value *src[3], *val;
924 // mov coordinates from lane l to all lanes
925 for (c = 0; c < dim; ++c)
926 bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);
927 // add dPdx from lane l to lanes dx
928 for (c = 0; c < dim; ++c)
929 bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);
930 // add dPdy from lane l to lanes dy
931 for (c = 0; c < dim; ++c)
932 bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);
933 // normalize cube coordinates
934 if (i->tex.target.isCube()) {
935 for (c = 0; c < 3; ++c)
936 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), crd[c]);
937 val = bld.getScratch();
938 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
939 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
940 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
941 for (c = 0; c < 3; ++c)
942 src[c] = bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(), crd[c], val);
943 } else {
944 for (c = 0; c < dim; ++c)
945 src[c] = crd[c];
946 }
947 // texture
948 bld.insert(tex = cloneForward(func, i));
949 for (c = 0; c < dim; ++c)
950 tex->setSrc(c + array, src[c]);
951 // save results
952 for (c = 0; i->defExists(c); ++c) {
953 Instruction *mov;
954 def[c][l] = bld.getSSA();
955 mov = bld.mkMov(def[c][l], tex->getDef(c));
956 mov->fixed = 1;
957 mov->lanes = 1 << l;
958 }
959 }
960 bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
961
962 for (c = 0; i->defExists(c); ++c) {
963 Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
964 for (l = 0; l < 4; ++l)
965 u->setSrc(l, def[c][l]);
966 }
967
968 i->bb->remove(i);
969 return true;
970 }
971
972 bool
973 NVC0LoweringPass::handleTXD(TexInstruction *txd)
974 {
975 int dim = txd->tex.target.getDim() + txd->tex.target.isCube();
976 unsigned arg = txd->tex.target.getArgCount();
977 unsigned expected_args = arg;
978 const int chipset = prog->getTarget()->getChipset();
979
980 if (chipset >= NVISA_GK104_CHIPSET) {
981 if (!txd->tex.target.isArray() && txd->tex.useOffsets)
982 expected_args++;
983 if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)
984 expected_args++;
985 } else {
986 if (txd->tex.useOffsets)
987 expected_args++;
988 if (!txd->tex.target.isArray() && (
989 txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))
990 expected_args++;
991 }
992
993 if (expected_args > 4 ||
994 dim > 2 ||
995 txd->tex.target.isShadow())
996 txd->op = OP_TEX;
997
998 handleTEX(txd);
999 while (txd->srcExists(arg))
1000 ++arg;
1001
1002 txd->tex.derivAll = true;
1003 if (txd->op == OP_TEX)
1004 return handleManualTXD(txd);
1005
1006 assert(arg == expected_args);
1007 for (int c = 0; c < dim; ++c) {
1008 txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);
1009 txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);
1010 txd->dPdx[c].set(NULL);
1011 txd->dPdy[c].set(NULL);
1012 }
1013
1014 // In this case we have fewer than 4 "real" arguments, which means that
1015 // handleTEX didn't apply any padding. However we have to make sure that
1016 // the second "group" of arguments still gets padded up to 4.
1017 if (chipset >= NVISA_GK104_CHIPSET) {
1018 int s = arg + 2 * dim;
1019 if (s >= 4 && s < 7) {
1020 if (txd->srcExists(s)) // move potential predicate out of the way
1021 txd->moveSources(s, 7 - s);
1022 while (s < 7)
1023 txd->setSrc(s++, bld.loadImm(NULL, 0));
1024 }
1025 }
1026
1027 return true;
1028 }
1029
1030 bool
1031 NVC0LoweringPass::handleTXQ(TexInstruction *txq)
1032 {
1033 const int chipset = prog->getTarget()->getChipset();
1034 if (chipset >= NVISA_GK104_CHIPSET && txq->tex.rIndirectSrc < 0)
1035 txq->tex.r += prog->driver->io.texBindBase / 4;
1036
1037 if (txq->tex.rIndirectSrc < 0)
1038 return true;
1039
1040 Value *ticRel = txq->getIndirectR();
1041
1042 txq->setIndirectS(NULL);
1043 txq->tex.sIndirectSrc = -1;
1044
1045 assert(ticRel);
1046
1047 if (chipset < NVISA_GK104_CHIPSET) {
1048 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
1049
1050 txq->setSrc(txq->tex.rIndirectSrc, NULL);
1051 if (txq->tex.r)
1052 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
1053 ticRel, bld.mkImm(txq->tex.r));
1054
1055 bld.mkOp2(OP_SHL, TYPE_U32, src, ticRel, bld.mkImm(0x17));
1056
1057 txq->moveSources(0, 1);
1058 txq->setSrc(0, src);
1059 } else {
1060 Value *hnd = loadTexHandle(txq->getIndirectR(), txq->tex.r);
1061 txq->tex.r = 0xff;
1062 txq->tex.s = 0x1f;
1063
1064 txq->setIndirectR(NULL);
1065 txq->moveSources(0, 1);
1066 txq->setSrc(0, hnd);
1067 txq->tex.rIndirectSrc = 0;
1068 }
1069
1070 return true;
1071 }
1072
1073 bool
1074 NVC0LoweringPass::handleTXLQ(TexInstruction *i)
1075 {
1076 /* The outputs are inverted compared to what the TGSI instruction
1077 * expects. Take that into account in the mask.
1078 */
1079 assert((i->tex.mask & ~3) == 0);
1080 if (i->tex.mask == 1)
1081 i->tex.mask = 2;
1082 else if (i->tex.mask == 2)
1083 i->tex.mask = 1;
1084 handleTEX(i);
1085 bld.setPosition(i, true);
1086
1087 /* The returned values are not quite what we want:
1088 * (a) convert from s16/u16 to f32
1089 * (b) multiply by 1/256
1090 */
1091 for (int def = 0; def < 2; ++def) {
1092 if (!i->defExists(def))
1093 continue;
1094 enum DataType type = TYPE_S16;
1095 if (i->tex.mask == 2 || def > 0)
1096 type = TYPE_U16;
1097 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));
1098 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),
1099 i->getDef(def), bld.loadImm(NULL, 1.0f / 256));
1100 }
1101 if (i->tex.mask == 3) {
1102 LValue *t = new_LValue(func, FILE_GPR);
1103 bld.mkMov(t, i->getDef(0));
1104 bld.mkMov(i->getDef(0), i->getDef(1));
1105 bld.mkMov(i->getDef(1), t);
1106 }
1107 return true;
1108 }
1109
1110 bool
1111 NVC0LoweringPass::handleBUFQ(Instruction *bufq)
1112 {
1113 bufq->op = OP_MOV;
1114 bufq->setSrc(0, loadBufLength32(bufq->getIndirect(0, 1),
1115 bufq->getSrc(0)->reg.fileIndex * 16));
1116 bufq->setIndirect(0, 0, NULL);
1117 bufq->setIndirect(0, 1, NULL);
1118 return true;
1119 }
1120
1121 void
1122 NVC0LoweringPass::handleSharedATOMNVE4(Instruction *atom)
1123 {
1124 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1125
1126 BasicBlock *currBB = atom->bb;
1127 BasicBlock *tryLockBB = atom->bb->splitBefore(atom, false);
1128 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1129 BasicBlock *setAndUnlockBB = new BasicBlock(func);
1130 BasicBlock *failLockBB = new BasicBlock(func);
1131
1132 bld.setPosition(currBB, true);
1133 assert(!currBB->joinAt);
1134 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1135
1136 CmpInstruction *pred =
1137 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1138 TYPE_U32, bld.mkImm(0), bld.mkImm(1));
1139
1140 bld.mkFlow(OP_BRA, tryLockBB, CC_ALWAYS, NULL);
1141 currBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::TREE);
1142
1143 bld.setPosition(tryLockBB, true);
1144
1145 Instruction *ld =
1146 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1147 atom->getIndirect(0, 0));
1148 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1149 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1150
1151 bld.mkFlow(OP_BRA, setAndUnlockBB, CC_P, ld->getDef(1));
1152 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1153 tryLockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::CROSS);
1154 tryLockBB->cfg.attach(&setAndUnlockBB->cfg, Graph::Edge::TREE);
1155
1156 tryLockBB->cfg.detach(&joinBB->cfg);
1157 bld.remove(atom);
1158
1159 bld.setPosition(setAndUnlockBB, true);
1160 Value *stVal;
1161 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1162 // Read the old value, and write the new one.
1163 stVal = atom->getSrc(1);
1164 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1165 CmpInstruction *set =
1166 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(),
1167 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1168
1169 bld.mkCmp(OP_SLCT, CC_NE, TYPE_U32, (stVal = bld.getSSA()),
1170 TYPE_U32, atom->getSrc(2), ld->getDef(0), set->getDef(0));
1171 } else {
1172 operation op;
1173
1174 switch (atom->subOp) {
1175 case NV50_IR_SUBOP_ATOM_ADD:
1176 op = OP_ADD;
1177 break;
1178 case NV50_IR_SUBOP_ATOM_AND:
1179 op = OP_AND;
1180 break;
1181 case NV50_IR_SUBOP_ATOM_OR:
1182 op = OP_OR;
1183 break;
1184 case NV50_IR_SUBOP_ATOM_XOR:
1185 op = OP_XOR;
1186 break;
1187 case NV50_IR_SUBOP_ATOM_MIN:
1188 op = OP_MIN;
1189 break;
1190 case NV50_IR_SUBOP_ATOM_MAX:
1191 op = OP_MAX;
1192 break;
1193 default:
1194 assert(0);
1195 return;
1196 }
1197
1198 stVal = bld.mkOp2v(op, atom->dType, bld.getSSA(), ld->getDef(0),
1199 atom->getSrc(1));
1200 }
1201
1202 Instruction *st =
1203 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1204 atom->getIndirect(0, 0), stVal);
1205 st->setDef(0, pred->getDef(0));
1206 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1207
1208 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1209 setAndUnlockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::TREE);
1210
1211 // Lock until the store has not been performed.
1212 bld.setPosition(failLockBB, true);
1213 bld.mkFlow(OP_BRA, tryLockBB, CC_NOT_P, pred->getDef(0));
1214 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1215 failLockBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::BACK);
1216 failLockBB->cfg.attach(&joinBB->cfg, Graph::Edge::TREE);
1217
1218 bld.setPosition(joinBB, false);
1219 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1220 }
1221
1222 void
1223 NVC0LoweringPass::handleSharedATOM(Instruction *atom)
1224 {
1225 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1226
1227 BasicBlock *currBB = atom->bb;
1228 BasicBlock *tryLockAndSetBB = atom->bb->splitBefore(atom, false);
1229 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1230
1231 bld.setPosition(currBB, true);
1232 assert(!currBB->joinAt);
1233 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1234
1235 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_ALWAYS, NULL);
1236 currBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::TREE);
1237
1238 bld.setPosition(tryLockAndSetBB, true);
1239
1240 Instruction *ld =
1241 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1242 atom->getIndirect(0, 0));
1243 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1244 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1245
1246 Value *stVal;
1247 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1248 // Read the old value, and write the new one.
1249 stVal = atom->getSrc(1);
1250 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1251 CmpInstruction *set =
1252 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1253 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1254 set->setPredicate(CC_P, ld->getDef(1));
1255
1256 Instruction *selp =
1257 bld.mkOp3(OP_SELP, TYPE_U32, bld.getSSA(), ld->getDef(0),
1258 atom->getSrc(2), set->getDef(0));
1259 selp->src(2).mod = Modifier(NV50_IR_MOD_NOT);
1260 selp->setPredicate(CC_P, ld->getDef(1));
1261
1262 stVal = selp->getDef(0);
1263 } else {
1264 operation op;
1265
1266 switch (atom->subOp) {
1267 case NV50_IR_SUBOP_ATOM_ADD:
1268 op = OP_ADD;
1269 break;
1270 case NV50_IR_SUBOP_ATOM_AND:
1271 op = OP_AND;
1272 break;
1273 case NV50_IR_SUBOP_ATOM_OR:
1274 op = OP_OR;
1275 break;
1276 case NV50_IR_SUBOP_ATOM_XOR:
1277 op = OP_XOR;
1278 break;
1279 case NV50_IR_SUBOP_ATOM_MIN:
1280 op = OP_MIN;
1281 break;
1282 case NV50_IR_SUBOP_ATOM_MAX:
1283 op = OP_MAX;
1284 break;
1285 default:
1286 assert(0);
1287 return;
1288 }
1289
1290 Instruction *i =
1291 bld.mkOp2(op, atom->dType, bld.getSSA(), ld->getDef(0),
1292 atom->getSrc(1));
1293 i->setPredicate(CC_P, ld->getDef(1));
1294
1295 stVal = i->getDef(0);
1296 }
1297
1298 Instruction *st =
1299 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1300 atom->getIndirect(0, 0), stVal);
1301 st->setPredicate(CC_P, ld->getDef(1));
1302 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1303
1304 // Loop until the lock is acquired.
1305 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_NOT_P, ld->getDef(1));
1306 tryLockAndSetBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::BACK);
1307 tryLockAndSetBB->cfg.attach(&joinBB->cfg, Graph::Edge::CROSS);
1308 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1309
1310 bld.remove(atom);
1311
1312 bld.setPosition(joinBB, false);
1313 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1314 }
1315
1316 bool
1317 NVC0LoweringPass::handleATOM(Instruction *atom)
1318 {
1319 SVSemantic sv;
1320 Value *ptr = atom->getIndirect(0, 0), *ind = atom->getIndirect(0, 1), *base;
1321
1322 switch (atom->src(0).getFile()) {
1323 case FILE_MEMORY_LOCAL:
1324 sv = SV_LBASE;
1325 break;
1326 case FILE_MEMORY_SHARED:
1327 // For Fermi/Kepler, we have to use ld lock/st unlock to perform atomic
1328 // operations on shared memory. For Maxwell, ATOMS is enough.
1329 if (targ->getChipset() < NVISA_GK104_CHIPSET)
1330 handleSharedATOM(atom);
1331 else if (targ->getChipset() < NVISA_GM107_CHIPSET)
1332 handleSharedATOMNVE4(atom);
1333 return true;
1334 default:
1335 assert(atom->src(0).getFile() == FILE_MEMORY_BUFFER);
1336 base = loadBufInfo64(ind, atom->getSrc(0)->reg.fileIndex * 16);
1337 assert(base->reg.size == 8);
1338 if (ptr)
1339 base = bld.mkOp2v(OP_ADD, TYPE_U64, base, base, ptr);
1340 assert(base->reg.size == 8);
1341 atom->setIndirect(0, 0, base);
1342 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1343
1344 // Harden against out-of-bounds accesses
1345 Value *offset = bld.loadImm(NULL, atom->getSrc(0)->reg.data.offset + typeSizeof(atom->sType));
1346 Value *length = loadBufLength32(ind, atom->getSrc(0)->reg.fileIndex * 16);
1347 Value *pred = new_LValue(func, FILE_PREDICATE);
1348 if (ptr)
1349 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, ptr);
1350 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
1351 atom->setPredicate(CC_NOT_P, pred);
1352 if (atom->defExists(0)) {
1353 Value *zero, *dst = atom->getDef(0);
1354 atom->setDef(0, bld.getSSA());
1355
1356 bld.setPosition(atom, true);
1357 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
1358 ->setPredicate(CC_P, pred);
1359 bld.mkOp2(OP_UNION, TYPE_U32, dst, atom->getDef(0), zero);
1360 }
1361
1362 return true;
1363 }
1364 base =
1365 bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));
1366
1367 atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));
1368 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1369 if (ptr)
1370 base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);
1371 atom->setIndirect(0, 1, NULL);
1372 atom->setIndirect(0, 0, base);
1373
1374 return true;
1375 }
1376
1377 bool
1378 NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)
1379 {
1380 if (targ->getChipset() < NVISA_GM107_CHIPSET) {
1381 if (cas->src(0).getFile() == FILE_MEMORY_SHARED) {
1382 // ATOM_CAS and ATOM_EXCH are handled in handleSharedATOM().
1383 return false;
1384 }
1385 }
1386
1387 if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&
1388 cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)
1389 return false;
1390 bld.setPosition(cas, true);
1391
1392 if (needCctl) {
1393 Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));
1394 cctl->setIndirect(0, 0, cas->getIndirect(0, 0));
1395 cctl->fixed = 1;
1396 cctl->subOp = NV50_IR_SUBOP_CCTL_IV;
1397 if (cas->isPredicated())
1398 cctl->setPredicate(cas->cc, cas->getPredicate());
1399 }
1400
1401 if (cas->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1402 // CAS is crazy. It's 2nd source is a double reg, and the 3rd source
1403 // should be set to the high part of the double reg or bad things will
1404 // happen elsewhere in the universe.
1405 // Also, it sometimes returns the new value instead of the old one
1406 // under mysterious circumstances.
1407 Value *dreg = bld.getSSA(8);
1408 bld.setPosition(cas, false);
1409 bld.mkOp2(OP_MERGE, TYPE_U64, dreg, cas->getSrc(1), cas->getSrc(2));
1410 cas->setSrc(1, dreg);
1411 cas->setSrc(2, dreg);
1412 }
1413
1414 return true;
1415 }
1416
1417 inline Value *
1418 NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off, uint16_t base)
1419 {
1420 uint8_t b = prog->driver->io.auxCBSlot;
1421 off += base;
1422
1423 return bld.
1424 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1425 }
1426
1427 inline Value *
1428 NVC0LoweringPass::loadResInfo64(Value *ptr, uint32_t off, uint16_t base)
1429 {
1430 uint8_t b = prog->driver->io.auxCBSlot;
1431 off += base;
1432
1433 if (ptr)
1434 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1435
1436 return bld.
1437 mkLoadv(TYPE_U64, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off), ptr);
1438 }
1439
1440 inline Value *
1441 NVC0LoweringPass::loadResLength32(Value *ptr, uint32_t off, uint16_t base)
1442 {
1443 uint8_t b = prog->driver->io.auxCBSlot;
1444 off += base;
1445
1446 if (ptr)
1447 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1448
1449 return bld.
1450 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off + 8), ptr);
1451 }
1452
1453 inline Value *
1454 NVC0LoweringPass::loadBufInfo64(Value *ptr, uint32_t off)
1455 {
1456 return loadResInfo64(ptr, off, prog->driver->io.bufInfoBase);
1457 }
1458
1459 inline Value *
1460 NVC0LoweringPass::loadBufLength32(Value *ptr, uint32_t off)
1461 {
1462 return loadResLength32(ptr, off, prog->driver->io.bufInfoBase);
1463 }
1464
1465 inline Value *
1466 NVC0LoweringPass::loadUboInfo64(Value *ptr, uint32_t off)
1467 {
1468 return loadResInfo64(ptr, off, prog->driver->io.uboInfoBase);
1469 }
1470
1471 inline Value *
1472 NVC0LoweringPass::loadUboLength32(Value *ptr, uint32_t off)
1473 {
1474 return loadResLength32(ptr, off, prog->driver->io.uboInfoBase);
1475 }
1476
1477 inline Value *
1478 NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)
1479 {
1480 uint8_t b = prog->driver->io.msInfoCBSlot;
1481 off += prog->driver->io.msInfoBase;
1482 return bld.
1483 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1484 }
1485
1486 /* On nvc0, surface info is obtained via the surface binding points passed
1487 * to the SULD/SUST instructions.
1488 * On nve4, surface info is stored in c[] and is used by various special
1489 * instructions, e.g. for clamping coordinates or generating an address.
1490 * They couldn't just have added an equivalent to TIC now, couldn't they ?
1491 */
1492 #define NVC0_SU_INFO_ADDR 0x00
1493 #define NVC0_SU_INFO_FMT 0x04
1494 #define NVC0_SU_INFO_DIM_X 0x08
1495 #define NVC0_SU_INFO_PITCH 0x0c
1496 #define NVC0_SU_INFO_DIM_Y 0x10
1497 #define NVC0_SU_INFO_ARRAY 0x14
1498 #define NVC0_SU_INFO_DIM_Z 0x18
1499 #define NVC0_SU_INFO_UNK1C 0x1c
1500 #define NVC0_SU_INFO_WIDTH 0x20
1501 #define NVC0_SU_INFO_HEIGHT 0x24
1502 #define NVC0_SU_INFO_DEPTH 0x28
1503 #define NVC0_SU_INFO_TARGET 0x2c
1504 #define NVC0_SU_INFO_BSIZE 0x30
1505 #define NVC0_SU_INFO_RAW_X 0x34
1506 #define NVC0_SU_INFO_MS_X 0x38
1507 #define NVC0_SU_INFO_MS_Y 0x3c
1508
1509 #define NVC0_SU_INFO__STRIDE 0x40
1510
1511 #define NVC0_SU_INFO_DIM(i) (0x08 + (i) * 8)
1512 #define NVC0_SU_INFO_SIZE(i) (0x20 + (i) * 4)
1513 #define NVC0_SU_INFO_MS(i) (0x38 + (i) * 4)
1514
1515 inline Value *
1516 NVC0LoweringPass::loadSuInfo32(Value *ptr, int slot, uint32_t off)
1517 {
1518 uint32_t base = slot * NVC0_SU_INFO__STRIDE;
1519
1520 if (ptr) {
1521 ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(slot));
1522 ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7));
1523 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(6));
1524 base = 0;
1525 }
1526 off += base;
1527
1528 return loadResInfo32(ptr, off, prog->driver->io.suInfoBase);
1529 }
1530
1531 static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)
1532 {
1533 switch (su->tex.target.getEnum()) {
1534 case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);
1535 case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1536 case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1537 case TEX_TARGET_1D_ARRAY: return (c == 1) ?
1538 NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :
1539 NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1540 case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1541 case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1542 case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1543 case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1544 case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1545 case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1546 case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1547 default:
1548 assert(0);
1549 return 0;
1550 }
1551 }
1552
1553 bool
1554 NVC0LoweringPass::handleSUQ(TexInstruction *suq)
1555 {
1556 int mask = suq->tex.mask;
1557 int dim = suq->tex.target.getDim();
1558 int arg = dim + (suq->tex.target.isArray() || suq->tex.target.isCube());
1559 Value *ind = suq->getIndirectR();
1560 int slot = suq->tex.r;
1561 int c, d;
1562
1563 for (c = 0, d = 0; c < 3; ++c, mask >>= 1) {
1564 if (c >= arg || !(mask & 1))
1565 continue;
1566
1567 int offset;
1568
1569 if (c == 1 && suq->tex.target == TEX_TARGET_1D_ARRAY) {
1570 offset = NVC0_SU_INFO_SIZE(2);
1571 } else {
1572 offset = NVC0_SU_INFO_SIZE(c);
1573 }
1574 bld.mkMov(suq->getDef(d++), loadSuInfo32(ind, slot, offset));
1575 if (c == 2 && suq->tex.target.isCube())
1576 bld.mkOp2(OP_DIV, TYPE_U32, suq->getDef(d - 1), suq->getDef(d - 1),
1577 bld.loadImm(NULL, 6));
1578 }
1579
1580 if (mask & 1) {
1581 if (suq->tex.target.isMS()) {
1582 Value *ms_x = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(0));
1583 Value *ms_y = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(1));
1584 Value *ms = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), ms_x, ms_y);
1585 bld.mkOp2(OP_SHL, TYPE_U32, suq->getDef(d++), bld.loadImm(NULL, 1), ms);
1586 } else {
1587 bld.mkMov(suq->getDef(d++), bld.loadImm(NULL, 1));
1588 }
1589 }
1590
1591 bld.remove(suq);
1592 return true;
1593 }
1594
1595 void
1596 NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex)
1597 {
1598 const int arg = tex->tex.target.getArgCount();
1599 int slot = tex->tex.r;
1600
1601 if (tex->tex.target == TEX_TARGET_2D_MS)
1602 tex->tex.target = TEX_TARGET_2D;
1603 else
1604 if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY)
1605 tex->tex.target = TEX_TARGET_2D_ARRAY;
1606 else
1607 return;
1608
1609 Value *x = tex->getSrc(0);
1610 Value *y = tex->getSrc(1);
1611 Value *s = tex->getSrc(arg - 1);
1612
1613 Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA();
1614 Value *ind = tex->getIndirectR();
1615
1616 Value *ms_x = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(0));
1617 Value *ms_y = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(1));
1618
1619 bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x);
1620 bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y);
1621
1622 s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7));
1623 s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3));
1624
1625 Value *dx = loadMsInfo32(ts, 0x0);
1626 Value *dy = loadMsInfo32(ts, 0x4);
1627
1628 bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx);
1629 bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy);
1630
1631 tex->setSrc(0, tx);
1632 tex->setSrc(1, ty);
1633 tex->moveSources(arg, -1);
1634 }
1635
1636 // Sets 64-bit "generic address", predicate and format sources for SULD/SUST.
1637 // They're computed from the coordinates using the surface info in c[] space.
1638 void
1639 NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su)
1640 {
1641 Instruction *insn;
1642 const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP;
1643 const bool raw =
1644 su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB;
1645 const int slot = su->tex.r;
1646 const int dim = su->tex.target.getDim();
1647 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
1648 int c;
1649 Value *zero = bld.mkImm(0);
1650 Value *p1 = NULL;
1651 Value *v;
1652 Value *src[3];
1653 Value *bf, *eau, *off;
1654 Value *addr, *pred;
1655 Value *ind = su->getIndirectR();
1656
1657 off = bld.getScratch(4);
1658 bf = bld.getScratch(4);
1659 addr = bld.getSSA(8);
1660 pred = bld.getScratch(1, FILE_PREDICATE);
1661
1662 bld.setPosition(su, false);
1663
1664 adjustCoordinatesMS(su);
1665
1666 // calculate clamped coordinates
1667 for (c = 0; c < arg; ++c) {
1668 int dimc = c;
1669
1670 if (c == 1 && su->tex.target == TEX_TARGET_1D_ARRAY) {
1671 // The array index is stored in the Z component for 1D arrays.
1672 dimc = 2;
1673 }
1674
1675 src[c] = bld.getScratch();
1676 if (c == 0 && raw)
1677 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_RAW_X);
1678 else
1679 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(dimc));
1680 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero)
1681 ->subOp = getSuClampSubOp(su, dimc);
1682 }
1683 for (; c < 3; ++c)
1684 src[c] = zero;
1685
1686 // set predicate output
1687 if (su->tex.target == TEX_TARGET_BUFFER) {
1688 src[0]->getInsn()->setFlagsDef(1, pred);
1689 } else
1690 if (su->tex.target.isArray() || su->tex.target.isCube()) {
1691 p1 = bld.getSSA(1, FILE_PREDICATE);
1692 src[dim]->getInsn()->setFlagsDef(1, p1);
1693 }
1694
1695 // calculate pixel offset
1696 if (dim == 1) {
1697 if (su->tex.target != TEX_TARGET_BUFFER)
1698 bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff));
1699 } else
1700 if (dim == 3) {
1701 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C);
1702 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1])
1703 ->subOp = NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1704
1705 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_PITCH);
1706 bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0])
1707 ->subOp = NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l
1708 } else {
1709 assert(dim == 2);
1710 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_PITCH);
1711 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[1], v, src[0])
1712 ->subOp = (su->tex.target.isArray() || su->tex.target.isCube()) ?
1713 NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1714 }
1715
1716 // calculate effective address part 1
1717 if (su->tex.target == TEX_TARGET_BUFFER) {
1718 if (raw) {
1719 bf = src[0];
1720 } else {
1721 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT);
1722 bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero)
1723 ->subOp = NV50_IR_SUBOP_V1(7,6,8|2);
1724 }
1725 } else {
1726 Value *y = src[1];
1727 Value *z = src[2];
1728 uint16_t subOp = 0;
1729
1730 switch (dim) {
1731 case 1:
1732 y = zero;
1733 z = zero;
1734 break;
1735 case 2:
1736 z = off;
1737 if (!su->tex.target.isArray() && !su->tex.target.isCube()) {
1738 z = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C);
1739 subOp = NV50_IR_SUBOP_SUBFM_3D;
1740 }
1741 break;
1742 default:
1743 subOp = NV50_IR_SUBOP_SUBFM_3D;
1744 assert(dim == 3);
1745 break;
1746 }
1747 insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z);
1748 insn->subOp = subOp;
1749 insn->setFlagsDef(1, pred);
1750 }
1751
1752 // part 2
1753 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR);
1754
1755 if (su->tex.target == TEX_TARGET_BUFFER) {
1756 eau = v;
1757 } else {
1758 eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v);
1759 }
1760 // add array layer offset
1761 if (su->tex.target.isArray() || su->tex.target.isCube()) {
1762 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY);
1763 if (dim == 1)
1764 bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau)
1765 ->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32
1766 else
1767 bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau)
1768 ->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32
1769 // combine predicates
1770 assert(p1);
1771 bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1);
1772 }
1773
1774 if (atom) {
1775 Value *lo = bf;
1776 if (su->tex.target == TEX_TARGET_BUFFER) {
1777 lo = zero;
1778 bld.mkMov(off, bf);
1779 }
1780 // bf == g[] address & 0xff
1781 // eau == g[] address >> 8
1782 bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau);
1783 bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau);
1784 } else
1785 if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) {
1786 // Convert from u32 to u8 address format, which is what the library code
1787 // doing SULDP currently uses.
1788 // XXX: can SUEAU do this ?
1789 // XXX: does it matter that we don't mask high bytes in bf ?
1790 // Grrr.
1791 bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8));
1792 bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off);
1793 }
1794
1795 bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau);
1796
1797 if (atom && su->tex.target == TEX_TARGET_BUFFER)
1798 bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off);
1799
1800 // let's just set it 0 for raw access and hope it works
1801 v = raw ?
1802 bld.mkImm(0) : loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT);
1803
1804 // get rid of old coordinate sources, make space for fmt info and predicate
1805 su->moveSources(arg, 3 - arg);
1806 // set 64 bit address and 32-bit format sources
1807 su->setSrc(0, addr);
1808 su->setSrc(1, v);
1809 su->setSrc(2, pred);
1810
1811 // prevent read fault when the image is not actually bound
1812 CmpInstruction *pred1 =
1813 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1814 TYPE_U32, bld.mkImm(0),
1815 loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR));
1816
1817 if (su->op != OP_SUSTP && su->tex.format) {
1818 const TexInstruction::ImgFormatDesc *format = su->tex.format;
1819 int blockwidth = format->bits[0] + format->bits[1] +
1820 format->bits[2] + format->bits[3];
1821
1822 // make sure that the format doesn't mismatch
1823 assert(format->components != 0);
1824 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred1->getDef(0),
1825 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
1826 loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE),
1827 pred1->getDef(0));
1828 }
1829 su->setPredicate(CC_NOT_P, pred1->getDef(0));
1830
1831 // TODO: initialize def values to 0 when the surface operation is not
1832 // performed (not needed for stores). Also, fix the "address bounds test"
1833 // subtests from arb_shader_image_load_store-invalid for buffers, because it
1834 // seems like that the predicate is not correctly set by suclamp.
1835 }
1836
1837 static DataType
1838 getSrcType(const TexInstruction::ImgFormatDesc *t, int c)
1839 {
1840 switch (t->type) {
1841 case FLOAT: return t->bits[c] == 16 ? TYPE_F16 : TYPE_F32;
1842 case UNORM: return t->bits[c] == 8 ? TYPE_U8 : TYPE_U16;
1843 case SNORM: return t->bits[c] == 8 ? TYPE_S8 : TYPE_S16;
1844 case UINT:
1845 return (t->bits[c] == 8 ? TYPE_U8 :
1846 (t->bits[c] == 16 ? TYPE_U16 : TYPE_U32));
1847 case SINT:
1848 return (t->bits[c] == 8 ? TYPE_S8 :
1849 (t->bits[c] == 16 ? TYPE_S16 : TYPE_S32));
1850 }
1851 return TYPE_NONE;
1852 }
1853
1854 static DataType
1855 getDestType(const ImgType type) {
1856 switch (type) {
1857 case FLOAT:
1858 case UNORM:
1859 case SNORM:
1860 return TYPE_F32;
1861 case UINT:
1862 return TYPE_U32;
1863 case SINT:
1864 return TYPE_S32;
1865 default:
1866 assert(!"Impossible type");
1867 return TYPE_NONE;
1868 }
1869 }
1870
1871 void
1872 NVC0LoweringPass::convertSurfaceFormat(TexInstruction *su)
1873 {
1874 const TexInstruction::ImgFormatDesc *format = su->tex.format;
1875 int width = format->bits[0] + format->bits[1] +
1876 format->bits[2] + format->bits[3];
1877 Value *untypedDst[4] = {};
1878 Value *typedDst[4] = {};
1879
1880 // We must convert this to a generic load.
1881 su->op = OP_SULDB;
1882
1883 su->dType = typeOfSize(width / 8);
1884 su->sType = TYPE_U8;
1885
1886 for (int i = 0; i < width / 32; i++)
1887 untypedDst[i] = bld.getSSA();
1888 if (width < 32)
1889 untypedDst[0] = bld.getSSA();
1890
1891 for (int i = 0; i < 4; i++) {
1892 typedDst[i] = su->getDef(i);
1893 }
1894
1895 // Set the untyped dsts as the su's destinations
1896 for (int i = 0; i < 4; i++)
1897 su->setDef(i, untypedDst[i]);
1898
1899 bld.setPosition(su, true);
1900
1901 // Unpack each component into the typed dsts
1902 int bits = 0;
1903 for (int i = 0; i < 4; bits += format->bits[i], i++) {
1904 if (!typedDst[i])
1905 continue;
1906 if (i >= format->components) {
1907 if (format->type == FLOAT ||
1908 format->type == UNORM ||
1909 format->type == SNORM)
1910 bld.loadImm(typedDst[i], i == 3 ? 1.0f : 0.0f);
1911 else
1912 bld.loadImm(typedDst[i], i == 3 ? 1 : 0);
1913 continue;
1914 }
1915
1916 // Get just that component's data into the relevant place
1917 if (format->bits[i] == 32)
1918 bld.mkMov(typedDst[i], untypedDst[i]);
1919 else if (format->bits[i] == 16)
1920 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
1921 getSrcType(format, i), untypedDst[i / 2])
1922 ->subOp = (i & 1) << (format->type == FLOAT ? 0 : 1);
1923 else if (format->bits[i] == 8)
1924 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
1925 getSrcType(format, i), untypedDst[0])->subOp = i;
1926 else {
1927 bld.mkOp2(OP_EXTBF, TYPE_U32, typedDst[i], untypedDst[bits / 32],
1928 bld.mkImm((bits % 32) | (format->bits[i] << 8)));
1929 if (format->type == UNORM || format->type == SNORM)
1930 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], getSrcType(format, i), typedDst[i]);
1931 }
1932
1933 // Normalize / convert as necessary
1934 if (format->type == UNORM)
1935 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << format->bits[i]) - 1)));
1936 else if (format->type == SNORM)
1937 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << (format->bits[i] - 1)) - 1)));
1938 else if (format->type == FLOAT && format->bits[i] < 16) {
1939 bld.mkOp2(OP_SHL, TYPE_U32, typedDst[i], typedDst[i], bld.loadImm(NULL, 15 - format->bits[i]));
1940 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], TYPE_F16, typedDst[i]);
1941 }
1942 }
1943
1944 if (format->bgra) {
1945 std::swap(typedDst[0], typedDst[2]);
1946 }
1947 }
1948
1949 void
1950 NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su)
1951 {
1952 processSurfaceCoordsNVE4(su);
1953
1954 if (su->op == OP_SULDP)
1955 convertSurfaceFormat(su);
1956
1957 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
1958 Value *pred = su->getSrc(2);
1959 CondCode cc = CC_NOT_P;
1960 if (su->getPredicate()) {
1961 pred = bld.getScratch(1, FILE_PREDICATE);
1962 cc = su->cc;
1963 if (cc == CC_NOT_P) {
1964 bld.mkOp2(OP_OR, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1965 } else {
1966 bld.mkOp2(OP_AND, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1967 pred->getInsn()->src(1).mod = Modifier(NV50_IR_MOD_NOT);
1968 }
1969 }
1970 Instruction *red = bld.mkOp(OP_ATOM, su->dType, bld.getSSA());
1971 red->subOp = su->subOp;
1972 if (!gMemBase)
1973 gMemBase = bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0);
1974 red->setSrc(0, gMemBase);
1975 red->setSrc(1, su->getSrc(3));
1976 if (su->subOp == NV50_IR_SUBOP_ATOM_CAS)
1977 red->setSrc(2, su->getSrc(4));
1978 red->setIndirect(0, 0, su->getSrc(0));
1979
1980 // make sure to initialize dst value when the atomic operation is not
1981 // performed
1982 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
1983
1984 assert(cc == CC_NOT_P);
1985 red->setPredicate(cc, pred);
1986 mov->setPredicate(CC_P, pred);
1987
1988 bld.mkOp2(OP_UNION, TYPE_U32, su->getDef(0),
1989 red->getDef(0), mov->getDef(0));
1990
1991 delete_Instruction(bld.getProgram(), su);
1992 handleCasExch(red, true);
1993 }
1994
1995 if (su->op == OP_SUSTB || su->op == OP_SUSTP)
1996 su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8;
1997 }
1998
1999 void
2000 NVC0LoweringPass::processSurfaceCoordsNVC0(TexInstruction *su)
2001 {
2002 const int slot = su->tex.r;
2003 const int dim = su->tex.target.getDim();
2004 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2005 int c;
2006 Value *zero = bld.mkImm(0);
2007 Value *src[3];
2008 Value *v;
2009 Value *ind = su->getIndirectR();
2010
2011 bld.setPosition(su, false);
2012
2013 adjustCoordinatesMS(su);
2014
2015 if (ind) {
2016 Value *ptr;
2017 ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ind, bld.mkImm(su->tex.r));
2018 ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7));
2019 su->setIndirectR(ptr);
2020 }
2021
2022 // get surface coordinates
2023 for (c = 0; c < arg; ++c)
2024 src[c] = su->getSrc(c);
2025 for (; c < 3; ++c)
2026 src[c] = zero;
2027
2028 // calculate pixel offset
2029 if (su->op == OP_SULDP || su->op == OP_SUREDP) {
2030 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE);
2031 su->setSrc(0, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[0], v));
2032 }
2033
2034 // add array layer offset
2035 if (su->tex.target.isArray() || su->tex.target.isCube()) {
2036 v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY);
2037 assert(dim > 1);
2038 su->setSrc(2, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[2], v));
2039 }
2040
2041 // prevent read fault when the image is not actually bound
2042 CmpInstruction *pred =
2043 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2044 TYPE_U32, bld.mkImm(0),
2045 loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR));
2046 if (su->op != OP_SUSTP && su->tex.format) {
2047 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2048 int blockwidth = format->bits[0] + format->bits[1] +
2049 format->bits[2] + format->bits[3];
2050
2051 assert(format->components != 0);
2052 // make sure that the format doesn't mismatch when it's not FMT_NONE
2053 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0),
2054 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
2055 loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE),
2056 pred->getDef(0));
2057 }
2058 su->setPredicate(CC_NOT_P, pred->getDef(0));
2059 }
2060
2061 void
2062 NVC0LoweringPass::handleSurfaceOpNVC0(TexInstruction *su)
2063 {
2064 if (su->tex.target == TEX_TARGET_1D_ARRAY) {
2065 /* As 1d arrays also need 3 coordinates, switching to TEX_TARGET_2D_ARRAY
2066 * will simplify the lowering pass and the texture constraints. */
2067 su->moveSources(1, 1);
2068 su->setSrc(1, bld.loadImm(NULL, 0));
2069 su->tex.target = TEX_TARGET_2D_ARRAY;
2070 }
2071
2072 processSurfaceCoordsNVC0(su);
2073
2074 if (su->op == OP_SULDP)
2075 convertSurfaceFormat(su);
2076
2077 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
2078 const int dim = su->tex.target.getDim();
2079 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2080 LValue *addr = bld.getSSA(8);
2081 Value *def = su->getDef(0);
2082
2083 su->op = OP_SULEA;
2084
2085 // Set the destination to the address
2086 su->dType = TYPE_U64;
2087 su->setDef(0, addr);
2088 su->setDef(1, su->getPredicate());
2089
2090 bld.setPosition(su, true);
2091
2092 // Perform the atomic op
2093 Instruction *red = bld.mkOp(OP_ATOM, su->sType, bld.getSSA());
2094 red->subOp = su->subOp;
2095 red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, su->sType, 0));
2096 red->setSrc(1, su->getSrc(arg));
2097 if (red->subOp == NV50_IR_SUBOP_ATOM_CAS)
2098 red->setSrc(2, su->getSrc(arg + 1));
2099 red->setIndirect(0, 0, addr);
2100
2101 // make sure to initialize dst value when the atomic operation is not
2102 // performed
2103 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2104
2105 assert(su->cc == CC_NOT_P);
2106 red->setPredicate(su->cc, su->getPredicate());
2107 mov->setPredicate(CC_P, su->getPredicate());
2108
2109 bld.mkOp2(OP_UNION, TYPE_U32, def, red->getDef(0), mov->getDef(0));
2110
2111 handleCasExch(red, false);
2112 }
2113 }
2114
2115 bool
2116 NVC0LoweringPass::handleWRSV(Instruction *i)
2117 {
2118 Instruction *st;
2119 Symbol *sym;
2120 uint32_t addr;
2121
2122 // must replace, $sreg are not writeable
2123 addr = targ->getSVAddress(FILE_SHADER_OUTPUT, i->getSrc(0)->asSym());
2124 if (addr >= 0x400)
2125 return false;
2126 sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
2127
2128 st = bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0),
2129 i->getSrc(1));
2130 st->perPatch = i->perPatch;
2131
2132 bld.getBB()->remove(i);
2133 return true;
2134 }
2135
2136 void
2137 NVC0LoweringPass::handleLDST(Instruction *i)
2138 {
2139 if (i->src(0).getFile() == FILE_SHADER_INPUT) {
2140 if (prog->getType() == Program::TYPE_COMPUTE) {
2141 i->getSrc(0)->reg.file = FILE_MEMORY_CONST;
2142 i->getSrc(0)->reg.fileIndex = 0;
2143 } else
2144 if (prog->getType() == Program::TYPE_GEOMETRY &&
2145 i->src(0).isIndirect(0)) {
2146 // XXX: this assumes vec4 units
2147 Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2148 i->getIndirect(0, 0), bld.mkImm(4));
2149 i->setIndirect(0, 0, ptr);
2150 i->op = OP_VFETCH;
2151 } else {
2152 i->op = OP_VFETCH;
2153 assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP
2154 }
2155 } else if (i->src(0).getFile() == FILE_MEMORY_CONST) {
2156 if (targ->getChipset() >= NVISA_GK104_CHIPSET &&
2157 prog->getType() == Program::TYPE_COMPUTE) {
2158 // The launch descriptor only allows to set up 8 CBs, but OpenGL
2159 // requires at least 12 UBOs. To bypass this limitation, we store the
2160 // addrs into the driver constbuf and we directly load from the global
2161 // memory.
2162 int8_t fileIndex = i->getSrc(0)->reg.fileIndex - 1;
2163 Value *ind = i->getIndirect(0, 1);
2164
2165 if (ind) {
2166 // Clamp the UBO index when an indirect access is used to avoid
2167 // loading information from the wrong place in the driver cb.
2168 ind = bld.mkOp2v(OP_MIN, TYPE_U32, ind,
2169 bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(),
2170 ind, bld.loadImm(NULL, fileIndex)),
2171 bld.loadImm(NULL, 12));
2172 }
2173
2174 if (i->src(0).isIndirect(1)) {
2175 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2176 Value *ptr = loadUboInfo64(ind, fileIndex * 16);
2177 Value *length = loadUboLength32(ind, fileIndex * 16);
2178 Value *pred = new_LValue(func, FILE_PREDICATE);
2179 if (i->src(0).isIndirect(0)) {
2180 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2181 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2182 }
2183 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2184 i->setIndirect(0, 1, NULL);
2185 i->setIndirect(0, 0, ptr);
2186 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2187 i->setPredicate(CC_NOT_P, pred);
2188 if (i->defExists(0)) {
2189 bld.mkMov(i->getDef(0), bld.mkImm(0));
2190 }
2191 } else if (fileIndex >= 0) {
2192 Value *ptr = loadUboInfo64(ind, fileIndex * 16);
2193 if (i->src(0).isIndirect(0)) {
2194 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2195 }
2196 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2197 i->setIndirect(0, 1, NULL);
2198 i->setIndirect(0, 0, ptr);
2199 }
2200 } else if (i->src(0).isIndirect(1)) {
2201 Value *ptr;
2202 if (i->src(0).isIndirect(0))
2203 ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(),
2204 i->getIndirect(0, 1), bld.mkImm(0x1010),
2205 i->getIndirect(0, 0));
2206 else
2207 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2208 i->getIndirect(0, 1), bld.mkImm(16));
2209 i->setIndirect(0, 1, NULL);
2210 i->setIndirect(0, 0, ptr);
2211 i->subOp = NV50_IR_SUBOP_LDC_IS;
2212 }
2213 } else if (i->src(0).getFile() == FILE_SHADER_OUTPUT) {
2214 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2215 i->op = OP_VFETCH;
2216 } else if (i->src(0).getFile() == FILE_MEMORY_BUFFER) {
2217 Value *ind = i->getIndirect(0, 1);
2218 Value *ptr = loadBufInfo64(ind, i->getSrc(0)->reg.fileIndex * 16);
2219 // XXX come up with a way not to do this for EVERY little access but
2220 // rather to batch these up somehow. Unfortunately we've lost the
2221 // information about the field width by the time we get here.
2222 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2223 Value *length = loadBufLength32(ind, i->getSrc(0)->reg.fileIndex * 16);
2224 Value *pred = new_LValue(func, FILE_PREDICATE);
2225 if (i->src(0).isIndirect(0)) {
2226 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2227 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2228 }
2229 i->setIndirect(0, 1, NULL);
2230 i->setIndirect(0, 0, ptr);
2231 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2232 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2233 i->setPredicate(CC_NOT_P, pred);
2234 if (i->defExists(0)) {
2235 Value *zero, *dst = i->getDef(0);
2236 i->setDef(0, bld.getSSA());
2237
2238 bld.setPosition(i, true);
2239 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
2240 ->setPredicate(CC_P, pred);
2241 bld.mkOp2(OP_UNION, TYPE_U32, dst, i->getDef(0), zero);
2242 }
2243 }
2244 }
2245
2246 void
2247 NVC0LoweringPass::readTessCoord(LValue *dst, int c)
2248 {
2249 Value *laneid = bld.getSSA();
2250 Value *x, *y;
2251
2252 bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0));
2253
2254 if (c == 0) {
2255 x = dst;
2256 y = NULL;
2257 } else
2258 if (c == 1) {
2259 x = NULL;
2260 y = dst;
2261 } else {
2262 assert(c == 2);
2263 if (prog->driver->prop.tp.domain != PIPE_PRIM_TRIANGLES) {
2264 bld.mkMov(dst, bld.loadImm(NULL, 0));
2265 return;
2266 }
2267 x = bld.getSSA();
2268 y = bld.getSSA();
2269 }
2270 if (x)
2271 bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid);
2272 if (y)
2273 bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid);
2274
2275 if (c == 2) {
2276 bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y);
2277 bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst);
2278 }
2279 }
2280
2281 bool
2282 NVC0LoweringPass::handleRDSV(Instruction *i)
2283 {
2284 Symbol *sym = i->getSrc(0)->asSym();
2285 const SVSemantic sv = sym->reg.data.sv.sv;
2286 Value *vtx = NULL;
2287 Instruction *ld;
2288 uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
2289
2290 if (addr >= 0x400) {
2291 // mov $sreg
2292 if (sym->reg.data.sv.index == 3) {
2293 // TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID
2294 i->op = OP_MOV;
2295 i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0));
2296 }
2297 if (sv == SV_VERTEX_COUNT) {
2298 bld.setPosition(i, true);
2299 bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0), bld.mkImm(0x808));
2300 }
2301 return true;
2302 }
2303
2304 switch (sv) {
2305 case SV_POSITION:
2306 assert(prog->getType() == Program::TYPE_FRAGMENT);
2307 if (i->srcExists(1)) {
2308 // Pass offset through to the interpolation logic
2309 ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET,
2310 i->getDef(0), addr, NULL);
2311 ld->setSrc(1, i->getSrc(1));
2312 } else {
2313 bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
2314 }
2315 break;
2316 case SV_FACE:
2317 {
2318 Value *face = i->getDef(0);
2319 bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL);
2320 if (i->dType == TYPE_F32) {
2321 bld.mkOp2(OP_OR, TYPE_U32, face, face, bld.mkImm(0x00000001));
2322 bld.mkOp1(OP_NEG, TYPE_S32, face, face);
2323 bld.mkCvt(OP_CVT, TYPE_F32, face, TYPE_S32, face);
2324 }
2325 }
2326 break;
2327 case SV_TESS_COORD:
2328 assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL);
2329 readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index);
2330 break;
2331 case SV_NTID:
2332 case SV_NCTAID:
2333 case SV_GRIDID:
2334 assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise
2335 if (sym->reg.data.sv.index == 3) {
2336 i->op = OP_MOV;
2337 i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1));
2338 return true;
2339 }
2340 // Fallthrough
2341 case SV_WORK_DIM:
2342 addr += prog->driver->prop.cp.gridInfoBase;
2343 bld.mkLoad(TYPE_U32, i->getDef(0),
2344 bld.mkSymbol(FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2345 TYPE_U32, addr), NULL);
2346 break;
2347 case SV_SAMPLE_INDEX:
2348 // TODO: Properly pass source as an address in the PIX address space
2349 // (which can be of the form [r0+offset]). But this is currently
2350 // unnecessary.
2351 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2352 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2353 break;
2354 case SV_SAMPLE_POS: {
2355 Value *off = new_LValue(func, FILE_GPR);
2356 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2357 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2358 bld.mkOp2(OP_SHL, TYPE_U32, off, i->getDef(0), bld.mkImm(3));
2359 bld.mkLoad(TYPE_F32,
2360 i->getDef(0),
2361 bld.mkSymbol(
2362 FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2363 TYPE_U32, prog->driver->io.sampleInfoBase +
2364 4 * sym->reg.data.sv.index),
2365 off);
2366 break;
2367 }
2368 case SV_SAMPLE_MASK: {
2369 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2370 ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK;
2371 Instruction *sampleid =
2372 bld.mkOp1(OP_PIXLD, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2373 sampleid->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2374 Value *masked =
2375 bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ld->getDef(0),
2376 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2377 bld.loadImm(NULL, 1), sampleid->getDef(0)));
2378 if (prog->driver->prop.fp.persampleInvocation) {
2379 bld.mkMov(i->getDef(0), masked);
2380 } else {
2381 bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), ld->getDef(0), masked,
2382 bld.mkImm(0))
2383 ->subOp = 1;
2384 }
2385 break;
2386 }
2387 case SV_BASEVERTEX:
2388 case SV_BASEINSTANCE:
2389 case SV_DRAWID:
2390 ld = bld.mkLoad(TYPE_U32, i->getDef(0),
2391 bld.mkSymbol(FILE_MEMORY_CONST,
2392 prog->driver->io.auxCBSlot,
2393 TYPE_U32,
2394 prog->driver->io.drawInfoBase +
2395 4 * (sv - SV_BASEVERTEX)),
2396 NULL);
2397 break;
2398 default:
2399 if (prog->getType() == Program::TYPE_TESSELLATION_EVAL && !i->perPatch)
2400 vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2401 ld = bld.mkFetch(i->getDef(0), i->dType,
2402 FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx);
2403 ld->perPatch = i->perPatch;
2404 break;
2405 }
2406 bld.getBB()->remove(i);
2407 return true;
2408 }
2409
2410 bool
2411 NVC0LoweringPass::handleDIV(Instruction *i)
2412 {
2413 if (!isFloatType(i->dType))
2414 return true;
2415 bld.setPosition(i, false);
2416 Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(typeSizeof(i->dType)), i->getSrc(1));
2417 i->op = OP_MUL;
2418 i->setSrc(1, rcp->getDef(0));
2419 return true;
2420 }
2421
2422 bool
2423 NVC0LoweringPass::handleMOD(Instruction *i)
2424 {
2425 if (!isFloatType(i->dType))
2426 return true;
2427 LValue *value = bld.getScratch(typeSizeof(i->dType));
2428 bld.mkOp1(OP_RCP, i->dType, value, i->getSrc(1));
2429 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(0), value);
2430 bld.mkOp1(OP_TRUNC, i->dType, value, value);
2431 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(1), value);
2432 i->op = OP_SUB;
2433 i->setSrc(1, value);
2434 return true;
2435 }
2436
2437 bool
2438 NVC0LoweringPass::handleSQRT(Instruction *i)
2439 {
2440 if (i->dType == TYPE_F64) {
2441 Value *pred = bld.getSSA(1, FILE_PREDICATE);
2442 Value *zero = bld.loadImm(NULL, 0.0);
2443 Value *dst = bld.getSSA(8);
2444 bld.mkOp1(OP_RSQ, i->dType, dst, i->getSrc(0));
2445 bld.mkCmp(OP_SET, CC_LE, i->dType, pred, i->dType, i->getSrc(0), zero);
2446 bld.mkOp3(OP_SELP, TYPE_U64, dst, zero, dst, pred);
2447 i->op = OP_MUL;
2448 i->setSrc(1, dst);
2449 // TODO: Handle this properly with a library function
2450 } else {
2451 bld.setPosition(i, true);
2452 i->op = OP_RSQ;
2453 bld.mkOp1(OP_RCP, i->dType, i->getDef(0), i->getDef(0));
2454 }
2455
2456 return true;
2457 }
2458
2459 bool
2460 NVC0LoweringPass::handlePOW(Instruction *i)
2461 {
2462 LValue *val = bld.getScratch();
2463
2464 bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
2465 bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
2466 bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
2467
2468 i->op = OP_EX2;
2469 i->setSrc(0, val);
2470 i->setSrc(1, NULL);
2471
2472 return true;
2473 }
2474
2475 bool
2476 NVC0LoweringPass::handleEXPORT(Instruction *i)
2477 {
2478 if (prog->getType() == Program::TYPE_FRAGMENT) {
2479 int id = i->getSrc(0)->reg.data.offset / 4;
2480
2481 if (i->src(0).isIndirect(0)) // TODO, ugly
2482 return false;
2483 i->op = OP_MOV;
2484 i->subOp = NV50_IR_SUBOP_MOV_FINAL;
2485 i->src(0).set(i->src(1));
2486 i->setSrc(1, NULL);
2487 i->setDef(0, new_LValue(func, FILE_GPR));
2488 i->getDef(0)->reg.data.id = id;
2489
2490 prog->maxGPR = MAX2(prog->maxGPR, id);
2491 } else
2492 if (prog->getType() == Program::TYPE_GEOMETRY) {
2493 i->setIndirect(0, 1, gpEmitAddress);
2494 }
2495 return true;
2496 }
2497
2498 bool
2499 NVC0LoweringPass::handleOUT(Instruction *i)
2500 {
2501 Instruction *prev = i->prev;
2502 ImmediateValue stream, prevStream;
2503
2504 // Only merge if the stream ids match. Also, note that the previous
2505 // instruction would have already been lowered, so we take arg1 from it.
2506 if (i->op == OP_RESTART && prev && prev->op == OP_EMIT &&
2507 i->src(0).getImmediate(stream) &&
2508 prev->src(1).getImmediate(prevStream) &&
2509 stream.reg.data.u32 == prevStream.reg.data.u32) {
2510 i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART;
2511 delete_Instruction(prog, i);
2512 } else {
2513 assert(gpEmitAddress);
2514 i->setDef(0, gpEmitAddress);
2515 i->setSrc(1, i->getSrc(0));
2516 i->setSrc(0, gpEmitAddress);
2517 }
2518 return true;
2519 }
2520
2521 // Generate a binary predicate if an instruction is predicated by
2522 // e.g. an f32 value.
2523 void
2524 NVC0LoweringPass::checkPredicate(Instruction *insn)
2525 {
2526 Value *pred = insn->getPredicate();
2527 Value *pdst;
2528
2529 if (!pred || pred->reg.file == FILE_PREDICATE)
2530 return;
2531 pdst = new_LValue(func, FILE_PREDICATE);
2532
2533 // CAUTION: don't use pdst->getInsn, the definition might not be unique,
2534 // delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass
2535
2536 bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred);
2537
2538 insn->setPredicate(insn->cc, pdst);
2539 }
2540
2541 //
2542 // - add quadop dance for texturing
2543 // - put FP outputs in GPRs
2544 // - convert instruction sequences
2545 //
2546 bool
2547 NVC0LoweringPass::visit(Instruction *i)
2548 {
2549 bool ret = true;
2550 bld.setPosition(i, false);
2551
2552 if (i->cc != CC_ALWAYS)
2553 checkPredicate(i);
2554
2555 switch (i->op) {
2556 case OP_TEX:
2557 case OP_TXB:
2558 case OP_TXL:
2559 case OP_TXF:
2560 case OP_TXG:
2561 return handleTEX(i->asTex());
2562 case OP_TXD:
2563 return handleTXD(i->asTex());
2564 case OP_TXLQ:
2565 return handleTXLQ(i->asTex());
2566 case OP_TXQ:
2567 return handleTXQ(i->asTex());
2568 case OP_EX2:
2569 bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
2570 i->setSrc(0, i->getDef(0));
2571 break;
2572 case OP_POW:
2573 return handlePOW(i);
2574 case OP_DIV:
2575 return handleDIV(i);
2576 case OP_MOD:
2577 return handleMOD(i);
2578 case OP_SQRT:
2579 return handleSQRT(i);
2580 case OP_EXPORT:
2581 ret = handleEXPORT(i);
2582 break;
2583 case OP_EMIT:
2584 case OP_RESTART:
2585 return handleOUT(i);
2586 case OP_RDSV:
2587 return handleRDSV(i);
2588 case OP_WRSV:
2589 return handleWRSV(i);
2590 case OP_STORE:
2591 case OP_LOAD:
2592 handleLDST(i);
2593 break;
2594 case OP_ATOM:
2595 {
2596 const bool cctl = i->src(0).getFile() == FILE_MEMORY_BUFFER;
2597 handleATOM(i);
2598 handleCasExch(i, cctl);
2599 }
2600 break;
2601 case OP_SULDB:
2602 case OP_SULDP:
2603 case OP_SUSTB:
2604 case OP_SUSTP:
2605 case OP_SUREDB:
2606 case OP_SUREDP:
2607 if (targ->getChipset() >= NVISA_GK104_CHIPSET)
2608 handleSurfaceOpNVE4(i->asTex());
2609 else
2610 handleSurfaceOpNVC0(i->asTex());
2611 break;
2612 case OP_SUQ:
2613 handleSUQ(i->asTex());
2614 break;
2615 case OP_BUFQ:
2616 handleBUFQ(i);
2617 break;
2618 default:
2619 break;
2620 }
2621
2622 /* Kepler+ has a special opcode to compute a new base address to be used
2623 * for indirect loads.
2624 */
2625 if (targ->getChipset() >= NVISA_GK104_CHIPSET && !i->perPatch &&
2626 (i->op == OP_VFETCH || i->op == OP_EXPORT) && i->src(0).isIndirect(0)) {
2627 Instruction *afetch = bld.mkOp1(OP_AFETCH, TYPE_U32, bld.getSSA(),
2628 cloneShallow(func, i->getSrc(0)));
2629 afetch->setIndirect(0, 0, i->getIndirect(0, 0));
2630 i->src(0).get()->reg.data.offset = 0;
2631 i->setIndirect(0, 0, afetch->getDef(0));
2632 }
2633
2634 return ret;
2635 }
2636
2637 bool
2638 TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const
2639 {
2640 if (stage == CG_STAGE_PRE_SSA) {
2641 NVC0LoweringPass pass(prog);
2642 return pass.run(prog, false, true);
2643 } else
2644 if (stage == CG_STAGE_POST_RA) {
2645 NVC0LegalizePostRA pass(prog);
2646 return pass.run(prog, false, true);
2647 } else
2648 if (stage == CG_STAGE_SSA) {
2649 NVC0LegalizeSSA pass;
2650 return pass.run(prog, false, true);
2651 }
2652 return false;
2653 }
2654
2655 } // namespace nv50_ir