projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nvc0/ir: fix indirect access for images
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_lowering_nvc0.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir.h"
24 #include "codegen/nv50_ir_build_util.h"
25
26 #include "codegen/nv50_ir_target_nvc0.h"
27 #include "codegen/nv50_ir_lowering_nvc0.h"
28
29 #include <limits>
30
31 namespace nv50_ir {
32
33 #define QOP_ADD 0
34 #define QOP_SUBR 1
35 #define QOP_SUB 2
36 #define QOP_MOV2 3
37
38 // UL UR LL LR
39 #define QUADOP(q, r, s, t) \
40 ((QOP_##q << 6) | (QOP_##r << 4) | \
41 (QOP_##s << 2) | (QOP_##t << 0))
42
43 void
44 NVC0LegalizeSSA::handleDIV(Instruction *i)
45 {
46 FlowInstruction *call;
47 int builtin;
48 Value *def[2];
49
50 bld.setPosition(i, false);
51 def[0] = bld.mkMovToReg(0, i->getSrc(0))->getDef(0);
52 def[1] = bld.mkMovToReg(1, i->getSrc(1))->getDef(0);
53 switch (i->dType) {
54 case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break;
55 case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break;
56 default:
57 return;
58 }
59 call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL);
60 bld.mkMov(i->getDef(0), def[(i->op == OP_DIV) ? 0 : 1]);
61 bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2);
62 bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0);
63
64 call->fixed = 1;
65 call->absolute = call->builtin = 1;
66 call->target.builtin = builtin;
67 delete_Instruction(prog, i);
68 }
69
70 void
71 NVC0LegalizeSSA::handleRCPRSQ(Instruction *i)
72 {
73 assert(i->dType == TYPE_F64);
74 // There are instructions that will compute the high 32 bits of the 64-bit
75 // float. We will just stick 0 in the bottom 32 bits.
76
77 bld.setPosition(i, false);
78
79 // 1. Take the source and it up.
80 Value *src[2], *dst[2], *def = i->getDef(0);
81 bld.mkSplit(src, 4, i->getSrc(0));
82
83 // 2. We don't care about the low 32 bits of the destination. Stick a 0 in.
84 dst[0] = bld.loadImm(NULL, 0);
85 dst[1] = bld.getSSA();
86
87 // 3. The new version of the instruction takes the high 32 bits of the
88 // source and outputs the high 32 bits of the destination.
89 i->setSrc(0, src[1]);
90 i->setDef(0, dst[1]);
91 i->setType(TYPE_F32);
92 i->subOp = NV50_IR_SUBOP_RCPRSQ_64H;
93
94 // 4. Recombine the two dst pieces back into the original destination.
95 bld.setPosition(i, true);
96 bld.mkOp2(OP_MERGE, TYPE_U64, def, dst[0], dst[1]);
97 }
98
99 void
100 NVC0LegalizeSSA::handleFTZ(Instruction *i)
101 {
102 // Only want to flush float inputs
103 assert(i->sType == TYPE_F32);
104
105 // If we're already flushing denorms (and NaN's) to zero, no need for this.
106 if (i->dnz)
107 return;
108
109 // Only certain classes of operations can flush
110 OpClass cls = prog->getTarget()->getOpClass(i->op);
111 if (cls != OPCLASS_ARITH && cls != OPCLASS_COMPARE &&
112 cls != OPCLASS_CONVERT)
113 return;
114
115 i->ftz = true;
116 }
117
118 bool
119 NVC0LegalizeSSA::visit(Function *fn)
120 {
121 bld.setProgram(fn->getProgram());
122 return true;
123 }
124
125 bool
126 NVC0LegalizeSSA::visit(BasicBlock *bb)
127 {
128 Instruction *next;
129 for (Instruction *i = bb->getEntry(); i; i = next) {
130 next = i->next;
131 if (i->sType == TYPE_F32) {
132 if (prog->getType() != Program::TYPE_COMPUTE)
133 handleFTZ(i);
134 continue;
135 }
136 switch (i->op) {
137 case OP_DIV:
138 case OP_MOD:
139 handleDIV(i);
140 break;
141 case OP_RCP:
142 case OP_RSQ:
143 if (i->dType == TYPE_F64)
144 handleRCPRSQ(i);
145 break;
146 default:
147 break;
148 }
149 }
150 return true;
151 }
152
153 NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog)
154 : rZero(NULL),
155 carry(NULL),
156 pOne(NULL),
157 needTexBar(prog->getTarget()->getChipset() >= 0xe0)
158 {
159 }
160
161 bool
162 NVC0LegalizePostRA::insnDominatedBy(const Instruction *later,
163 const Instruction *early) const
164 {
165 if (early->bb == later->bb)
166 return early->serial < later->serial;
167 return later->bb->dominatedBy(early->bb);
168 }
169
170 void
171 NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses,
172 Instruction *usei, const Instruction *texi)
173 {
174 bool add = true;
175 for (std::list<TexUse>::iterator it = uses.begin();
176 it != uses.end();) {
177 if (insnDominatedBy(usei, it->insn)) {
178 add = false;
179 break;
180 }
181 if (insnDominatedBy(it->insn, usei))
182 it = uses.erase(it);
183 else
184 ++it;
185 }
186 if (add)
187 uses.push_back(TexUse(usei, texi));
188 }
189
190 // While it might be tempting to use the an algorithm that just looks at tex
191 // uses, not all texture results are guaranteed to be used on all paths. In
192 // the case where along some control flow path a texture result is never used,
193 // we might reuse that register for something else, creating a
194 // write-after-write hazard. So we have to manually look through all
195 // instructions looking for ones that reference the registers in question.
196 void
197 NVC0LegalizePostRA::findFirstUses(
198 Instruction *texi, std::list<TexUse> &uses)
199 {
200 int minGPR = texi->def(0).rep()->reg.data.id;
201 int maxGPR = minGPR + texi->def(0).rep()->reg.size / 4 - 1;
202
203 unordered_set<const BasicBlock *> visited;
204 findFirstUsesBB(minGPR, maxGPR, texi->next, texi, uses, visited);
205 }
206
207 void
208 NVC0LegalizePostRA::findFirstUsesBB(
209 int minGPR, int maxGPR, Instruction *start,
210 const Instruction *texi, std::list<TexUse> &uses,
211 unordered_set<const BasicBlock *> &visited)
212 {
213 const BasicBlock *bb = start->bb;
214
215 // We don't process the whole bb the first time around. This is correct,
216 // however we might be in a loop and hit this BB again, and need to process
217 // the full thing. So only mark a bb as visited if we processed it from the
218 // beginning.
219 if (start == bb->getEntry()) {
220 if (visited.find(bb) != visited.end())
221 return;
222 visited.insert(bb);
223 }
224
225 for (Instruction *insn = start; insn != bb->getExit(); insn = insn->next) {
226 if (insn->isNop())
227 continue;
228
229 for (int d = 0; insn->defExists(d); ++d) {
230 if (insn->def(d).getFile() != FILE_GPR ||
231 insn->def(d).rep()->reg.data.id < minGPR ||
232 insn->def(d).rep()->reg.data.id > maxGPR)
233 continue;
234 addTexUse(uses, insn, texi);
235 return;
236 }
237
238 for (int s = 0; insn->srcExists(s); ++s) {
239 if (insn->src(s).getFile() != FILE_GPR ||
240 insn->src(s).rep()->reg.data.id < minGPR ||
241 insn->src(s).rep()->reg.data.id > maxGPR)
242 continue;
243 addTexUse(uses, insn, texi);
244 return;
245 }
246 }
247
248 for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) {
249 findFirstUsesBB(minGPR, maxGPR, BasicBlock::get(ei.getNode())->getEntry(),
250 texi, uses, visited);
251 }
252 }
253
254 // Texture barriers:
255 // This pass is a bit long and ugly and can probably be optimized.
256 //
257 // 1. obtain a list of TEXes and their outputs' first use(s)
258 // 2. calculate the barrier level of each first use (minimal number of TEXes,
259 // over all paths, between the TEX and the use in question)
260 // 3. for each barrier, if all paths from the source TEX to that barrier
261 // contain a barrier of lesser level, it can be culled
262 bool
263 NVC0LegalizePostRA::insertTextureBarriers(Function *fn)
264 {
265 std::list<TexUse> *uses;
266 std::vector<Instruction *> texes;
267 std::vector<int> bbFirstTex;
268 std::vector<int> bbFirstUse;
269 std::vector<int> texCounts;
270 std::vector<TexUse> useVec;
271 ArrayList insns;
272
273 fn->orderInstructions(insns);
274
275 texCounts.resize(fn->allBBlocks.getSize(), 0);
276 bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize());
277 bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize());
278
279 // tag BB CFG nodes by their id for later
280 for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) {
281 BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get());
282 if (bb)
283 bb->cfg.tag = bb->getId();
284 }
285
286 // gather the first uses for each TEX
287 for (int i = 0; i < insns.getSize(); ++i) {
288 Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i));
289 if (isTextureOp(tex->op)) {
290 texes.push_back(tex);
291 if (!texCounts.at(tex->bb->getId()))
292 bbFirstTex[tex->bb->getId()] = texes.size() - 1;
293 texCounts[tex->bb->getId()]++;
294 }
295 }
296 insns.clear();
297 if (texes.empty())
298 return false;
299 uses = new std::list<TexUse>[texes.size()];
300 if (!uses)
301 return false;
302 for (size_t i = 0; i < texes.size(); ++i) {
303 findFirstUses(texes[i], uses[i]);
304 }
305
306 // determine the barrier level at each use
307 for (size_t i = 0; i < texes.size(); ++i) {
308 for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end();
309 ++u) {
310 BasicBlock *tb = texes[i]->bb;
311 BasicBlock *ub = u->insn->bb;
312 if (tb == ub) {
313 u->level = 0;
314 for (size_t j = i + 1; j < texes.size() &&
315 texes[j]->bb == tb && texes[j]->serial < u->insn->serial;
316 ++j)
317 u->level++;
318 } else {
319 u->level = fn->cfg.findLightestPathWeight(&tb->cfg,
320 &ub->cfg, texCounts);
321 if (u->level < 0) {
322 WARN("Failed to find path TEX -> TEXBAR\n");
323 u->level = 0;
324 continue;
325 }
326 // this counted all TEXes in the origin block, correct that
327 u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */;
328 // and did not count the TEXes in the destination block, add those
329 for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() &&
330 texes[j]->bb == ub && texes[j]->serial < u->insn->serial;
331 ++j)
332 u->level++;
333 }
334 assert(u->level >= 0);
335 useVec.push_back(*u);
336 }
337 }
338 delete[] uses;
339
340 // insert the barriers
341 for (size_t i = 0; i < useVec.size(); ++i) {
342 Instruction *prev = useVec[i].insn->prev;
343 if (useVec[i].level < 0)
344 continue;
345 if (prev && prev->op == OP_TEXBAR) {
346 if (prev->subOp > useVec[i].level)
347 prev->subOp = useVec[i].level;
348 prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0));
349 } else {
350 Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE);
351 bar->fixed = 1;
352 bar->subOp = useVec[i].level;
353 // make use explicit to ease latency calculation
354 bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0));
355 useVec[i].insn->bb->insertBefore(useVec[i].insn, bar);
356 }
357 }
358
359 if (fn->getProgram()->optLevel < 3)
360 return true;
361
362 std::vector<Limits> limitT, limitB, limitS; // entry, exit, single
363
364 limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0));
365 limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0));
366 limitS.resize(fn->allBBlocks.getSize());
367
368 // cull unneeded barriers (should do that earlier, but for simplicity)
369 IteratorRef bi = fn->cfg.iteratorCFG();
370 // first calculate min/max outstanding TEXes for each BB
371 for (bi->reset(); !bi->end(); bi->next()) {
372 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
373 BasicBlock *bb = BasicBlock::get(n);
374 int min = 0;
375 int max = std::numeric_limits<int>::max();
376 for (Instruction *i = bb->getFirst(); i; i = i->next) {
377 if (isTextureOp(i->op)) {
378 min++;
379 if (max < std::numeric_limits<int>::max())
380 max++;
381 } else
382 if (i->op == OP_TEXBAR) {
383 min = MIN2(min, i->subOp);
384 max = MIN2(max, i->subOp);
385 }
386 }
387 // limits when looking at an isolated block
388 limitS[bb->getId()].min = min;
389 limitS[bb->getId()].max = max;
390 }
391 // propagate the min/max values
392 for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) {
393 for (bi->reset(); !bi->end(); bi->next()) {
394 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
395 BasicBlock *bb = BasicBlock::get(n);
396 const int bbId = bb->getId();
397 for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) {
398 BasicBlock *in = BasicBlock::get(ei.getNode());
399 const int inId = in->getId();
400 limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min);
401 limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max);
402 }
403 // I just hope this is correct ...
404 if (limitS[bbId].max == std::numeric_limits<int>::max()) {
405 // no barrier
406 limitB[bbId].min = limitT[bbId].min + limitS[bbId].min;
407 limitB[bbId].max = limitT[bbId].max + limitS[bbId].min;
408 } else {
409 // block contained a barrier
410 limitB[bbId].min = MIN2(limitS[bbId].max,
411 limitT[bbId].min + limitS[bbId].min);
412 limitB[bbId].max = MIN2(limitS[bbId].max,
413 limitT[bbId].max + limitS[bbId].min);
414 }
415 }
416 }
417 // finally delete unnecessary barriers
418 for (bi->reset(); !bi->end(); bi->next()) {
419 Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get());
420 BasicBlock *bb = BasicBlock::get(n);
421 Instruction *prev = NULL;
422 Instruction *next;
423 int max = limitT[bb->getId()].max;
424 for (Instruction *i = bb->getFirst(); i; i = next) {
425 next = i->next;
426 if (i->op == OP_TEXBAR) {
427 if (i->subOp >= max) {
428 delete_Instruction(prog, i);
429 i = NULL;
430 } else {
431 max = i->subOp;
432 if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) {
433 delete_Instruction(prog, prev);
434 prev = NULL;
435 }
436 }
437 } else
438 if (isTextureOp(i->op)) {
439 max++;
440 }
441 if (i && !i->isNop())
442 prev = i;
443 }
444 }
445 return true;
446 }
447
448 bool
449 NVC0LegalizePostRA::visit(Function *fn)
450 {
451 if (needTexBar)
452 insertTextureBarriers(fn);
453
454 rZero = new_LValue(fn, FILE_GPR);
455 pOne = new_LValue(fn, FILE_PREDICATE);
456 carry = new_LValue(fn, FILE_FLAGS);
457
458 rZero->reg.data.id = prog->getTarget()->getFileSize(FILE_GPR);
459 carry->reg.data.id = 0;
460 pOne->reg.data.id = 7;
461
462 return true;
463 }
464
465 void
466 NVC0LegalizePostRA::replaceZero(Instruction *i)
467 {
468 for (int s = 0; i->srcExists(s); ++s) {
469 if (s == 2 && i->op == OP_SUCLAMP)
470 continue;
471 ImmediateValue *imm = i->getSrc(s)->asImm();
472 if (imm) {
473 if (i->op == OP_SELP && s == 2) {
474 i->setSrc(s, pOne);
475 if (imm->reg.data.u64 == 0)
476 i->src(s).mod = i->src(s).mod ^ Modifier(NV50_IR_MOD_NOT);
477 } else if (imm->reg.data.u64 == 0) {
478 i->setSrc(s, rZero);
479 }
480 }
481 }
482 }
483
484 // replace CONT with BRA for single unconditional continue
485 bool
486 NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb)
487 {
488 if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT)
489 return false;
490 Graph::EdgeIterator ei = bb->cfg.incident();
491 if (ei.getType() != Graph::Edge::BACK)
492 ei.next();
493 if (ei.getType() != Graph::Edge::BACK)
494 return false;
495 BasicBlock *contBB = BasicBlock::get(ei.getNode());
496
497 if (!contBB->getExit() || contBB->getExit()->op != OP_CONT ||
498 contBB->getExit()->getPredicate())
499 return false;
500 contBB->getExit()->op = OP_BRA;
501 bb->remove(bb->getEntry()); // delete PRECONT
502
503 ei.next();
504 assert(ei.end() || ei.getType() != Graph::Edge::BACK);
505 return true;
506 }
507
508 // replace branches to join blocks with join ops
509 void
510 NVC0LegalizePostRA::propagateJoin(BasicBlock *bb)
511 {
512 if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit)
513 return;
514 for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) {
515 BasicBlock *in = BasicBlock::get(ei.getNode());
516 Instruction *exit = in->getExit();
517 if (!exit) {
518 in->insertTail(new FlowInstruction(func, OP_JOIN, bb));
519 // there should always be a terminator instruction
520 WARN("inserted missing terminator in BB:%i\n", in->getId());
521 } else
522 if (exit->op == OP_BRA) {
523 exit->op = OP_JOIN;
524 exit->asFlow()->limit = 1; // must-not-propagate marker
525 }
526 }
527 bb->remove(bb->getEntry());
528 }
529
530 bool
531 NVC0LegalizePostRA::visit(BasicBlock *bb)
532 {
533 Instruction *i, *next;
534
535 // remove pseudo operations and non-fixed no-ops, split 64 bit operations
536 for (i = bb->getFirst(); i; i = next) {
537 next = i->next;
538 if (i->op == OP_EMIT || i->op == OP_RESTART) {
539 if (!i->getDef(0)->refCount())
540 i->setDef(0, NULL);
541 if (i->src(0).getFile() == FILE_IMMEDIATE)
542 i->setSrc(0, rZero); // initial value must be 0
543 replaceZero(i);
544 } else
545 if (i->isNop()) {
546 bb->remove(i);
547 } else
548 if (i->op == OP_BAR && i->subOp == NV50_IR_SUBOP_BAR_SYNC &&
549 prog->getType() != Program::TYPE_COMPUTE) {
550 // It seems like barriers are never required for tessellation since
551 // the warp size is 32, and there are always at most 32 tcs threads.
552 bb->remove(i);
553 } else
554 if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LDC_IS) {
555 int offset = i->src(0).get()->reg.data.offset;
556 if (abs(offset) > 0x10000)
557 i->src(0).get()->reg.fileIndex += offset >> 16;
558 i->src(0).get()->reg.data.offset = (int)(short)offset;
559 } else {
560 // TODO: Move this to before register allocation for operations that
561 // need the $c register !
562 if (typeSizeof(i->dType) == 8) {
563 Instruction *hi;
564 hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry);
565 if (hi)
566 next = hi;
567 }
568
569 if (i->op != OP_MOV && i->op != OP_PFETCH)
570 replaceZero(i);
571 }
572 }
573 if (!bb->getEntry())
574 return true;
575
576 if (!tryReplaceContWithBra(bb))
577 propagateJoin(bb);
578
579 return true;
580 }
581
582 NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget())
583 {
584 bld.setProgram(prog);
585 gMemBase = NULL;
586 }
587
588 bool
589 NVC0LoweringPass::visit(Function *fn)
590 {
591 if (prog->getType() == Program::TYPE_GEOMETRY) {
592 assert(!strncmp(fn->getName(), "MAIN", 4));
593 // TODO: when we generate actual functions pass this value along somehow
594 bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false);
595 gpEmitAddress = bld.loadImm(NULL, 0)->asLValue();
596 if (fn->cfgExit) {
597 bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false);
598 bld.mkMovToReg(0, gpEmitAddress);
599 }
600 }
601 return true;
602 }
603
604 bool
605 NVC0LoweringPass::visit(BasicBlock *bb)
606 {
607 return true;
608 }
609
610 inline Value *
611 NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot)
612 {
613 uint8_t b = prog->driver->io.auxCBSlot;
614 uint32_t off = prog->driver->io.texBindBase + slot * 4;
615 return bld.
616 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
617 }
618
619 // move array source to first slot, convert to u16, add indirections
620 bool
621 NVC0LoweringPass::handleTEX(TexInstruction *i)
622 {
623 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
624 const int arg = i->tex.target.getArgCount();
625 const int lyr = arg - (i->tex.target.isMS() ? 2 : 1);
626 const int chipset = prog->getTarget()->getChipset();
627
628 /* Only normalize in the non-explicit derivatives case. For explicit
629 * derivatives, this is handled in handleManualTXD.
630 */
631 if (i->tex.target.isCube() && i->dPdx[0].get() == NULL) {
632 Value *src[3], *val;
633 int c;
634 for (c = 0; c < 3; ++c)
635 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), i->getSrc(c));
636 val = bld.getScratch();
637 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
638 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
639 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
640 for (c = 0; c < 3; ++c) {
641 i->setSrc(c, bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(),
642 i->getSrc(c), val));
643 }
644 }
645
646 // Arguments to the TEX instruction are a little insane. Even though the
647 // encoding is identical between SM20 and SM30, the arguments mean
648 // different things between Fermi and Kepler+. A lot of arguments are
649 // optional based on flags passed to the instruction. This summarizes the
650 // order of things.
651 //
652 // Fermi:
653 // array/indirect
654 // coords
655 // sample
656 // lod bias
657 // depth compare
658 // offsets:
659 // - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg)
660 // - other: 4 bits each, single reg
661 //
662 // Kepler+:
663 // indirect handle
664 // array (+ offsets for txd in upper 16 bits)
665 // coords
666 // sample
667 // lod bias
668 // depth compare
669 // offsets (same as fermi, except txd which takes it with array)
670 //
671 // Maxwell (tex):
672 // array
673 // coords
674 // indirect handle
675 // sample
676 // lod bias
677 // depth compare
678 // offsets
679 //
680 // Maxwell (txd):
681 // indirect handle
682 // coords
683 // array + offsets
684 // derivatives
685
686 if (chipset >= NVISA_GK104_CHIPSET) {
687 if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
688 // XXX this ignores tsc, and assumes a 1:1 mapping
689 assert(i->tex.rIndirectSrc >= 0);
690 Value *hnd = loadTexHandle(
691 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
692 i->getIndirectR(), bld.mkImm(2)),
693 i->tex.r);
694 i->tex.r = 0xff;
695 i->tex.s = 0x1f;
696 i->setIndirectR(hnd);
697 i->setIndirectS(NULL);
698 } else if (i->tex.r == i->tex.s || i->op == OP_TXF) {
699 i->tex.r += prog->driver->io.texBindBase / 4;
700 i->tex.s = 0; // only a single cX[] value possible here
701 } else {
702 Value *hnd = bld.getScratch();
703 Value *rHnd = loadTexHandle(NULL, i->tex.r);
704 Value *sHnd = loadTexHandle(NULL, i->tex.s);
705
706 bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd);
707
708 i->tex.r = 0; // not used for indirect tex
709 i->tex.s = 0;
710 i->setIndirectR(hnd);
711 }
712 if (i->tex.target.isArray()) {
713 LValue *layer = new_LValue(func, FILE_GPR);
714 Value *src = i->getSrc(lyr);
715 const int sat = (i->op == OP_TXF) ? 1 : 0;
716 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
717 bld.mkCvt(OP_CVT, TYPE_U16, layer, sTy, src)->saturate = sat;
718 if (i->op != OP_TXD || chipset < NVISA_GM107_CHIPSET) {
719 for (int s = dim; s >= 1; --s)
720 i->setSrc(s, i->getSrc(s - 1));
721 i->setSrc(0, layer);
722 } else {
723 i->setSrc(dim, layer);
724 }
725 }
726 // Move the indirect reference to the first place
727 if (i->tex.rIndirectSrc >= 0 && (
728 i->op == OP_TXD || chipset < NVISA_GM107_CHIPSET)) {
729 Value *hnd = i->getIndirectR();
730
731 i->setIndirectR(NULL);
732 i->moveSources(0, 1);
733 i->setSrc(0, hnd);
734 i->tex.rIndirectSrc = 0;
735 i->tex.sIndirectSrc = -1;
736 }
737 } else
738 // (nvc0) generate and move the tsc/tic/array source to the front
739 if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) {
740 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
741
742 Value *ticRel = i->getIndirectR();
743 Value *tscRel = i->getIndirectS();
744
745 if (ticRel) {
746 i->setSrc(i->tex.rIndirectSrc, NULL);
747 if (i->tex.r)
748 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
749 ticRel, bld.mkImm(i->tex.r));
750 }
751 if (tscRel) {
752 i->setSrc(i->tex.sIndirectSrc, NULL);
753 if (i->tex.s)
754 tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
755 tscRel, bld.mkImm(i->tex.s));
756 }
757
758 Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL;
759 if (arrayIndex) {
760 for (int s = dim; s >= 1; --s)
761 i->setSrc(s, i->getSrc(s - 1));
762 i->setSrc(0, arrayIndex);
763 } else {
764 i->moveSources(0, 1);
765 }
766
767 if (arrayIndex) {
768 int sat = (i->op == OP_TXF) ? 1 : 0;
769 DataType sTy = (i->op == OP_TXF) ? TYPE_U32 : TYPE_F32;
770 bld.mkCvt(OP_CVT, TYPE_U16, src, sTy, arrayIndex)->saturate = sat;
771 } else {
772 bld.loadImm(src, 0);
773 }
774
775 if (ticRel)
776 bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src);
777 if (tscRel)
778 bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src);
779
780 i->setSrc(0, src);
781 }
782
783 // For nvc0, the sample id has to be in the second operand, as the offset
784 // does. Right now we don't know how to pass both in, and this case can't
785 // happen with OpenGL. On nve0, the sample id is part of the texture
786 // coordinate argument.
787 assert(chipset >= NVISA_GK104_CHIPSET ||
788 !i->tex.useOffsets || !i->tex.target.isMS());
789
790 // offset is between lod and dc
791 if (i->tex.useOffsets) {
792 int n, c;
793 int s = i->srcCount(0xff, true);
794 if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) {
795 if (i->tex.target.isShadow())
796 s--;
797 if (i->srcExists(s)) // move potential predicate out of the way
798 i->moveSources(s, 1);
799 if (i->tex.useOffsets == 4 && i->srcExists(s + 1))
800 i->moveSources(s + 1, 1);
801 }
802 if (i->op == OP_TXG) {
803 // Either there is 1 offset, which goes into the 2 low bytes of the
804 // first source, or there are 4 offsets, which go into 2 sources (8
805 // values, 1 byte each).
806 Value *offs[2] = {NULL, NULL};
807 for (n = 0; n < i->tex.useOffsets; n++) {
808 for (c = 0; c < 2; ++c) {
809 if ((n % 2) == 0 && c == 0)
810 offs[n / 2] = i->offset[n][c].get();
811 else
812 bld.mkOp3(OP_INSBF, TYPE_U32,
813 offs[n / 2],
814 i->offset[n][c].get(),
815 bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)),
816 offs[n / 2]);
817 }
818 }
819 i->setSrc(s, offs[0]);
820 if (offs[1])
821 i->setSrc(s + 1, offs[1]);
822 } else {
823 unsigned imm = 0;
824 assert(i->tex.useOffsets == 1);
825 for (c = 0; c < 3; ++c) {
826 ImmediateValue val;
827 if (!i->offset[0][c].getImmediate(val))
828 assert(!"non-immediate offset passed to non-TXG");
829 imm |= (val.reg.data.u32 & 0xf) << (c * 4);
830 }
831 if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) {
832 // The offset goes into the upper 16 bits of the array index. So
833 // create it if it's not already there, and INSBF it if it already
834 // is.
835 s = (i->tex.rIndirectSrc >= 0) ? 1 : 0;
836 if (chipset >= NVISA_GM107_CHIPSET)
837 s += dim;
838 if (i->tex.target.isArray()) {
839 bld.mkOp3(OP_INSBF, TYPE_U32, i->getSrc(s),
840 bld.loadImm(NULL, imm), bld.mkImm(0xc10),
841 i->getSrc(s));
842 } else {
843 i->moveSources(s, 1);
844 i->setSrc(s, bld.loadImm(NULL, imm << 16));
845 }
846 } else {
847 i->setSrc(s, bld.loadImm(NULL, imm));
848 }
849 }
850 }
851
852 if (chipset >= NVISA_GK104_CHIPSET) {
853 //
854 // If TEX requires more than 4 sources, the 2nd register tuple must be
855 // aligned to 4, even if it consists of just a single 4-byte register.
856 //
857 // XXX HACK: We insert 0 sources to avoid the 5 or 6 regs case.
858 //
859 int s = i->srcCount(0xff, true);
860 if (s > 4 && s < 7) {
861 if (i->srcExists(s)) // move potential predicate out of the way
862 i->moveSources(s, 7 - s);
863 while (s < 7)
864 i->setSrc(s++, bld.loadImm(NULL, 0));
865 }
866 }
867
868 return true;
869 }
870
871 bool
872 NVC0LoweringPass::handleManualTXD(TexInstruction *i)
873 {
874 static const uint8_t qOps[4][2] =
875 {
876 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }, // l0
877 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(MOV2, MOV2, ADD, ADD) }, // l1
878 { QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l2
879 { QUADOP(SUBR, MOV2, SUBR, MOV2), QUADOP(SUBR, SUBR, MOV2, MOV2) }, // l3
880 };
881 Value *def[4][4];
882 Value *crd[3];
883 Instruction *tex;
884 Value *zero = bld.loadImm(bld.getSSA(), 0);
885 int l, c;
886 const int dim = i->tex.target.getDim() + i->tex.target.isCube();
887
888 // This function is invoked after handleTEX lowering, so we have to expect
889 // the arguments in the order that the hw wants them. For Fermi, array and
890 // indirect are both in the leading arg, while for Kepler, array and
891 // indirect are separate (and both precede the coordinates). Maxwell is
892 // handled in a separate function.
893 unsigned array;
894 if (targ->getChipset() < NVISA_GK104_CHIPSET)
895 array = i->tex.target.isArray() || i->tex.rIndirectSrc >= 0;
896 else
897 array = i->tex.target.isArray() + (i->tex.rIndirectSrc >= 0);
898
899 i->op = OP_TEX; // no need to clone dPdx/dPdy later
900
901 for (c = 0; c < dim; ++c)
902 crd[c] = bld.getScratch();
903
904 bld.mkOp(OP_QUADON, TYPE_NONE, NULL);
905 for (l = 0; l < 4; ++l) {
906 Value *src[3], *val;
907 // mov coordinates from lane l to all lanes
908 for (c = 0; c < dim; ++c)
909 bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero);
910 // add dPdx from lane l to lanes dx
911 for (c = 0; c < dim; ++c)
912 bld.mkQuadop(qOps[l][0], crd[c], l, i->dPdx[c].get(), crd[c]);
913 // add dPdy from lane l to lanes dy
914 for (c = 0; c < dim; ++c)
915 bld.mkQuadop(qOps[l][1], crd[c], l, i->dPdy[c].get(), crd[c]);
916 // normalize cube coordinates
917 if (i->tex.target.isCube()) {
918 for (c = 0; c < 3; ++c)
919 src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), crd[c]);
920 val = bld.getScratch();
921 bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]);
922 bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val);
923 bld.mkOp1(OP_RCP, TYPE_F32, val, val);
924 for (c = 0; c < 3; ++c)
925 src[c] = bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(), crd[c], val);
926 } else {
927 for (c = 0; c < dim; ++c)
928 src[c] = crd[c];
929 }
930 // texture
931 bld.insert(tex = cloneForward(func, i));
932 for (c = 0; c < dim; ++c)
933 tex->setSrc(c + array, src[c]);
934 // save results
935 for (c = 0; i->defExists(c); ++c) {
936 Instruction *mov;
937 def[c][l] = bld.getSSA();
938 mov = bld.mkMov(def[c][l], tex->getDef(c));
939 mov->fixed = 1;
940 mov->lanes = 1 << l;
941 }
942 }
943 bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL);
944
945 for (c = 0; i->defExists(c); ++c) {
946 Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c));
947 for (l = 0; l < 4; ++l)
948 u->setSrc(l, def[c][l]);
949 }
950
951 i->bb->remove(i);
952 return true;
953 }
954
955 bool
956 NVC0LoweringPass::handleTXD(TexInstruction *txd)
957 {
958 int dim = txd->tex.target.getDim() + txd->tex.target.isCube();
959 unsigned arg = txd->tex.target.getArgCount();
960 unsigned expected_args = arg;
961 const int chipset = prog->getTarget()->getChipset();
962
963 if (chipset >= NVISA_GK104_CHIPSET) {
964 if (!txd->tex.target.isArray() && txd->tex.useOffsets)
965 expected_args++;
966 if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)
967 expected_args++;
968 } else {
969 if (txd->tex.useOffsets)
970 expected_args++;
971 if (!txd->tex.target.isArray() && (
972 txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0))
973 expected_args++;
974 }
975
976 if (expected_args > 4 ||
977 dim > 2 ||
978 txd->tex.target.isShadow())
979 txd->op = OP_TEX;
980
981 handleTEX(txd);
982 while (txd->srcExists(arg))
983 ++arg;
984
985 txd->tex.derivAll = true;
986 if (txd->op == OP_TEX)
987 return handleManualTXD(txd);
988
989 assert(arg == expected_args);
990 for (int c = 0; c < dim; ++c) {
991 txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]);
992 txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]);
993 txd->dPdx[c].set(NULL);
994 txd->dPdy[c].set(NULL);
995 }
996
997 // In this case we have fewer than 4 "real" arguments, which means that
998 // handleTEX didn't apply any padding. However we have to make sure that
999 // the second "group" of arguments still gets padded up to 4.
1000 if (chipset >= NVISA_GK104_CHIPSET) {
1001 int s = arg + 2 * dim;
1002 if (s >= 4 && s < 7) {
1003 if (txd->srcExists(s)) // move potential predicate out of the way
1004 txd->moveSources(s, 7 - s);
1005 while (s < 7)
1006 txd->setSrc(s++, bld.loadImm(NULL, 0));
1007 }
1008 }
1009
1010 return true;
1011 }
1012
1013 bool
1014 NVC0LoweringPass::handleTXQ(TexInstruction *txq)
1015 {
1016 const int chipset = prog->getTarget()->getChipset();
1017 if (chipset >= NVISA_GK104_CHIPSET && txq->tex.rIndirectSrc < 0)
1018 txq->tex.r += prog->driver->io.texBindBase / 4;
1019
1020 if (txq->tex.rIndirectSrc < 0)
1021 return true;
1022
1023 Value *ticRel = txq->getIndirectR();
1024
1025 txq->setIndirectS(NULL);
1026 txq->tex.sIndirectSrc = -1;
1027
1028 assert(ticRel);
1029
1030 if (chipset < NVISA_GK104_CHIPSET) {
1031 LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa
1032
1033 txq->setSrc(txq->tex.rIndirectSrc, NULL);
1034 if (txq->tex.r)
1035 ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(),
1036 ticRel, bld.mkImm(txq->tex.r));
1037
1038 bld.mkOp2(OP_SHL, TYPE_U32, src, ticRel, bld.mkImm(0x17));
1039
1040 txq->moveSources(0, 1);
1041 txq->setSrc(0, src);
1042 } else {
1043 Value *hnd = loadTexHandle(
1044 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1045 txq->getIndirectR(), bld.mkImm(2)),
1046 txq->tex.r);
1047 txq->tex.r = 0xff;
1048 txq->tex.s = 0x1f;
1049
1050 txq->setIndirectR(NULL);
1051 txq->moveSources(0, 1);
1052 txq->setSrc(0, hnd);
1053 txq->tex.rIndirectSrc = 0;
1054 }
1055
1056 return true;
1057 }
1058
1059 bool
1060 NVC0LoweringPass::handleTXLQ(TexInstruction *i)
1061 {
1062 /* The outputs are inverted compared to what the TGSI instruction
1063 * expects. Take that into account in the mask.
1064 */
1065 assert((i->tex.mask & ~3) == 0);
1066 if (i->tex.mask == 1)
1067 i->tex.mask = 2;
1068 else if (i->tex.mask == 2)
1069 i->tex.mask = 1;
1070 handleTEX(i);
1071 bld.setPosition(i, true);
1072
1073 /* The returned values are not quite what we want:
1074 * (a) convert from s16/u16 to f32
1075 * (b) multiply by 1/256
1076 */
1077 for (int def = 0; def < 2; ++def) {
1078 if (!i->defExists(def))
1079 continue;
1080 enum DataType type = TYPE_S16;
1081 if (i->tex.mask == 2 || def > 0)
1082 type = TYPE_U16;
1083 bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def));
1084 bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def),
1085 i->getDef(def), bld.loadImm(NULL, 1.0f / 256));
1086 }
1087 if (i->tex.mask == 3) {
1088 LValue *t = new_LValue(func, FILE_GPR);
1089 bld.mkMov(t, i->getDef(0));
1090 bld.mkMov(i->getDef(0), i->getDef(1));
1091 bld.mkMov(i->getDef(1), t);
1092 }
1093 return true;
1094 }
1095
1096 bool
1097 NVC0LoweringPass::handleBUFQ(Instruction *bufq)
1098 {
1099 bufq->op = OP_MOV;
1100 bufq->setSrc(0, loadBufLength32(bufq->getIndirect(0, 1),
1101 bufq->getSrc(0)->reg.fileIndex * 16));
1102 bufq->setIndirect(0, 0, NULL);
1103 bufq->setIndirect(0, 1, NULL);
1104 return true;
1105 }
1106
1107 void
1108 NVC0LoweringPass::handleSharedATOMNVE4(Instruction *atom)
1109 {
1110 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1111
1112 BasicBlock *currBB = atom->bb;
1113 BasicBlock *tryLockBB = atom->bb->splitBefore(atom, false);
1114 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1115 BasicBlock *setAndUnlockBB = new BasicBlock(func);
1116 BasicBlock *failLockBB = new BasicBlock(func);
1117
1118 bld.setPosition(currBB, true);
1119 assert(!currBB->joinAt);
1120 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1121
1122 CmpInstruction *pred =
1123 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1124 TYPE_U32, bld.mkImm(0), bld.mkImm(1));
1125
1126 bld.mkFlow(OP_BRA, tryLockBB, CC_ALWAYS, NULL);
1127 currBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::TREE);
1128
1129 bld.setPosition(tryLockBB, true);
1130
1131 Instruction *ld =
1132 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1133 atom->getIndirect(0, 0));
1134 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1135 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1136
1137 bld.mkFlow(OP_BRA, setAndUnlockBB, CC_P, ld->getDef(1));
1138 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1139 tryLockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::CROSS);
1140 tryLockBB->cfg.attach(&setAndUnlockBB->cfg, Graph::Edge::TREE);
1141
1142 tryLockBB->cfg.detach(&joinBB->cfg);
1143 bld.remove(atom);
1144
1145 bld.setPosition(setAndUnlockBB, true);
1146 Value *stVal;
1147 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1148 // Read the old value, and write the new one.
1149 stVal = atom->getSrc(1);
1150 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1151 CmpInstruction *set =
1152 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(),
1153 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1154
1155 bld.mkCmp(OP_SLCT, CC_NE, TYPE_U32, (stVal = bld.getSSA()),
1156 TYPE_U32, atom->getSrc(2), ld->getDef(0), set->getDef(0));
1157 } else {
1158 operation op;
1159
1160 switch (atom->subOp) {
1161 case NV50_IR_SUBOP_ATOM_ADD:
1162 op = OP_ADD;
1163 break;
1164 case NV50_IR_SUBOP_ATOM_AND:
1165 op = OP_AND;
1166 break;
1167 case NV50_IR_SUBOP_ATOM_OR:
1168 op = OP_OR;
1169 break;
1170 case NV50_IR_SUBOP_ATOM_XOR:
1171 op = OP_XOR;
1172 break;
1173 case NV50_IR_SUBOP_ATOM_MIN:
1174 op = OP_MIN;
1175 break;
1176 case NV50_IR_SUBOP_ATOM_MAX:
1177 op = OP_MAX;
1178 break;
1179 default:
1180 assert(0);
1181 return;
1182 }
1183
1184 stVal = bld.mkOp2v(op, atom->dType, bld.getSSA(), ld->getDef(0),
1185 atom->getSrc(1));
1186 }
1187
1188 Instruction *st =
1189 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1190 atom->getIndirect(0, 0), stVal);
1191 st->setDef(0, pred->getDef(0));
1192 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1193
1194 bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL);
1195 setAndUnlockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::TREE);
1196
1197 // Lock until the store has not been performed.
1198 bld.setPosition(failLockBB, true);
1199 bld.mkFlow(OP_BRA, tryLockBB, CC_NOT_P, pred->getDef(0));
1200 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1201 failLockBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::BACK);
1202 failLockBB->cfg.attach(&joinBB->cfg, Graph::Edge::TREE);
1203
1204 bld.setPosition(joinBB, false);
1205 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1206 }
1207
1208 void
1209 NVC0LoweringPass::handleSharedATOM(Instruction *atom)
1210 {
1211 assert(atom->src(0).getFile() == FILE_MEMORY_SHARED);
1212
1213 BasicBlock *currBB = atom->bb;
1214 BasicBlock *tryLockAndSetBB = atom->bb->splitBefore(atom, false);
1215 BasicBlock *joinBB = atom->bb->splitAfter(atom);
1216
1217 bld.setPosition(currBB, true);
1218 assert(!currBB->joinAt);
1219 currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL);
1220
1221 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_ALWAYS, NULL);
1222 currBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::TREE);
1223
1224 bld.setPosition(tryLockAndSetBB, true);
1225
1226 Instruction *ld =
1227 bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(),
1228 atom->getIndirect(0, 0));
1229 ld->setDef(1, bld.getSSA(1, FILE_PREDICATE));
1230 ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED;
1231
1232 Value *stVal;
1233 if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
1234 // Read the old value, and write the new one.
1235 stVal = atom->getSrc(1);
1236 } else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1237 CmpInstruction *set =
1238 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1239 TYPE_U32, ld->getDef(0), atom->getSrc(1));
1240 set->setPredicate(CC_P, ld->getDef(1));
1241
1242 Instruction *selp =
1243 bld.mkOp3(OP_SELP, TYPE_U32, bld.getSSA(), ld->getDef(0),
1244 atom->getSrc(2), set->getDef(0));
1245 selp->src(2).mod = Modifier(NV50_IR_MOD_NOT);
1246 selp->setPredicate(CC_P, ld->getDef(1));
1247
1248 stVal = selp->getDef(0);
1249 } else {
1250 operation op;
1251
1252 switch (atom->subOp) {
1253 case NV50_IR_SUBOP_ATOM_ADD:
1254 op = OP_ADD;
1255 break;
1256 case NV50_IR_SUBOP_ATOM_AND:
1257 op = OP_AND;
1258 break;
1259 case NV50_IR_SUBOP_ATOM_OR:
1260 op = OP_OR;
1261 break;
1262 case NV50_IR_SUBOP_ATOM_XOR:
1263 op = OP_XOR;
1264 break;
1265 case NV50_IR_SUBOP_ATOM_MIN:
1266 op = OP_MIN;
1267 break;
1268 case NV50_IR_SUBOP_ATOM_MAX:
1269 op = OP_MAX;
1270 break;
1271 default:
1272 assert(0);
1273 return;
1274 }
1275
1276 Instruction *i =
1277 bld.mkOp2(op, atom->dType, bld.getSSA(), ld->getDef(0),
1278 atom->getSrc(1));
1279 i->setPredicate(CC_P, ld->getDef(1));
1280
1281 stVal = i->getDef(0);
1282 }
1283
1284 Instruction *st =
1285 bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(),
1286 atom->getIndirect(0, 0), stVal);
1287 st->setPredicate(CC_P, ld->getDef(1));
1288 st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED;
1289
1290 // Loop until the lock is acquired.
1291 bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_NOT_P, ld->getDef(1));
1292 tryLockAndSetBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::BACK);
1293 tryLockAndSetBB->cfg.attach(&joinBB->cfg, Graph::Edge::CROSS);
1294 bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL);
1295
1296 bld.remove(atom);
1297
1298 bld.setPosition(joinBB, false);
1299 bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1;
1300 }
1301
1302 bool
1303 NVC0LoweringPass::handleATOM(Instruction *atom)
1304 {
1305 SVSemantic sv;
1306 Value *ptr = atom->getIndirect(0, 0), *ind = atom->getIndirect(0, 1), *base;
1307
1308 switch (atom->src(0).getFile()) {
1309 case FILE_MEMORY_LOCAL:
1310 sv = SV_LBASE;
1311 break;
1312 case FILE_MEMORY_SHARED:
1313 // For Fermi/Kepler, we have to use ld lock/st unlock to perform atomic
1314 // operations on shared memory. For Maxwell, ATOMS is enough.
1315 if (targ->getChipset() < NVISA_GK104_CHIPSET)
1316 handleSharedATOM(atom);
1317 else if (targ->getChipset() < NVISA_GM107_CHIPSET)
1318 handleSharedATOMNVE4(atom);
1319 return true;
1320 default:
1321 assert(atom->src(0).getFile() == FILE_MEMORY_BUFFER);
1322 base = loadBufInfo64(ind, atom->getSrc(0)->reg.fileIndex * 16);
1323 assert(base->reg.size == 8);
1324 if (ptr)
1325 base = bld.mkOp2v(OP_ADD, TYPE_U64, base, base, ptr);
1326 assert(base->reg.size == 8);
1327 atom->setIndirect(0, 0, base);
1328 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1329
1330 // Harden against out-of-bounds accesses
1331 Value *offset = bld.loadImm(NULL, atom->getSrc(0)->reg.data.offset + typeSizeof(atom->sType));
1332 Value *length = loadBufLength32(ind, atom->getSrc(0)->reg.fileIndex * 16);
1333 Value *pred = new_LValue(func, FILE_PREDICATE);
1334 if (ptr)
1335 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, ptr);
1336 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
1337 atom->setPredicate(CC_NOT_P, pred);
1338 if (atom->defExists(0)) {
1339 Value *zero, *dst = atom->getDef(0);
1340 atom->setDef(0, bld.getSSA());
1341
1342 bld.setPosition(atom, true);
1343 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
1344 ->setPredicate(CC_P, pred);
1345 bld.mkOp2(OP_UNION, TYPE_U32, dst, atom->getDef(0), zero);
1346 }
1347
1348 return true;
1349 }
1350 base =
1351 bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0));
1352
1353 atom->setSrc(0, cloneShallow(func, atom->getSrc(0)));
1354 atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
1355 if (ptr)
1356 base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr);
1357 atom->setIndirect(0, 1, NULL);
1358 atom->setIndirect(0, 0, base);
1359
1360 return true;
1361 }
1362
1363 bool
1364 NVC0LoweringPass::handleCasExch(Instruction *cas, bool needCctl)
1365 {
1366 if (targ->getChipset() < NVISA_GM107_CHIPSET) {
1367 if (cas->src(0).getFile() == FILE_MEMORY_SHARED) {
1368 // ATOM_CAS and ATOM_EXCH are handled in handleSharedATOM().
1369 return false;
1370 }
1371 }
1372
1373 if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS &&
1374 cas->subOp != NV50_IR_SUBOP_ATOM_EXCH)
1375 return false;
1376 bld.setPosition(cas, true);
1377
1378 if (needCctl) {
1379 Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, cas->getSrc(0));
1380 cctl->setIndirect(0, 0, cas->getIndirect(0, 0));
1381 cctl->fixed = 1;
1382 cctl->subOp = NV50_IR_SUBOP_CCTL_IV;
1383 if (cas->isPredicated())
1384 cctl->setPredicate(cas->cc, cas->getPredicate());
1385 }
1386
1387 if (cas->subOp == NV50_IR_SUBOP_ATOM_CAS) {
1388 // CAS is crazy. It's 2nd source is a double reg, and the 3rd source
1389 // should be set to the high part of the double reg or bad things will
1390 // happen elsewhere in the universe.
1391 // Also, it sometimes returns the new value instead of the old one
1392 // under mysterious circumstances.
1393 Value *dreg = bld.getSSA(8);
1394 bld.setPosition(cas, false);
1395 bld.mkOp2(OP_MERGE, TYPE_U64, dreg, cas->getSrc(1), cas->getSrc(2));
1396 cas->setSrc(1, dreg);
1397 cas->setSrc(2, dreg);
1398 }
1399
1400 return true;
1401 }
1402
1403 inline Value *
1404 NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off, uint16_t base)
1405 {
1406 uint8_t b = prog->driver->io.auxCBSlot;
1407 off += base;
1408
1409 return bld.
1410 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1411 }
1412
1413 inline Value *
1414 NVC0LoweringPass::loadResInfo64(Value *ptr, uint32_t off, uint16_t base)
1415 {
1416 uint8_t b = prog->driver->io.auxCBSlot;
1417 off += base;
1418
1419 if (ptr)
1420 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1421
1422 return bld.
1423 mkLoadv(TYPE_U64, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off), ptr);
1424 }
1425
1426 inline Value *
1427 NVC0LoweringPass::loadResLength32(Value *ptr, uint32_t off, uint16_t base)
1428 {
1429 uint8_t b = prog->driver->io.auxCBSlot;
1430 off += base;
1431
1432 if (ptr)
1433 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4));
1434
1435 return bld.
1436 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off + 8), ptr);
1437 }
1438
1439 inline Value *
1440 NVC0LoweringPass::loadSuInfo32(Value *ptr, uint32_t off)
1441 {
1442 return loadResInfo32(ptr, off, prog->driver->io.suInfoBase);
1443 }
1444
1445 inline Value *
1446 NVC0LoweringPass::loadSuInfo64(Value *ptr, uint32_t off)
1447 {
1448 return loadResInfo64(ptr, off, prog->driver->io.suInfoBase);
1449 }
1450
1451 inline Value *
1452 NVC0LoweringPass::loadSuLength32(Value *ptr, uint32_t off)
1453 {
1454 return loadResLength32(ptr, off, prog->driver->io.suInfoBase);
1455 }
1456
1457 inline Value *
1458 NVC0LoweringPass::loadBufInfo32(Value *ptr, uint32_t off)
1459 {
1460 return loadResInfo32(ptr, off, prog->driver->io.bufInfoBase);
1461 }
1462
1463 inline Value *
1464 NVC0LoweringPass::loadBufInfo64(Value *ptr, uint32_t off)
1465 {
1466 return loadResInfo64(ptr, off, prog->driver->io.bufInfoBase);
1467 }
1468
1469 inline Value *
1470 NVC0LoweringPass::loadBufLength32(Value *ptr, uint32_t off)
1471 {
1472 return loadResLength32(ptr, off, prog->driver->io.bufInfoBase);
1473 }
1474
1475 inline Value *
1476 NVC0LoweringPass::loadUboInfo32(Value *ptr, uint32_t off)
1477 {
1478 return loadResInfo32(ptr, off, prog->driver->io.uboInfoBase);
1479 }
1480
1481 inline Value *
1482 NVC0LoweringPass::loadUboInfo64(Value *ptr, uint32_t off)
1483 {
1484 return loadResInfo64(ptr, off, prog->driver->io.uboInfoBase);
1485 }
1486
1487 inline Value *
1488 NVC0LoweringPass::loadUboLength32(Value *ptr, uint32_t off)
1489 {
1490 return loadResLength32(ptr, off, prog->driver->io.uboInfoBase);
1491 }
1492
1493 inline Value *
1494 NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off)
1495 {
1496 uint8_t b = prog->driver->io.msInfoCBSlot;
1497 off += prog->driver->io.msInfoBase;
1498 return bld.
1499 mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr);
1500 }
1501
1502 /* On nvc0, surface info is obtained via the surface binding points passed
1503 * to the SULD/SUST instructions.
1504 * On nve4, surface info is stored in c[] and is used by various special
1505 * instructions, e.g. for clamping coordiantes or generating an address.
1506 * They couldn't just have added an equivalent to TIC now, couldn't they ?
1507 */
1508 #define NVE4_SU_INFO_ADDR 0x00
1509 #define NVE4_SU_INFO_FMT 0x04
1510 #define NVE4_SU_INFO_DIM_X 0x08
1511 #define NVE4_SU_INFO_PITCH 0x0c
1512 #define NVE4_SU_INFO_DIM_Y 0x10
1513 #define NVE4_SU_INFO_ARRAY 0x14
1514 #define NVE4_SU_INFO_DIM_Z 0x18
1515 #define NVE4_SU_INFO_UNK1C 0x1c
1516 #define NVE4_SU_INFO_WIDTH 0x20
1517 #define NVE4_SU_INFO_HEIGHT 0x24
1518 #define NVE4_SU_INFO_DEPTH 0x28
1519 #define NVE4_SU_INFO_TARGET 0x2c
1520 #define NVE4_SU_INFO_BSIZE 0x30
1521 #define NVE4_SU_INFO_RAW_X 0x34
1522 #define NVE4_SU_INFO_MS_X 0x38
1523 #define NVE4_SU_INFO_MS_Y 0x3c
1524
1525 #define NVE4_SU_INFO__STRIDE 0x40
1526
1527 #define NVE4_SU_INFO_DIM(i) (0x08 + (i) * 8)
1528 #define NVE4_SU_INFO_SIZE(i) (0x20 + (i) * 4)
1529 #define NVE4_SU_INFO_MS(i) (0x38 + (i) * 4)
1530
1531 static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c)
1532 {
1533 switch (su->tex.target.getEnum()) {
1534 case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1);
1535 case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1536 case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1537 case TEX_TARGET_1D_ARRAY: return (c == 1) ?
1538 NV50_IR_SUBOP_SUCLAMP_PL(0, 2) :
1539 NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1540 case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1541 case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2);
1542 case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1543 case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1544 case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1545 case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1546 case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2);
1547 default:
1548 assert(0);
1549 return 0;
1550 }
1551 }
1552
1553 bool
1554 NVC0LoweringPass::handleSUQ(TexInstruction *suq)
1555 {
1556 int dim = suq->tex.target.getDim();
1557 int arg = dim + (suq->tex.target.isArray() || suq->tex.target.isCube());
1558 uint8_t s = prog->driver->io.auxCBSlot;
1559 Value *ind = suq->getIndirectR();
1560 uint32_t base;
1561 int c;
1562
1563 base = prog->driver->io.suInfoBase + suq->tex.r * NVE4_SU_INFO__STRIDE;
1564
1565 if (ind)
1566 ind = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(),
1567 ind, bld.mkImm(6));
1568
1569 for (c = 0; c < arg; ++c) {
1570 if (suq->defExists(c)) {
1571 int offset;
1572
1573 if (c == 1 && suq->tex.target == TEX_TARGET_1D_ARRAY) {
1574 offset = base + NVE4_SU_INFO_SIZE(2);
1575 } else {
1576 offset = base + NVE4_SU_INFO_SIZE(c);
1577 }
1578 bld.mkLoad(TYPE_U32, suq->getDef(c),
1579 bld.mkSymbol(FILE_MEMORY_CONST, s, TYPE_U32, offset), ind);
1580 }
1581 }
1582
1583 if (suq->tex.target.isCube()) {
1584 if (suq->defExists(2)) {
1585 bld.mkOp2(OP_DIV, TYPE_U32, suq->getDef(2), suq->getDef(2),
1586 bld.loadImm(NULL, 6));
1587 }
1588 }
1589
1590 if (suq->defExists(3)) {
1591 // .w contains the number of samples for multi-sampled images but we
1592 // don't support them for now.
1593 bld.mkMov(suq->getDef(3), bld.loadImm(NULL, 1));
1594 }
1595
1596 bld.remove(suq);
1597 return true;
1598 }
1599
1600 void
1601 NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex)
1602 {
1603 const uint16_t base = tex->tex.r * NVE4_SU_INFO__STRIDE;
1604 const int arg = tex->tex.target.getArgCount();
1605
1606 if (tex->tex.target == TEX_TARGET_2D_MS)
1607 tex->tex.target = TEX_TARGET_2D;
1608 else
1609 if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY)
1610 tex->tex.target = TEX_TARGET_2D_ARRAY;
1611 else
1612 return;
1613
1614 Value *x = tex->getSrc(0);
1615 Value *y = tex->getSrc(1);
1616 Value *s = tex->getSrc(arg - 1);
1617
1618 Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA();
1619 Value *ind = NULL;
1620
1621 if (tex->tex.rIndirectSrc >= 0) {
1622 assert(tex->tex.r == 0);
1623 // FIXME: out of bounds
1624 ind = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
1625 tex->getIndirectR(), bld.mkImm(6));
1626 }
1627
1628 Value *ms_x = loadSuInfo32(ind, base + NVE4_SU_INFO_MS(0));
1629 Value *ms_y = loadSuInfo32(ind, base + NVE4_SU_INFO_MS(1));
1630
1631 bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x);
1632 bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y);
1633
1634 s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7));
1635 s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3));
1636
1637 Value *dx = loadMsInfo32(ts, 0x0);
1638 Value *dy = loadMsInfo32(ts, 0x4);
1639
1640 bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx);
1641 bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy);
1642
1643 tex->setSrc(0, tx);
1644 tex->setSrc(1, ty);
1645 tex->moveSources(arg, -1);
1646 }
1647
1648 // Sets 64-bit "generic address", predicate and format sources for SULD/SUST.
1649 // They're computed from the coordinates using the surface info in c[] space.
1650 void
1651 NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su)
1652 {
1653 Instruction *insn;
1654 const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP;
1655 const bool raw =
1656 su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB;
1657 const int idx = su->tex.r;
1658 const int dim = su->tex.target.getDim();
1659 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
1660 const uint16_t base = idx * NVE4_SU_INFO__STRIDE;
1661 int c;
1662 Value *zero = bld.mkImm(0);
1663 Value *p1 = NULL;
1664 Value *v;
1665 Value *src[3];
1666 Value *bf, *eau, *off;
1667 Value *addr, *pred;
1668 Value *ind = NULL;
1669
1670 off = bld.getScratch(4);
1671 bf = bld.getScratch(4);
1672 addr = bld.getSSA(8);
1673 pred = bld.getScratch(1, FILE_PREDICATE);
1674
1675 bld.setPosition(su, false);
1676
1677 adjustCoordinatesMS(su);
1678
1679 if (su->tex.rIndirectSrc >= 0) {
1680 ind = su->getIndirectR();
1681 if (su->tex.r > 0) {
1682 ind = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ind,
1683 bld.loadImm(NULL, su->tex.r));
1684 }
1685 ind = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ind, bld.mkImm(7));
1686 ind = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ind, bld.mkImm(6));
1687 }
1688
1689 // calculate clamped coordinates
1690 for (c = 0; c < arg; ++c) {
1691 int dimc = c;
1692
1693 if (c == 1 && su->tex.target == TEX_TARGET_1D_ARRAY) {
1694 // The array index is stored in the Z component for 1D arrays.
1695 dimc = 2;
1696 }
1697
1698 src[c] = bld.getScratch();
1699 if (c == 0 && raw)
1700 v = loadSuInfo32(ind, base + NVE4_SU_INFO_RAW_X);
1701 else
1702 v = loadSuInfo32(ind, base + NVE4_SU_INFO_DIM(dimc));
1703 bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero)
1704 ->subOp = getSuClampSubOp(su, dimc);
1705 }
1706 for (; c < 3; ++c)
1707 src[c] = zero;
1708
1709 // set predicate output
1710 if (su->tex.target == TEX_TARGET_BUFFER) {
1711 src[0]->getInsn()->setFlagsDef(1, pred);
1712 } else
1713 if (su->tex.target.isArray() || su->tex.target.isCube()) {
1714 p1 = bld.getSSA(1, FILE_PREDICATE);
1715 src[dim]->getInsn()->setFlagsDef(1, p1);
1716 }
1717
1718 // calculate pixel offset
1719 if (dim == 1) {
1720 if (su->tex.target != TEX_TARGET_BUFFER)
1721 bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff));
1722 } else
1723 if (dim == 3) {
1724 v = loadSuInfo32(ind, base + NVE4_SU_INFO_UNK1C);
1725 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1])
1726 ->subOp = NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1727
1728 v = loadSuInfo32(ind, base + NVE4_SU_INFO_PITCH);
1729 bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0])
1730 ->subOp = NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l
1731 } else {
1732 assert(dim == 2);
1733 v = loadSuInfo32(ind, base + NVE4_SU_INFO_PITCH);
1734 bld.mkOp3(OP_MADSP, TYPE_U32, off, src[1], v, src[0])
1735 ->subOp = (su->tex.target.isArray() || su->tex.target.isCube()) ?
1736 NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(4,2,8); // u16l u16l u16l
1737 }
1738
1739 // calculate effective address part 1
1740 if (su->tex.target == TEX_TARGET_BUFFER) {
1741 if (raw) {
1742 bf = src[0];
1743 } else {
1744 v = loadSuInfo32(ind, base + NVE4_SU_INFO_FMT);
1745 bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero)
1746 ->subOp = NV50_IR_SUBOP_V1(7,6,8|2);
1747 }
1748 } else {
1749 Value *y = src[1];
1750 Value *z = src[2];
1751 uint16_t subOp = 0;
1752
1753 switch (dim) {
1754 case 1:
1755 y = zero;
1756 z = zero;
1757 break;
1758 case 2:
1759 z = off;
1760 if (!su->tex.target.isArray() && !su->tex.target.isCube()) {
1761 z = loadSuInfo32(ind, base + NVE4_SU_INFO_UNK1C);
1762 subOp = NV50_IR_SUBOP_SUBFM_3D;
1763 }
1764 break;
1765 default:
1766 subOp = NV50_IR_SUBOP_SUBFM_3D;
1767 assert(dim == 3);
1768 break;
1769 }
1770 insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z);
1771 insn->subOp = subOp;
1772 insn->setFlagsDef(1, pred);
1773 }
1774
1775 // part 2
1776 v = loadSuInfo32(ind, base + NVE4_SU_INFO_ADDR);
1777
1778 if (su->tex.target == TEX_TARGET_BUFFER) {
1779 eau = v;
1780 } else {
1781 eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v);
1782 }
1783 // add array layer offset
1784 if (su->tex.target.isArray() || su->tex.target.isCube()) {
1785 v = loadSuInfo32(ind, base + NVE4_SU_INFO_ARRAY);
1786 if (dim == 1)
1787 bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau)
1788 ->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32
1789 else
1790 bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau)
1791 ->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32
1792 // combine predicates
1793 assert(p1);
1794 bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1);
1795 }
1796
1797 if (atom) {
1798 Value *lo = bf;
1799 if (su->tex.target == TEX_TARGET_BUFFER) {
1800 lo = zero;
1801 bld.mkMov(off, bf);
1802 }
1803 // bf == g[] address & 0xff
1804 // eau == g[] address >> 8
1805 bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau);
1806 bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau);
1807 } else
1808 if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) {
1809 // Convert from u32 to u8 address format, which is what the library code
1810 // doing SULDP currently uses.
1811 // XXX: can SUEAU do this ?
1812 // XXX: does it matter that we don't mask high bytes in bf ?
1813 // Grrr.
1814 bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8));
1815 bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off);
1816 }
1817
1818 bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau);
1819
1820 if (atom && su->tex.target == TEX_TARGET_BUFFER)
1821 bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off);
1822
1823 // let's just set it 0 for raw access and hope it works
1824 v = raw ?
1825 bld.mkImm(0) : loadSuInfo32(ind, base + NVE4_SU_INFO_FMT);
1826
1827 // get rid of old coordinate sources, make space for fmt info and predicate
1828 su->moveSources(arg, 3 - arg);
1829 // set 64 bit address and 32-bit format sources
1830 su->setSrc(0, addr);
1831 su->setSrc(1, v);
1832 su->setSrc(2, pred);
1833
1834 // prevent read fault when the image is not actually bound
1835 CmpInstruction *pred1 =
1836 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
1837 TYPE_U32, bld.mkImm(0),
1838 loadSuInfo32(ind, base + NVE4_SU_INFO_ADDR));
1839
1840 if (su->op != OP_SUSTP && su->tex.format) {
1841 const TexInstruction::ImgFormatDesc *format = su->tex.format;
1842 int blockwidth = format->bits[0] + format->bits[1] +
1843 format->bits[2] + format->bits[3];
1844
1845 // make sure that the format doesn't mismatch
1846 assert(format->components != 0);
1847 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred1->getDef(0),
1848 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
1849 loadSuInfo32(ind, base + NVE4_SU_INFO_BSIZE),
1850 pred1->getDef(0));
1851 }
1852 su->setPredicate(CC_NOT_P, pred1->getDef(0));
1853
1854 // TODO: initialize def values to 0 when the surface operation is not
1855 // performed (not needed for stores). Also, fix the "address bounds test"
1856 // subtests from arb_shader_image_load_store-invalid for buffers, because it
1857 // seems like that the predicate is not correctly set by suclamp.
1858 }
1859
1860 static DataType
1861 getSrcType(const TexInstruction::ImgFormatDesc *t, int c)
1862 {
1863 switch (t->type) {
1864 case FLOAT: return t->bits[c] == 16 ? TYPE_F16 : TYPE_F32;
1865 case UNORM: return t->bits[c] == 8 ? TYPE_U8 : TYPE_U16;
1866 case SNORM: return t->bits[c] == 8 ? TYPE_S8 : TYPE_S16;
1867 case UINT:
1868 return (t->bits[c] == 8 ? TYPE_U8 :
1869 (t->bits[c] == 16 ? TYPE_U16 : TYPE_U32));
1870 case SINT:
1871 return (t->bits[c] == 8 ? TYPE_S8 :
1872 (t->bits[c] == 16 ? TYPE_S16 : TYPE_S32));
1873 }
1874 return TYPE_NONE;
1875 }
1876
1877 static DataType
1878 getDestType(const ImgType type) {
1879 switch (type) {
1880 case FLOAT:
1881 case UNORM:
1882 case SNORM:
1883 return TYPE_F32;
1884 case UINT:
1885 return TYPE_U32;
1886 case SINT:
1887 return TYPE_S32;
1888 default:
1889 assert(!"Impossible type");
1890 return TYPE_NONE;
1891 }
1892 }
1893
1894 void
1895 NVC0LoweringPass::convertSurfaceFormat(TexInstruction *su)
1896 {
1897 const TexInstruction::ImgFormatDesc *format = su->tex.format;
1898 int width = format->bits[0] + format->bits[1] +
1899 format->bits[2] + format->bits[3];
1900 Value *untypedDst[4] = {};
1901 Value *typedDst[4] = {};
1902
1903 // We must convert this to a generic load.
1904 su->op = OP_SULDB;
1905
1906 su->dType = typeOfSize(width / 8);
1907 su->sType = TYPE_U8;
1908
1909 for (int i = 0; i < width / 32; i++)
1910 untypedDst[i] = bld.getSSA();
1911 if (width < 32)
1912 untypedDst[0] = bld.getSSA();
1913
1914 for (int i = 0; i < 4; i++) {
1915 typedDst[i] = su->getDef(i);
1916 }
1917
1918 // Set the untyped dsts as the su's destinations
1919 for (int i = 0; i < 4; i++)
1920 su->setDef(i, untypedDst[i]);
1921
1922 bld.setPosition(su, true);
1923
1924 // Unpack each component into the typed dsts
1925 int bits = 0;
1926 for (int i = 0; i < 4; bits += format->bits[i], i++) {
1927 if (!typedDst[i])
1928 continue;
1929 if (i >= format->components) {
1930 if (format->type == FLOAT ||
1931 format->type == UNORM ||
1932 format->type == SNORM)
1933 bld.loadImm(typedDst[i], i == 3 ? 1.0f : 0.0f);
1934 else
1935 bld.loadImm(typedDst[i], i == 3 ? 1 : 0);
1936 continue;
1937 }
1938
1939 // Get just that component's data into the relevant place
1940 if (format->bits[i] == 32)
1941 bld.mkMov(typedDst[i], untypedDst[i]);
1942 else if (format->bits[i] == 16)
1943 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
1944 getSrcType(format, i), untypedDst[i / 2])
1945 ->subOp = (i & 1) << (format->type == FLOAT ? 0 : 1);
1946 else if (format->bits[i] == 8)
1947 bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i],
1948 getSrcType(format, i), untypedDst[0])->subOp = i;
1949 else {
1950 bld.mkOp2(OP_EXTBF, TYPE_U32, typedDst[i], untypedDst[bits / 32],
1951 bld.mkImm((bits % 32) | (format->bits[i] << 8)));
1952 if (format->type == UNORM || format->type == SNORM)
1953 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], getSrcType(format, i), typedDst[i]);
1954 }
1955
1956 // Normalize / convert as necessary
1957 if (format->type == UNORM)
1958 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << format->bits[i]) - 1)));
1959 else if (format->type == SNORM)
1960 bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << (format->bits[i] - 1)) - 1)));
1961 else if (format->type == FLOAT && format->bits[i] < 16) {
1962 bld.mkOp2(OP_SHL, TYPE_U32, typedDst[i], typedDst[i], bld.loadImm(NULL, 15 - format->bits[i]));
1963 bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], TYPE_F16, typedDst[i]);
1964 }
1965 }
1966 }
1967
1968 void
1969 NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su)
1970 {
1971 processSurfaceCoordsNVE4(su);
1972
1973 if (su->op == OP_SULDP)
1974 convertSurfaceFormat(su);
1975
1976 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
1977 Value *pred = su->getSrc(2);
1978 CondCode cc = CC_NOT_P;
1979 if (su->getPredicate()) {
1980 pred = bld.getScratch(1, FILE_PREDICATE);
1981 cc = su->cc;
1982 if (cc == CC_NOT_P) {
1983 bld.mkOp2(OP_OR, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1984 } else {
1985 bld.mkOp2(OP_AND, TYPE_U8, pred, su->getPredicate(), su->getSrc(2));
1986 pred->getInsn()->src(1).mod = Modifier(NV50_IR_MOD_NOT);
1987 }
1988 }
1989 Instruction *red = bld.mkOp(OP_ATOM, su->dType, bld.getSSA());
1990 red->subOp = su->subOp;
1991 if (!gMemBase)
1992 gMemBase = bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0);
1993 red->setSrc(0, gMemBase);
1994 red->setSrc(1, su->getSrc(3));
1995 if (su->subOp == NV50_IR_SUBOP_ATOM_CAS)
1996 red->setSrc(2, su->getSrc(4));
1997 red->setIndirect(0, 0, su->getSrc(0));
1998
1999 // make sure to initialize dst value when the atomic operation is not
2000 // performed
2001 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2002
2003 assert(cc == CC_NOT_P);
2004 red->setPredicate(cc, pred);
2005 mov->setPredicate(CC_P, pred);
2006
2007 bld.mkOp2(OP_UNION, TYPE_U32, su->getDef(0),
2008 red->getDef(0), mov->getDef(0));
2009
2010 delete_Instruction(bld.getProgram(), su);
2011 handleCasExch(red, true);
2012 }
2013
2014 if (su->op == OP_SUSTB || su->op == OP_SUSTP)
2015 su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8;
2016 }
2017
2018 void
2019 NVC0LoweringPass::processSurfaceCoordsNVC0(TexInstruction *su)
2020 {
2021 const int idx = su->tex.r;
2022 const int dim = su->tex.target.getDim();
2023 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2024 const uint16_t base = idx * NVE4_SU_INFO__STRIDE;
2025 int c;
2026 Value *zero = bld.mkImm(0);
2027 Value *src[3];
2028 Value *v;
2029 Value *ind = NULL;
2030
2031 if (su->tex.rIndirectSrc >= 0) {
2032 ind = su->getIndirectR();
2033 if (su->tex.r > 0) {
2034 ind = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ind,
2035 bld.loadImm(NULL, su->tex.r));
2036 }
2037 ind = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ind, bld.mkImm(7));
2038 ind = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ind, bld.mkImm(6));
2039 }
2040
2041 // get surface coordinates
2042 for (c = 0; c < arg; ++c)
2043 src[c] = su->getSrc(c);
2044 for (; c < 3; ++c)
2045 src[c] = zero;
2046
2047 // calculate pixel offset
2048 if (su->op == OP_SULDP || su->op == OP_SUREDP) {
2049 v = loadSuInfo32(ind, base + NVE4_SU_INFO_BSIZE);
2050 su->setSrc(0, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[0], v));
2051 }
2052
2053 // add array layer offset
2054 if (su->tex.target.isArray() || su->tex.target.isCube()) {
2055 v = loadSuInfo32(ind, base + NVE4_SU_INFO_ARRAY);
2056 assert(dim > 1);
2057 su->setSrc(2, bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[2], v));
2058 }
2059
2060 // prevent read fault when the image is not actually bound
2061 CmpInstruction *pred =
2062 bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE),
2063 TYPE_U32, bld.mkImm(0),
2064 loadSuInfo32(ind, base + NVE4_SU_INFO_ADDR));
2065 if (su->op != OP_SUSTP && su->tex.format) {
2066 const TexInstruction::ImgFormatDesc *format = su->tex.format;
2067 int blockwidth = format->bits[0] + format->bits[1] +
2068 format->bits[2] + format->bits[3];
2069
2070 assert(format->components != 0);
2071 // make sure that the format doesn't mismatch when it's not FMT_NONE
2072 bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0),
2073 TYPE_U32, bld.loadImm(NULL, blockwidth / 8),
2074 loadSuInfo32(ind, base + NVE4_SU_INFO_BSIZE),
2075 pred->getDef(0));
2076 }
2077 su->setPredicate(CC_NOT_P, pred->getDef(0));
2078 }
2079
2080 void
2081 NVC0LoweringPass::handleSurfaceOpNVC0(TexInstruction *su)
2082 {
2083 if (su->tex.target == TEX_TARGET_1D_ARRAY) {
2084 /* As 1d arrays also need 3 coordinates, switching to TEX_TARGET_2D_ARRAY
2085 * will simplify the lowering pass and the texture constraints. */
2086 su->moveSources(1, 1);
2087 su->setSrc(1, bld.loadImm(NULL, 0));
2088 su->tex.target = TEX_TARGET_2D_ARRAY;
2089 }
2090
2091 processSurfaceCoordsNVC0(su);
2092
2093 if (su->op == OP_SULDP)
2094 convertSurfaceFormat(su);
2095
2096 if (su->op == OP_SUREDB || su->op == OP_SUREDP) {
2097 const int dim = su->tex.target.getDim();
2098 const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube());
2099 LValue *addr = bld.getSSA(8);
2100 Value *def = su->getDef(0);
2101
2102 su->op = OP_SULEA;
2103
2104 // Set the destination to the address
2105 su->dType = TYPE_U64;
2106 su->setDef(0, addr);
2107 su->setDef(1, su->getPredicate());
2108
2109 bld.setPosition(su, true);
2110
2111 // Perform the atomic op
2112 Instruction *red = bld.mkOp(OP_ATOM, su->sType, bld.getSSA());
2113 red->subOp = su->subOp;
2114 red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, su->sType, 0));
2115 red->setSrc(1, su->getSrc(arg));
2116 if (red->subOp == NV50_IR_SUBOP_ATOM_CAS)
2117 red->setSrc(2, su->getSrc(arg + 1));
2118 red->setIndirect(0, 0, addr);
2119
2120 // make sure to initialize dst value when the atomic operation is not
2121 // performed
2122 Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0));
2123
2124 assert(su->cc == CC_NOT_P);
2125 red->setPredicate(su->cc, su->getPredicate());
2126 mov->setPredicate(CC_P, su->getPredicate());
2127
2128 bld.mkOp2(OP_UNION, TYPE_U32, def, red->getDef(0), mov->getDef(0));
2129
2130 handleCasExch(red, false);
2131 }
2132 }
2133
2134 bool
2135 NVC0LoweringPass::handleWRSV(Instruction *i)
2136 {
2137 Instruction *st;
2138 Symbol *sym;
2139 uint32_t addr;
2140
2141 // must replace, $sreg are not writeable
2142 addr = targ->getSVAddress(FILE_SHADER_OUTPUT, i->getSrc(0)->asSym());
2143 if (addr >= 0x400)
2144 return false;
2145 sym = bld.mkSymbol(FILE_SHADER_OUTPUT, 0, i->sType, addr);
2146
2147 st = bld.mkStore(OP_EXPORT, i->dType, sym, i->getIndirect(0, 0),
2148 i->getSrc(1));
2149 st->perPatch = i->perPatch;
2150
2151 bld.getBB()->remove(i);
2152 return true;
2153 }
2154
2155 void
2156 NVC0LoweringPass::handleLDST(Instruction *i)
2157 {
2158 if (i->src(0).getFile() == FILE_SHADER_INPUT) {
2159 if (prog->getType() == Program::TYPE_COMPUTE) {
2160 i->getSrc(0)->reg.file = FILE_MEMORY_CONST;
2161 i->getSrc(0)->reg.fileIndex = 0;
2162 } else
2163 if (prog->getType() == Program::TYPE_GEOMETRY &&
2164 i->src(0).isIndirect(0)) {
2165 // XXX: this assumes vec4 units
2166 Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2167 i->getIndirect(0, 0), bld.mkImm(4));
2168 i->setIndirect(0, 0, ptr);
2169 i->op = OP_VFETCH;
2170 } else {
2171 i->op = OP_VFETCH;
2172 assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP
2173 }
2174 } else if (i->src(0).getFile() == FILE_MEMORY_CONST) {
2175 if (targ->getChipset() >= NVISA_GK104_CHIPSET &&
2176 prog->getType() == Program::TYPE_COMPUTE) {
2177 // The launch descriptor only allows to set up 8 CBs, but OpenGL
2178 // requires at least 12 UBOs. To bypass this limitation, we store the
2179 // addrs into the driver constbuf and we directly load from the global
2180 // memory.
2181 int8_t fileIndex = i->getSrc(0)->reg.fileIndex - 1;
2182 Value *ind = i->getIndirect(0, 1);
2183 Value *ptr = loadUboInfo64(ind, fileIndex * 16);
2184
2185 // TODO: clamp the offset to the maximum number of const buf.
2186 if (i->src(0).isIndirect(1)) {
2187 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2188 Value *length = loadUboLength32(ind, fileIndex * 16);
2189 Value *pred = new_LValue(func, FILE_PREDICATE);
2190 if (i->src(0).isIndirect(0)) {
2191 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2192 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2193 }
2194 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2195 i->setIndirect(0, 1, NULL);
2196 i->setIndirect(0, 0, ptr);
2197 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2198 i->setPredicate(CC_NOT_P, pred);
2199 if (i->defExists(0)) {
2200 bld.mkMov(i->getDef(0), bld.mkImm(0));
2201 }
2202 } else if (fileIndex >= 0) {
2203 if (i->src(0).isIndirect(0)) {
2204 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2205 }
2206 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2207 i->setIndirect(0, 1, NULL);
2208 i->setIndirect(0, 0, ptr);
2209 }
2210 } else if (i->src(0).isIndirect(1)) {
2211 Value *ptr;
2212 if (i->src(0).isIndirect(0))
2213 ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(),
2214 i->getIndirect(0, 1), bld.mkImm(0x1010),
2215 i->getIndirect(0, 0));
2216 else
2217 ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2218 i->getIndirect(0, 1), bld.mkImm(16));
2219 i->setIndirect(0, 1, NULL);
2220 i->setIndirect(0, 0, ptr);
2221 i->subOp = NV50_IR_SUBOP_LDC_IS;
2222 }
2223 } else if (i->src(0).getFile() == FILE_SHADER_OUTPUT) {
2224 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
2225 i->op = OP_VFETCH;
2226 } else if (i->src(0).getFile() == FILE_MEMORY_BUFFER) {
2227 Value *ind = i->getIndirect(0, 1);
2228 Value *ptr = loadBufInfo64(ind, i->getSrc(0)->reg.fileIndex * 16);
2229 // XXX come up with a way not to do this for EVERY little access but
2230 // rather to batch these up somehow. Unfortunately we've lost the
2231 // information about the field width by the time we get here.
2232 Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType));
2233 Value *length = loadBufLength32(ind, i->getSrc(0)->reg.fileIndex * 16);
2234 Value *pred = new_LValue(func, FILE_PREDICATE);
2235 if (i->src(0).isIndirect(0)) {
2236 bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0));
2237 bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0));
2238 }
2239 i->setIndirect(0, 1, NULL);
2240 i->setIndirect(0, 0, ptr);
2241 i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL;
2242 bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length);
2243 i->setPredicate(CC_NOT_P, pred);
2244 if (i->defExists(0)) {
2245 Value *zero, *dst = i->getDef(0);
2246 i->setDef(0, bld.getSSA());
2247
2248 bld.setPosition(i, true);
2249 bld.mkMov((zero = bld.getSSA()), bld.mkImm(0))
2250 ->setPredicate(CC_P, pred);
2251 bld.mkOp2(OP_UNION, TYPE_U32, dst, i->getDef(0), zero);
2252 }
2253 }
2254 }
2255
2256 void
2257 NVC0LoweringPass::readTessCoord(LValue *dst, int c)
2258 {
2259 Value *laneid = bld.getSSA();
2260 Value *x, *y;
2261
2262 bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0));
2263
2264 if (c == 0) {
2265 x = dst;
2266 y = NULL;
2267 } else
2268 if (c == 1) {
2269 x = NULL;
2270 y = dst;
2271 } else {
2272 assert(c == 2);
2273 if (prog->driver->prop.tp.domain != PIPE_PRIM_TRIANGLES) {
2274 bld.mkMov(dst, bld.loadImm(NULL, 0));
2275 return;
2276 }
2277 x = bld.getSSA();
2278 y = bld.getSSA();
2279 }
2280 if (x)
2281 bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid);
2282 if (y)
2283 bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid);
2284
2285 if (c == 2) {
2286 bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y);
2287 bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst);
2288 }
2289 }
2290
2291 bool
2292 NVC0LoweringPass::handleRDSV(Instruction *i)
2293 {
2294 Symbol *sym = i->getSrc(0)->asSym();
2295 const SVSemantic sv = sym->reg.data.sv.sv;
2296 Value *vtx = NULL;
2297 Instruction *ld;
2298 uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym);
2299
2300 if (addr >= 0x400) {
2301 // mov $sreg
2302 if (sym->reg.data.sv.index == 3) {
2303 // TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID
2304 i->op = OP_MOV;
2305 i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0));
2306 }
2307 if (sv == SV_VERTEX_COUNT) {
2308 bld.setPosition(i, true);
2309 bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0), bld.mkImm(0x808));
2310 }
2311 return true;
2312 }
2313
2314 switch (sv) {
2315 case SV_POSITION:
2316 assert(prog->getType() == Program::TYPE_FRAGMENT);
2317 if (i->srcExists(1)) {
2318 // Pass offset through to the interpolation logic
2319 ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET,
2320 i->getDef(0), addr, NULL);
2321 ld->setSrc(1, i->getSrc(1));
2322 } else {
2323 bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL);
2324 }
2325 break;
2326 case SV_FACE:
2327 {
2328 Value *face = i->getDef(0);
2329 bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL);
2330 if (i->dType == TYPE_F32) {
2331 bld.mkOp2(OP_OR, TYPE_U32, face, face, bld.mkImm(0x00000001));
2332 bld.mkOp1(OP_NEG, TYPE_S32, face, face);
2333 bld.mkCvt(OP_CVT, TYPE_F32, face, TYPE_S32, face);
2334 }
2335 }
2336 break;
2337 case SV_TESS_COORD:
2338 assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL);
2339 readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index);
2340 break;
2341 case SV_NTID:
2342 case SV_NCTAID:
2343 case SV_GRIDID:
2344 assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise
2345 if (sym->reg.data.sv.index == 3) {
2346 i->op = OP_MOV;
2347 i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1));
2348 return true;
2349 }
2350 addr += prog->driver->prop.cp.gridInfoBase;
2351 bld.mkLoad(TYPE_U32, i->getDef(0),
2352 bld.mkSymbol(FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2353 TYPE_U32, addr), NULL);
2354 break;
2355 case SV_SAMPLE_INDEX:
2356 // TODO: Properly pass source as an address in the PIX address space
2357 // (which can be of the form [r0+offset]). But this is currently
2358 // unnecessary.
2359 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2360 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2361 break;
2362 case SV_SAMPLE_POS: {
2363 Value *off = new_LValue(func, FILE_GPR);
2364 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2365 ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2366 bld.mkOp2(OP_SHL, TYPE_U32, off, i->getDef(0), bld.mkImm(3));
2367 bld.mkLoad(TYPE_F32,
2368 i->getDef(0),
2369 bld.mkSymbol(
2370 FILE_MEMORY_CONST, prog->driver->io.auxCBSlot,
2371 TYPE_U32, prog->driver->io.sampleInfoBase +
2372 4 * sym->reg.data.sv.index),
2373 off);
2374 break;
2375 }
2376 case SV_SAMPLE_MASK: {
2377 ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0));
2378 ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK;
2379 Instruction *sampleid =
2380 bld.mkOp1(OP_PIXLD, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2381 sampleid->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID;
2382 Value *masked =
2383 bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ld->getDef(0),
2384 bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(),
2385 bld.loadImm(NULL, 1), sampleid->getDef(0)));
2386 if (prog->driver->prop.fp.persampleInvocation) {
2387 bld.mkMov(i->getDef(0), masked);
2388 } else {
2389 bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), ld->getDef(0), masked,
2390 bld.mkImm(0))
2391 ->subOp = 1;
2392 }
2393 break;
2394 }
2395 case SV_BASEVERTEX:
2396 case SV_BASEINSTANCE:
2397 case SV_DRAWID:
2398 ld = bld.mkLoad(TYPE_U32, i->getDef(0),
2399 bld.mkSymbol(FILE_MEMORY_CONST,
2400 prog->driver->io.auxCBSlot,
2401 TYPE_U32,
2402 prog->driver->io.drawInfoBase +
2403 4 * (sv - SV_BASEVERTEX)),
2404 NULL);
2405 break;
2406 default:
2407 if (prog->getType() == Program::TYPE_TESSELLATION_EVAL && !i->perPatch)
2408 vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0));
2409 ld = bld.mkFetch(i->getDef(0), i->dType,
2410 FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx);
2411 ld->perPatch = i->perPatch;
2412 break;
2413 }
2414 bld.getBB()->remove(i);
2415 return true;
2416 }
2417
2418 bool
2419 NVC0LoweringPass::handleDIV(Instruction *i)
2420 {
2421 if (!isFloatType(i->dType))
2422 return true;
2423 bld.setPosition(i, false);
2424 Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(typeSizeof(i->dType)), i->getSrc(1));
2425 i->op = OP_MUL;
2426 i->setSrc(1, rcp->getDef(0));
2427 return true;
2428 }
2429
2430 bool
2431 NVC0LoweringPass::handleMOD(Instruction *i)
2432 {
2433 if (!isFloatType(i->dType))
2434 return true;
2435 LValue *value = bld.getScratch(typeSizeof(i->dType));
2436 bld.mkOp1(OP_RCP, i->dType, value, i->getSrc(1));
2437 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(0), value);
2438 bld.mkOp1(OP_TRUNC, i->dType, value, value);
2439 bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(1), value);
2440 i->op = OP_SUB;
2441 i->setSrc(1, value);
2442 return true;
2443 }
2444
2445 bool
2446 NVC0LoweringPass::handleSQRT(Instruction *i)
2447 {
2448 if (i->dType == TYPE_F64) {
2449 Value *pred = bld.getSSA(1, FILE_PREDICATE);
2450 Value *zero = bld.loadImm(NULL, 0.0);
2451 Value *dst = bld.getSSA(8);
2452 bld.mkOp1(OP_RSQ, i->dType, dst, i->getSrc(0));
2453 bld.mkCmp(OP_SET, CC_LE, i->dType, pred, i->dType, i->getSrc(0), zero);
2454 bld.mkOp3(OP_SELP, TYPE_U64, dst, zero, dst, pred);
2455 i->op = OP_MUL;
2456 i->setSrc(1, dst);
2457 // TODO: Handle this properly with a library function
2458 } else {
2459 bld.setPosition(i, true);
2460 i->op = OP_RSQ;
2461 bld.mkOp1(OP_RCP, i->dType, i->getDef(0), i->getDef(0));
2462 }
2463
2464 return true;
2465 }
2466
2467 bool
2468 NVC0LoweringPass::handlePOW(Instruction *i)
2469 {
2470 LValue *val = bld.getScratch();
2471
2472 bld.mkOp1(OP_LG2, TYPE_F32, val, i->getSrc(0));
2473 bld.mkOp2(OP_MUL, TYPE_F32, val, i->getSrc(1), val)->dnz = 1;
2474 bld.mkOp1(OP_PREEX2, TYPE_F32, val, val);
2475
2476 i->op = OP_EX2;
2477 i->setSrc(0, val);
2478 i->setSrc(1, NULL);
2479
2480 return true;
2481 }
2482
2483 bool
2484 NVC0LoweringPass::handleEXPORT(Instruction *i)
2485 {
2486 if (prog->getType() == Program::TYPE_FRAGMENT) {
2487 int id = i->getSrc(0)->reg.data.offset / 4;
2488
2489 if (i->src(0).isIndirect(0)) // TODO, ugly
2490 return false;
2491 i->op = OP_MOV;
2492 i->subOp = NV50_IR_SUBOP_MOV_FINAL;
2493 i->src(0).set(i->src(1));
2494 i->setSrc(1, NULL);
2495 i->setDef(0, new_LValue(func, FILE_GPR));
2496 i->getDef(0)->reg.data.id = id;
2497
2498 prog->maxGPR = MAX2(prog->maxGPR, id);
2499 } else
2500 if (prog->getType() == Program::TYPE_GEOMETRY) {
2501 i->setIndirect(0, 1, gpEmitAddress);
2502 }
2503 return true;
2504 }
2505
2506 bool
2507 NVC0LoweringPass::handleOUT(Instruction *i)
2508 {
2509 Instruction *prev = i->prev;
2510 ImmediateValue stream, prevStream;
2511
2512 // Only merge if the stream ids match. Also, note that the previous
2513 // instruction would have already been lowered, so we take arg1 from it.
2514 if (i->op == OP_RESTART && prev && prev->op == OP_EMIT &&
2515 i->src(0).getImmediate(stream) &&
2516 prev->src(1).getImmediate(prevStream) &&
2517 stream.reg.data.u32 == prevStream.reg.data.u32) {
2518 i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART;
2519 delete_Instruction(prog, i);
2520 } else {
2521 assert(gpEmitAddress);
2522 i->setDef(0, gpEmitAddress);
2523 i->setSrc(1, i->getSrc(0));
2524 i->setSrc(0, gpEmitAddress);
2525 }
2526 return true;
2527 }
2528
2529 // Generate a binary predicate if an instruction is predicated by
2530 // e.g. an f32 value.
2531 void
2532 NVC0LoweringPass::checkPredicate(Instruction *insn)
2533 {
2534 Value *pred = insn->getPredicate();
2535 Value *pdst;
2536
2537 if (!pred || pred->reg.file == FILE_PREDICATE)
2538 return;
2539 pdst = new_LValue(func, FILE_PREDICATE);
2540
2541 // CAUTION: don't use pdst->getInsn, the definition might not be unique,
2542 // delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass
2543
2544 bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred);
2545
2546 insn->setPredicate(insn->cc, pdst);
2547 }
2548
2549 //
2550 // - add quadop dance for texturing
2551 // - put FP outputs in GPRs
2552 // - convert instruction sequences
2553 //
2554 bool
2555 NVC0LoweringPass::visit(Instruction *i)
2556 {
2557 bool ret = true;
2558 bld.setPosition(i, false);
2559
2560 if (i->cc != CC_ALWAYS)
2561 checkPredicate(i);
2562
2563 switch (i->op) {
2564 case OP_TEX:
2565 case OP_TXB:
2566 case OP_TXL:
2567 case OP_TXF:
2568 case OP_TXG:
2569 return handleTEX(i->asTex());
2570 case OP_TXD:
2571 return handleTXD(i->asTex());
2572 case OP_TXLQ:
2573 return handleTXLQ(i->asTex());
2574 case OP_TXQ:
2575 return handleTXQ(i->asTex());
2576 case OP_EX2:
2577 bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0));
2578 i->setSrc(0, i->getDef(0));
2579 break;
2580 case OP_POW:
2581 return handlePOW(i);
2582 case OP_DIV:
2583 return handleDIV(i);
2584 case OP_MOD:
2585 return handleMOD(i);
2586 case OP_SQRT:
2587 return handleSQRT(i);
2588 case OP_EXPORT:
2589 ret = handleEXPORT(i);
2590 break;
2591 case OP_EMIT:
2592 case OP_RESTART:
2593 return handleOUT(i);
2594 case OP_RDSV:
2595 return handleRDSV(i);
2596 case OP_WRSV:
2597 return handleWRSV(i);
2598 case OP_STORE:
2599 case OP_LOAD:
2600 handleLDST(i);
2601 break;
2602 case OP_ATOM:
2603 {
2604 const bool cctl = i->src(0).getFile() == FILE_MEMORY_BUFFER;
2605 handleATOM(i);
2606 handleCasExch(i, cctl);
2607 }
2608 break;
2609 case OP_SULDB:
2610 case OP_SULDP:
2611 case OP_SUSTB:
2612 case OP_SUSTP:
2613 case OP_SUREDB:
2614 case OP_SUREDP:
2615 if (targ->getChipset() >= NVISA_GK104_CHIPSET)
2616 handleSurfaceOpNVE4(i->asTex());
2617 else
2618 handleSurfaceOpNVC0(i->asTex());
2619 break;
2620 case OP_SUQ:
2621 handleSUQ(i->asTex());
2622 break;
2623 case OP_BUFQ:
2624 handleBUFQ(i);
2625 break;
2626 default:
2627 break;
2628 }
2629
2630 /* Kepler+ has a special opcode to compute a new base address to be used
2631 * for indirect loads.
2632 */
2633 if (targ->getChipset() >= NVISA_GK104_CHIPSET && !i->perPatch &&
2634 (i->op == OP_VFETCH || i->op == OP_EXPORT) && i->src(0).isIndirect(0)) {
2635 Instruction *afetch = bld.mkOp1(OP_AFETCH, TYPE_U32, bld.getSSA(),
2636 cloneShallow(func, i->getSrc(0)));
2637 afetch->setIndirect(0, 0, i->getIndirect(0, 0));
2638 i->src(0).get()->reg.data.offset = 0;
2639 i->setIndirect(0, 0, afetch->getDef(0));
2640 }
2641
2642 return ret;
2643 }
2644
2645 bool
2646 TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const
2647 {
2648 if (stage == CG_STAGE_PRE_SSA) {
2649 NVC0LoweringPass pass(prog);
2650 return pass.run(prog, false, true);
2651 } else
2652 if (stage == CG_STAGE_POST_RA) {
2653 NVC0LegalizePostRA pass(prog);
2654 return pass.run(prog, false, true);
2655 } else
2656 if (stage == CG_STAGE_SSA) {
2657 NVC0LegalizeSSA pass;
2658 return pass.run(prog, false, true);
2659 }
2660 return false;
2661 }
2662
2663 } // namespace nv50_ir