projects / mesa.git / blob
? search:


d7c6b808fbf8a0fb4763bb7cdf9c3e89f25452cf
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_emit_gk110.cpp
1 /*
2 * Copyright 2012 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "codegen/nv50_ir_target_nvc0.h"
24
25 // CodeEmitter for GK110 encoding of the Fermi/Kepler ISA.
26
27 namespace nv50_ir {
28
29 class CodeEmitterGK110 : public CodeEmitter
30 {
31 public:
32 CodeEmitterGK110(const TargetNVC0 *);
33
34 virtual bool emitInstruction(Instruction *);
35 virtual uint32_t getMinEncodingSize(const Instruction *) const;
36 virtual void prepareEmission(Function *);
37
38 inline void setProgramType(Program::Type pType) { progType = pType; }
39
40 private:
41 const TargetNVC0 *targNVC0;
42
43 Program::Type progType;
44
45 const bool writeIssueDelays;
46
47 private:
48 void emitForm_21(const Instruction *, uint32_t opc2, uint32_t opc1);
49 void emitForm_C(const Instruction *, uint32_t opc, uint8_t ctg);
50 void emitForm_L(const Instruction *, uint32_t opc, uint8_t ctg, Modifier);
51
52 void emitPredicate(const Instruction *);
53
54 void setCAddress14(const ValueRef&);
55 void setShortImmediate(const Instruction *, const int s);
56 void setImmediate32(const Instruction *, const int s, Modifier);
57
58 void modNegAbsF32_3b(const Instruction *, const int s);
59
60 void emitCondCode(CondCode cc, int pos, uint8_t mask);
61 void emitInterpMode(const Instruction *);
62 void emitLoadStoreType(DataType ty, const int pos);
63 void emitCachingMode(CacheMode c, const int pos);
64
65 inline uint8_t getSRegEncoding(const ValueRef&);
66
67 void emitRoundMode(RoundMode, const int pos, const int rintPos);
68 void emitRoundModeF(RoundMode, const int pos);
69 void emitRoundModeI(RoundMode, const int pos);
70
71 void emitNegAbs12(const Instruction *);
72
73 void emitNOP(const Instruction *);
74
75 void emitLOAD(const Instruction *);
76 void emitSTORE(const Instruction *);
77 void emitMOV(const Instruction *);
78
79 void emitINTERP(const Instruction *);
80 void emitPFETCH(const Instruction *);
81 void emitVFETCH(const Instruction *);
82 void emitEXPORT(const Instruction *);
83 void emitOUT(const Instruction *);
84
85 void emitUADD(const Instruction *);
86 void emitFADD(const Instruction *);
87 void emitDADD(const Instruction *);
88 void emitIMUL(const Instruction *);
89 void emitFMUL(const Instruction *);
90 void emitDMUL(const Instruction *);
91 void emitIMAD(const Instruction *);
92 void emitISAD(const Instruction *);
93 void emitFMAD(const Instruction *);
94 void emitDMAD(const Instruction *);
95
96 void emitNOT(const Instruction *);
97 void emitLogicOp(const Instruction *, uint8_t subOp);
98 void emitPOPC(const Instruction *);
99 void emitINSBF(const Instruction *);
100 void emitEXTBF(const Instruction *);
101 void emitBFIND(const Instruction *);
102 void emitShift(const Instruction *);
103
104 void emitSFnOp(const Instruction *, uint8_t subOp);
105
106 void emitCVT(const Instruction *);
107 void emitMINMAX(const Instruction *);
108 void emitPreOp(const Instruction *);
109
110 void emitSET(const CmpInstruction *);
111 void emitSLCT(const CmpInstruction *);
112 void emitSELP(const Instruction *);
113
114 void emitTEXBAR(const Instruction *);
115 void emitTEX(const TexInstruction *);
116 void emitTEXCSAA(const TexInstruction *);
117 void emitTXQ(const TexInstruction *);
118
119 void emitQUADOP(const Instruction *, uint8_t qOp, uint8_t laneMask);
120
121 void emitPIXLD(const Instruction *);
122
123 void emitFlow(const Instruction *);
124
125 inline void defId(const ValueDef&, const int pos);
126 inline void srcId(const ValueRef&, const int pos);
127 inline void srcId(const ValueRef *, const int pos);
128 inline void srcId(const Instruction *, int s, const int pos);
129
130 inline void srcAddr32(const ValueRef&, const int pos); // address / 4
131
132 inline bool isLIMM(const ValueRef&, DataType ty, bool mod = false);
133 };
134
135 #define GK110_GPR_ZERO 255
136
137 #define NEG_(b, s) \
138 if (i->src(s).mod.neg()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
139 #define ABS_(b, s) \
140 if (i->src(s).mod.abs()) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
141
142 #define NOT_(b, s) if (i->src(s).mod & Modifier(NV50_IR_MOD_NOT)) \
143 code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
144
145 #define FTZ_(b) if (i->ftz) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
146 #define DNZ_(b) if (i->dnz) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
147
148 #define SAT_(b) if (i->saturate) code[(0x##b) / 32] |= 1 << ((0x##b) % 32)
149
150 #define RND_(b, t) emitRoundMode##t(i->rnd, 0x##b)
151
152 #define SDATA(a) ((a).rep()->reg.data)
153 #define DDATA(a) ((a).rep()->reg.data)
154
155 void CodeEmitterGK110::srcId(const ValueRef& src, const int pos)
156 {
157 code[pos / 32] |= (src.get() ? SDATA(src).id : GK110_GPR_ZERO) << (pos % 32);
158 }
159
160 void CodeEmitterGK110::srcId(const ValueRef *src, const int pos)
161 {
162 code[pos / 32] |= (src ? SDATA(*src).id : GK110_GPR_ZERO) << (pos % 32);
163 }
164
165 void CodeEmitterGK110::srcId(const Instruction *insn, int s, int pos)
166 {
167 int r = insn->srcExists(s) ? SDATA(insn->src(s)).id : GK110_GPR_ZERO;
168 code[pos / 32] |= r << (pos % 32);
169 }
170
171 void CodeEmitterGK110::srcAddr32(const ValueRef& src, const int pos)
172 {
173 code[pos / 32] |= (SDATA(src).offset >> 2) << (pos % 32);
174 }
175
176 void CodeEmitterGK110::defId(const ValueDef& def, const int pos)
177 {
178 code[pos / 32] |= (def.get() ? DDATA(def).id : GK110_GPR_ZERO) << (pos % 32);
179 }
180
181 bool CodeEmitterGK110::isLIMM(const ValueRef& ref, DataType ty, bool mod)
182 {
183 const ImmediateValue *imm = ref.get()->asImm();
184
185 return imm && (imm->reg.data.u32 & ((ty == TYPE_F32) ? 0xfff : 0xfff00000));
186 }
187
188 void
189 CodeEmitterGK110::emitRoundMode(RoundMode rnd, const int pos, const int rintPos)
190 {
191 bool rint = false;
192 uint8_t n;
193
194 switch (rnd) {
195 case ROUND_MI: rint = true; /* fall through */ case ROUND_M: n = 1; break;
196 case ROUND_PI: rint = true; /* fall through */ case ROUND_P: n = 2; break;
197 case ROUND_ZI: rint = true; /* fall through */ case ROUND_Z: n = 3; break;
198 default:
199 rint = rnd == ROUND_NI;
200 n = 0;
201 assert(rnd == ROUND_N || rnd == ROUND_NI);
202 break;
203 }
204 code[pos / 32] |= n << (pos % 32);
205 if (rint && rintPos >= 0)
206 code[rintPos / 32] |= 1 << (rintPos % 32);
207 }
208
209 void
210 CodeEmitterGK110::emitRoundModeF(RoundMode rnd, const int pos)
211 {
212 uint8_t n;
213
214 switch (rnd) {
215 case ROUND_M: n = 1; break;
216 case ROUND_P: n = 2; break;
217 case ROUND_Z: n = 3; break;
218 default:
219 n = 0;
220 assert(rnd == ROUND_N);
221 break;
222 }
223 code[pos / 32] |= n << (pos % 32);
224 }
225
226 void
227 CodeEmitterGK110::emitRoundModeI(RoundMode rnd, const int pos)
228 {
229 uint8_t n;
230
231 switch (rnd) {
232 case ROUND_MI: n = 1; break;
233 case ROUND_PI: n = 2; break;
234 case ROUND_ZI: n = 3; break;
235 default:
236 n = 0;
237 assert(rnd == ROUND_NI);
238 break;
239 }
240 code[pos / 32] |= n << (pos % 32);
241 }
242
243 void CodeEmitterGK110::emitCondCode(CondCode cc, int pos, uint8_t mask)
244 {
245 uint8_t n;
246
247 switch (cc) {
248 case CC_FL: n = 0x00; break;
249 case CC_LT: n = 0x01; break;
250 case CC_EQ: n = 0x02; break;
251 case CC_LE: n = 0x03; break;
252 case CC_GT: n = 0x04; break;
253 case CC_NE: n = 0x05; break;
254 case CC_GE: n = 0x06; break;
255 case CC_LTU: n = 0x09; break;
256 case CC_EQU: n = 0x0a; break;
257 case CC_LEU: n = 0x0b; break;
258 case CC_GTU: n = 0x0c; break;
259 case CC_NEU: n = 0x0d; break;
260 case CC_GEU: n = 0x0e; break;
261 case CC_TR: n = 0x0f; break;
262 case CC_NO: n = 0x10; break;
263 case CC_NC: n = 0x11; break;
264 case CC_NS: n = 0x12; break;
265 case CC_NA: n = 0x13; break;
266 case CC_A: n = 0x14; break;
267 case CC_S: n = 0x15; break;
268 case CC_C: n = 0x16; break;
269 case CC_O: n = 0x17; break;
270 default:
271 n = 0;
272 assert(!"invalid condition code");
273 break;
274 }
275 code[pos / 32] |= (n & mask) << (pos % 32);
276 }
277
278 void
279 CodeEmitterGK110::emitPredicate(const Instruction *i)
280 {
281 if (i->predSrc >= 0) {
282 srcId(i->src(i->predSrc), 18);
283 if (i->cc == CC_NOT_P)
284 code[0] |= 8 << 18; // negate
285 assert(i->getPredicate()->reg.file == FILE_PREDICATE);
286 } else {
287 code[0] |= 7 << 18;
288 }
289 }
290
291 void
292 CodeEmitterGK110::setCAddress14(const ValueRef& src)
293 {
294 const Storage& res = src.get()->asSym()->reg;
295 const int32_t addr = res.data.offset / 4;
296
297 code[0] |= (addr & 0x01ff) << 23;
298 code[1] |= (addr & 0x3e00) >> 9;
299 code[1] |= res.fileIndex << 5;
300 }
301
302 void
303 CodeEmitterGK110::setShortImmediate(const Instruction *i, const int s)
304 {
305 const uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;
306 const uint64_t u64 = i->getSrc(s)->asImm()->reg.data.u64;
307
308 if (i->sType == TYPE_F32) {
309 assert(!(u32 & 0x00000fff));
310 code[0] |= ((u32 & 0x001ff000) >> 12) << 23;
311 code[1] |= ((u32 & 0x7fe00000) >> 21);
312 code[1] |= ((u32 & 0x80000000) >> 4);
313 } else
314 if (i->sType == TYPE_F64) {
315 assert(!(u64 & 0x00000fffffffffffULL));
316 code[0] |= ((u64 & 0x001ff00000000000ULL) >> 44) << 23;
317 code[1] |= ((u64 & 0x7fe0000000000000ULL) >> 53);
318 code[1] |= ((u64 & 0x8000000000000000ULL) >> 36);
319 } else {
320 assert((u32 & 0xfff00000) == 0 || (u32 & 0xfff00000) == 0xfff00000);
321 code[0] |= (u32 & 0x001ff) << 23;
322 code[1] |= (u32 & 0x7fe00) >> 9;
323 code[1] |= (u32 & 0x80000) << 8;
324 }
325 }
326
327 void
328 CodeEmitterGK110::setImmediate32(const Instruction *i, const int s,
329 Modifier mod)
330 {
331 uint32_t u32 = i->getSrc(s)->asImm()->reg.data.u32;
332
333 if (mod) {
334 ImmediateValue imm(i->getSrc(s)->asImm(), i->sType);
335 mod.applyTo(imm);
336 u32 = imm.reg.data.u32;
337 }
338
339 code[0] |= u32 << 23;
340 code[1] |= u32 >> 9;
341 }
342
343 void
344 CodeEmitterGK110::emitForm_L(const Instruction *i, uint32_t opc, uint8_t ctg,
345 Modifier mod)
346 {
347 code[0] = ctg;
348 code[1] = opc << 20;
349
350 emitPredicate(i);
351
352 defId(i->def(0), 2);
353
354 for (int s = 0; s < 3 && i->srcExists(s); ++s) {
355 switch (i->src(s).getFile()) {
356 case FILE_GPR:
357 srcId(i->src(s), s ? 42 : 10);
358 break;
359 case FILE_IMMEDIATE:
360 setImmediate32(i, s, mod);
361 break;
362 default:
363 break;
364 }
365 }
366 }
367
368
369 void
370 CodeEmitterGK110::emitForm_C(const Instruction *i, uint32_t opc, uint8_t ctg)
371 {
372 code[0] = ctg;
373 code[1] = opc << 20;
374
375 emitPredicate(i);
376
377 defId(i->def(0), 2);
378
379 switch (i->src(0).getFile()) {
380 case FILE_MEMORY_CONST:
381 code[1] |= 0x4 << 28;
382 setCAddress14(i->src(0));
383 break;
384 case FILE_GPR:
385 code[1] |= 0xc << 28;
386 srcId(i->src(0), 23);
387 break;
388 default:
389 assert(0);
390 break;
391 }
392 }
393
394 // 0x2 for GPR, c[] and 0x1 for short immediate
395 void
396 CodeEmitterGK110::emitForm_21(const Instruction *i, uint32_t opc2,
397 uint32_t opc1)
398 {
399 const bool imm = i->srcExists(1) && i->src(1).getFile() == FILE_IMMEDIATE;
400
401 int s1 = 23;
402 if (i->srcExists(2) && i->src(2).getFile() == FILE_MEMORY_CONST)
403 s1 = 42;
404
405 if (imm) {
406 code[0] = 0x1;
407 code[1] = opc1 << 20;
408 } else {
409 code[0] = 0x2;
410 code[1] = (0xc << 28) | (opc2 << 20);
411 }
412
413 emitPredicate(i);
414
415 defId(i->def(0), 2);
416
417 for (int s = 0; s < 3 && i->srcExists(s); ++s) {
418 switch (i->src(s).getFile()) {
419 case FILE_MEMORY_CONST:
420 code[1] &= (s == 2) ? ~(0x4 << 28) : ~(0x8 << 28);
421 setCAddress14(i->src(s));
422 break;
423 case FILE_IMMEDIATE:
424 setShortImmediate(i, s);
425 break;
426 case FILE_GPR:
427 srcId(i->src(s), s ? ((s == 2) ? 42 : s1) : 10);
428 break;
429 default:
430 // ignore here, can be predicate or flags, but must not be address
431 break;
432 }
433 }
434 // 0x0 = invalid
435 // 0xc = rrr
436 // 0x8 = rrc
437 // 0x4 = rcr
438 assert(imm || (code[1] & (0xc << 28)));
439 }
440
441 inline void
442 CodeEmitterGK110::modNegAbsF32_3b(const Instruction *i, const int s)
443 {
444 if (i->src(s).mod.abs()) code[1] &= ~(1 << 27);
445 if (i->src(s).mod.neg()) code[1] ^= (1 << 27);
446 }
447
448 void
449 CodeEmitterGK110::emitNOP(const Instruction *i)
450 {
451 code[0] = 0x00003c02;
452 code[1] = 0x85800000;
453
454 if (i)
455 emitPredicate(i);
456 else
457 code[0] = 0x001c3c02;
458 }
459
460 void
461 CodeEmitterGK110::emitFMAD(const Instruction *i)
462 {
463 assert(!isLIMM(i->src(1), TYPE_F32));
464
465 emitForm_21(i, 0x0c0, 0x940);
466
467 NEG_(34, 2);
468 SAT_(35);
469 RND_(36, F);
470 FTZ_(38);
471 DNZ_(39);
472
473 bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
474
475 if (code[0] & 0x1) {
476 if (neg1)
477 code[1] ^= 1 << 27;
478 } else
479 if (neg1) {
480 code[1] |= 1 << 19;
481 }
482 }
483
484 void
485 CodeEmitterGK110::emitDMAD(const Instruction *i)
486 {
487 assert(!i->saturate);
488 assert(!i->ftz);
489
490 emitForm_21(i, 0x1b8, 0xb38);
491
492 NEG_(34, 2);
493 RND_(36, F);
494
495 bool neg1 = (i->src(0).mod ^ i->src(1).mod).neg();
496
497 if (code[0] & 0x1) {
498 if (neg1)
499 code[1] ^= 1 << 27;
500 } else
501 if (neg1) {
502 code[1] |= 1 << 19;
503 }
504 }
505
506 void
507 CodeEmitterGK110::emitFMUL(const Instruction *i)
508 {
509 bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
510
511 assert(i->postFactor >= -3 && i->postFactor <= 3);
512
513 if (isLIMM(i->src(1), TYPE_F32)) {
514 emitForm_L(i, 0x200, 0x2, Modifier(0));
515
516 FTZ_(38);
517 DNZ_(39);
518 SAT_(3a);
519 if (neg)
520 code[1] ^= 1 << 22;
521
522 assert(i->postFactor == 0);
523 } else {
524 emitForm_21(i, 0x234, 0xc34);
525 code[1] |= ((i->postFactor > 0) ?
526 (7 - i->postFactor) : (0 - i->postFactor)) << 12;
527
528 RND_(2a, F);
529 FTZ_(2f);
530 DNZ_(30);
531 SAT_(35);
532
533 if (code[0] & 0x1) {
534 if (neg)
535 code[1] ^= 1 << 27;
536 } else
537 if (neg) {
538 code[1] |= 1 << 19;
539 }
540 }
541 }
542
543 void
544 CodeEmitterGK110::emitDMUL(const Instruction *i)
545 {
546 bool neg = (i->src(0).mod ^ i->src(1).mod).neg();
547
548 assert(!i->postFactor);
549 assert(!i->saturate);
550 assert(!i->ftz);
551 assert(!i->dnz);
552
553 emitForm_21(i, 0x240, 0xc40);
554
555 RND_(2a, F);
556
557 if (code[0] & 0x1) {
558 if (neg)
559 code[1] ^= 1 << 27;
560 } else
561 if (neg) {
562 code[1] |= 1 << 19;
563 }
564 }
565
566 void
567 CodeEmitterGK110::emitIMUL(const Instruction *i)
568 {
569 assert(!i->src(0).mod.neg() && !i->src(1).mod.neg());
570 assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
571
572 if (isLIMM(i->src(1), TYPE_S32)) {
573 emitForm_L(i, 0x280, 2, Modifier(0));
574
575 assert(i->subOp != NV50_IR_SUBOP_MUL_HIGH);
576
577 if (i->sType == TYPE_S32)
578 code[1] |= 3 << 25;
579 } else {
580 emitForm_21(i, 0x21c, 0xc1c);
581
582 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
583 code[1] |= 1 << 10;
584 if (i->sType == TYPE_S32)
585 code[1] |= 3 << 11;
586 }
587 }
588
589 void
590 CodeEmitterGK110::emitFADD(const Instruction *i)
591 {
592 if (isLIMM(i->src(1), TYPE_F32)) {
593 assert(i->rnd == ROUND_N);
594 assert(!i->saturate);
595
596 Modifier mod = i->src(1).mod ^
597 Modifier(i->op == OP_SUB ? NV50_IR_MOD_NEG : 0);
598
599 emitForm_L(i, 0x400, 0, mod);
600
601 FTZ_(3a);
602 NEG_(3b, 0);
603 ABS_(39, 0);
604 } else {
605 emitForm_21(i, 0x22c, 0xc2c);
606
607 FTZ_(2f);
608 RND_(2a, F);
609 ABS_(31, 0);
610 NEG_(33, 0);
611 SAT_(35);
612
613 if (code[0] & 0x1) {
614 modNegAbsF32_3b(i, 1);
615 if (i->op == OP_SUB) code[1] ^= 1 << 27;
616 } else {
617 ABS_(34, 1);
618 NEG_(30, 1);
619 if (i->op == OP_SUB) code[1] ^= 1 << 16;
620 }
621 }
622 }
623
624 void
625 CodeEmitterGK110::emitDADD(const Instruction *i)
626 {
627 assert(!i->saturate);
628 assert(!i->ftz);
629
630 emitForm_21(i, 0x238, 0xc38);
631 RND_(2a, F);
632 ABS_(31, 0);
633 NEG_(33, 0);
634 if (code[0] & 0x1) {
635 modNegAbsF32_3b(i, 1);
636 if (i->op == OP_SUB) code[1] ^= 1 << 27;
637 } else {
638 NEG_(30, 1);
639 ABS_(34, 1);
640 if (i->op == OP_SUB) code[1] ^= 1 << 16;
641 }
642 }
643
644 void
645 CodeEmitterGK110::emitUADD(const Instruction *i)
646 {
647 uint8_t addOp = (i->src(0).mod.neg() << 1) | i->src(1).mod.neg();
648
649 if (i->op == OP_SUB)
650 addOp ^= 1;
651
652 assert(!i->src(0).mod.abs() && !i->src(1).mod.abs());
653
654 if (isLIMM(i->src(1), TYPE_S32)) {
655 emitForm_L(i, 0x400, 1, Modifier((addOp & 1) ? NV50_IR_MOD_NEG : 0));
656
657 if (addOp & 2)
658 code[1] |= 1 << 27;
659
660 assert(!i->defExists(1));
661 assert(i->flagsSrc < 0);
662
663 SAT_(39);
664 } else {
665 emitForm_21(i, 0x208, 0xc08);
666
667 assert(addOp != 3); // would be add-plus-one
668
669 code[1] |= addOp << 19;
670
671 if (i->defExists(1))
672 code[1] |= 1 << 18; // write carry
673 if (i->flagsSrc >= 0)
674 code[1] |= 1 << 14; // add carry
675
676 SAT_(35);
677 }
678 }
679
680 // TODO: shl-add
681 void
682 CodeEmitterGK110::emitIMAD(const Instruction *i)
683 {
684 uint8_t addOp =
685 (i->src(2).mod.neg() << 1) | (i->src(0).mod.neg() ^ i->src(1).mod.neg());
686
687 emitForm_21(i, 0x100, 0xa00);
688
689 assert(addOp != 3);
690 code[1] |= addOp << 26;
691
692 if (i->sType == TYPE_S32)
693 code[1] |= (1 << 19) | (1 << 24);
694
695 if (code[0] & 0x1) {
696 assert(!i->subOp);
697 SAT_(39);
698 } else {
699 if (i->subOp == NV50_IR_SUBOP_MUL_HIGH)
700 code[1] |= 1 << 25;
701 SAT_(35);
702 }
703 }
704
705 void
706 CodeEmitterGK110::emitISAD(const Instruction *i)
707 {
708 assert(i->dType == TYPE_S32 || i->dType == TYPE_U32);
709
710 emitForm_21(i, 0x1f4, 0xb74);
711
712 if (i->dType == TYPE_S32)
713 code[1] |= 1 << 19;
714 }
715
716 void
717 CodeEmitterGK110::emitNOT(const Instruction *i)
718 {
719 code[0] = 0x0003fc02; // logop(mov2) dst, 0, not src
720 code[1] = 0x22003800;
721
722 emitPredicate(i);
723
724 defId(i->def(0), 2);
725
726 switch (i->src(0).getFile()) {
727 case FILE_GPR:
728 code[1] |= 0xc << 28;
729 srcId(i->src(0), 23);
730 break;
731 case FILE_MEMORY_CONST:
732 code[1] |= 0x4 << 28;
733 setCAddress14(i->src(1));
734 break;
735 default:
736 assert(0);
737 break;
738 }
739 }
740
741 void
742 CodeEmitterGK110::emitLogicOp(const Instruction *i, uint8_t subOp)
743 {
744 if (isLIMM(i->src(1), TYPE_S32)) {
745 emitForm_L(i, 0x200, 0, i->src(1).mod);
746 code[1] |= subOp << 24;
747 NOT_(3a, 0);
748 } else {
749 emitForm_21(i, 0x220, 0xc20);
750 code[1] |= subOp << 12;
751 NOT_(2a, 0);
752 NOT_(2b, 1);
753 }
754 }
755
756 void
757 CodeEmitterGK110::emitPOPC(const Instruction *i)
758 {
759 assert(!isLIMM(i->src(1), TYPE_S32, true));
760
761 emitForm_21(i, 0x204, 0xc04);
762
763 NOT_(2a, 0);
764 if (!(code[0] & 0x1))
765 NOT_(2b, 1);
766 }
767
768 void
769 CodeEmitterGK110::emitINSBF(const Instruction *i)
770 {
771 emitForm_21(i, 0x1f8, 0xb78);
772 }
773
774 void
775 CodeEmitterGK110::emitEXTBF(const Instruction *i)
776 {
777 emitForm_21(i, 0x600, 0xc00);
778
779 if (i->dType == TYPE_S32)
780 code[1] |= 0x80000;
781 if (i->subOp == NV50_IR_SUBOP_EXTBF_REV)
782 code[1] |= 0x800;
783 }
784
785 void
786 CodeEmitterGK110::emitBFIND(const Instruction *i)
787 {
788 emitForm_C(i, 0x218, 0x2);
789
790 if (i->dType == TYPE_S32)
791 code[1] |= 0x80000;
792 if (i->src(0).mod == Modifier(NV50_IR_MOD_NOT))
793 code[1] |= 0x800;
794 if (i->subOp == NV50_IR_SUBOP_BFIND_SAMT)
795 code[1] |= 0x1000;
796 }
797
798 void
799 CodeEmitterGK110::emitShift(const Instruction *i)
800 {
801 if (i->op == OP_SHR) {
802 emitForm_21(i, 0x214, 0xc14);
803 if (isSignedType(i->dType))
804 code[1] |= 1 << 19;
805 } else {
806 emitForm_21(i, 0x224, 0xc24);
807 }
808
809 if (i->subOp == NV50_IR_SUBOP_SHIFT_WRAP)
810 code[1] |= 1 << 10;
811 }
812
813 void
814 CodeEmitterGK110::emitPreOp(const Instruction *i)
815 {
816 emitForm_C(i, 0x248, 0x2);
817
818 if (i->op == OP_PREEX2)
819 code[1] |= 1 << 10;
820
821 NEG_(30, 0);
822 ABS_(34, 0);
823 }
824
825 void
826 CodeEmitterGK110::emitSFnOp(const Instruction *i, uint8_t subOp)
827 {
828 code[0] = 0x00000002 | (subOp << 23);
829 code[1] = 0x84000000;
830
831 emitPredicate(i);
832
833 defId(i->def(0), 2);
834 srcId(i->src(0), 10);
835
836 NEG_(33, 0);
837 ABS_(31, 0);
838 SAT_(35);
839 }
840
841 void
842 CodeEmitterGK110::emitMINMAX(const Instruction *i)
843 {
844 uint32_t op2, op1;
845
846 switch (i->dType) {
847 case TYPE_U32:
848 case TYPE_S32:
849 op2 = 0x210;
850 op1 = 0xc10;
851 break;
852 case TYPE_F32:
853 op2 = 0x230;
854 op1 = 0xc30;
855 break;
856 case TYPE_F64:
857 op2 = 0x228;
858 op1 = 0xc28;
859 break;
860 default:
861 assert(0);
862 op2 = 0;
863 op1 = 0;
864 break;
865 }
866 emitForm_21(i, op2, op1);
867
868 if (i->dType == TYPE_S32)
869 code[1] |= 1 << 19;
870 code[1] |= (i->op == OP_MIN) ? 0x1c00 : 0x3c00; // [!]pt
871
872 FTZ_(2f);
873 ABS_(31, 0);
874 NEG_(33, 0);
875 if (code[0] & 0x1) {
876 modNegAbsF32_3b(i, 1);
877 } else {
878 ABS_(34, 1);
879 NEG_(30, 1);
880 }
881 }
882
883 void
884 CodeEmitterGK110::emitCVT(const Instruction *i)
885 {
886 const bool f2f = isFloatType(i->dType) && isFloatType(i->sType);
887 const bool f2i = !isFloatType(i->dType) && isFloatType(i->sType);
888 const bool i2f = isFloatType(i->dType) && !isFloatType(i->sType);
889
890 bool sat = i->saturate;
891 bool abs = i->src(0).mod.abs();
892 bool neg = i->src(0).mod.neg();
893
894 RoundMode rnd = i->rnd;
895
896 switch (i->op) {
897 case OP_CEIL: rnd = f2f ? ROUND_PI : ROUND_P; break;
898 case OP_FLOOR: rnd = f2f ? ROUND_MI : ROUND_M; break;
899 case OP_TRUNC: rnd = f2f ? ROUND_ZI : ROUND_Z; break;
900 case OP_SAT: sat = true; break;
901 case OP_NEG: neg = !neg; break;
902 case OP_ABS: abs = true; neg = false; break;
903 default:
904 break;
905 }
906
907 DataType dType;
908
909 if (i->op == OP_NEG && i->dType == TYPE_U32)
910 dType = TYPE_S32;
911 else
912 dType = i->dType;
913
914
915 uint32_t op;
916
917 if (f2f) op = 0x254;
918 else if (f2i) op = 0x258;
919 else if (i2f) op = 0x25c;
920 else op = 0x260;
921
922 emitForm_C(i, op, 0x2);
923
924 FTZ_(2f);
925 if (neg) code[1] |= 1 << 16;
926 if (abs) code[1] |= 1 << 20;
927 if (sat) code[1] |= 1 << 21;
928
929 emitRoundMode(rnd, 32 + 10, f2f ? (32 + 13) : -1);
930
931 code[0] |= typeSizeofLog2(dType) << 10;
932 code[0] |= typeSizeofLog2(i->sType) << 12;
933
934 if (isSignedIntType(dType))
935 code[0] |= 0x4000;
936 if (isSignedIntType(i->sType))
937 code[0] |= 0x8000;
938 }
939
940 void
941 CodeEmitterGK110::emitSET(const CmpInstruction *i)
942 {
943 uint16_t op1, op2;
944
945 if (i->def(0).getFile() == FILE_PREDICATE) {
946 switch (i->sType) {
947 case TYPE_F32: op2 = 0x1d8; op1 = 0xb58; break;
948 case TYPE_F64: op2 = 0x1c0; op1 = 0xb40; break;
949 default:
950 op2 = 0x1b0;
951 op1 = 0xb30;
952 break;
953 }
954 emitForm_21(i, op2, op1);
955
956 NEG_(2e, 0);
957 ABS_(9, 0);
958 if (!(code[0] & 0x1)) {
959 NEG_(8, 1);
960 ABS_(2f, 1);
961 } else {
962 modNegAbsF32_3b(i, 1);
963 }
964 FTZ_(32);
965
966 // normal DST field is negated predicate result
967 code[0] = (code[0] & ~0xfc) | ((code[0] << 3) & 0xe0);
968 if (i->defExists(1))
969 defId(i->def(1), 2);
970 else
971 code[0] |= 0x1c;
972 } else {
973 switch (i->sType) {
974 case TYPE_F32: op2 = 0x000; op1 = 0x800; break;
975 case TYPE_F64: op2 = 0x080; op1 = 0x900; break;
976 default:
977 op2 = 0x1a8;
978 op1 = 0xb28;
979 break;
980 }
981 emitForm_21(i, op2, op1);
982
983 NEG_(2e, 0);
984 ABS_(39, 0);
985 if (!(code[0] & 0x1)) {
986 NEG_(38, 1);
987 ABS_(2f, 1);
988 } else {
989 modNegAbsF32_3b(i, 1);
990 }
991 FTZ_(3a);
992
993 if (i->dType == TYPE_F32)
994 code[1] |= 1 << 23;
995 }
996 if (i->sType == TYPE_S32)
997 code[1] |= 1 << 19;
998
999 if (i->op != OP_SET) {
1000 switch (i->op) {
1001 case OP_SET_AND: code[1] |= 0x0 << 16; break;
1002 case OP_SET_OR: code[1] |= 0x1 << 16; break;
1003 case OP_SET_XOR: code[1] |= 0x2 << 16; break;
1004 default:
1005 assert(0);
1006 break;
1007 }
1008 srcId(i->src(2), 0x2a);
1009 } else {
1010 code[1] |= 0x7 << 10;
1011 }
1012 emitCondCode(i->setCond,
1013 isFloatType(i->sType) ? 0x33 : 0x34,
1014 isFloatType(i->sType) ? 0xf : 0x7);
1015 }
1016
1017 void
1018 CodeEmitterGK110::emitSLCT(const CmpInstruction *i)
1019 {
1020 CondCode cc = i->setCond;
1021 if (i->src(2).mod.neg())
1022 cc = reverseCondCode(cc);
1023
1024 if (i->dType == TYPE_F32) {
1025 emitForm_21(i, 0x1d0, 0xb50);
1026 FTZ_(32);
1027 emitCondCode(cc, 0x33, 0xf);
1028 } else {
1029 emitForm_21(i, 0x1a0, 0xb20);
1030 emitCondCode(cc, 0x34, 0x7);
1031 }
1032 }
1033
1034 void CodeEmitterGK110::emitSELP(const Instruction *i)
1035 {
1036 emitForm_21(i, 0x250, 0x050);
1037
1038 if ((i->cc == CC_NOT_P) ^ (bool)(i->src(2).mod & Modifier(NV50_IR_MOD_NOT)))
1039 code[1] |= 1 << 13;
1040 }
1041
1042 void CodeEmitterGK110::emitTEXBAR(const Instruction *i)
1043 {
1044 code[0] = 0x0000003e | (i->subOp << 23);
1045 code[1] = 0x77000000;
1046
1047 emitPredicate(i);
1048 }
1049
1050 void CodeEmitterGK110::emitTEXCSAA(const TexInstruction *i)
1051 {
1052 code[0] = 0x00000002;
1053 code[1] = 0x76c00000;
1054
1055 code[1] |= i->tex.r << 9;
1056 // code[1] |= i->tex.s << (9 + 8);
1057
1058 if (i->tex.liveOnly)
1059 code[0] |= 0x80000000;
1060
1061 defId(i->def(0), 2);
1062 srcId(i->src(0), 10);
1063 }
1064
1065 static inline bool
1066 isNextIndependentTex(const TexInstruction *i)
1067 {
1068 if (!i->next || !isTextureOp(i->next->op))
1069 return false;
1070 if (i->getDef(0)->interfers(i->next->getSrc(0)))
1071 return false;
1072 return !i->next->srcExists(1) || !i->getDef(0)->interfers(i->next->getSrc(1));
1073 }
1074
1075 void
1076 CodeEmitterGK110::emitTEX(const TexInstruction *i)
1077 {
1078 const bool ind = i->tex.rIndirectSrc >= 0;
1079
1080 if (ind) {
1081 code[0] = 0x00000002;
1082 switch (i->op) {
1083 case OP_TXD:
1084 code[1] = 0x7e000000;
1085 break;
1086 case OP_TXLQ:
1087 code[1] = 0x7e800000;
1088 break;
1089 case OP_TXF:
1090 code[1] = 0x78000000;
1091 break;
1092 case OP_TXG:
1093 code[1] = 0x7dc00000;
1094 break;
1095 default:
1096 code[1] = 0x7d800000;
1097 break;
1098 }
1099 } else {
1100 switch (i->op) {
1101 case OP_TXD:
1102 code[0] = 0x00000002;
1103 code[1] = 0x76000000;
1104 code[1] |= i->tex.r << 9;
1105 break;
1106 case OP_TXLQ:
1107 code[0] = 0x00000002;
1108 code[1] = 0x76800000;
1109 code[1] |= i->tex.r << 9;
1110 break;
1111 case OP_TXF:
1112 code[0] = 0x00000002;
1113 code[1] = 0x70000000;
1114 code[1] |= i->tex.r << 13;
1115 break;
1116 case OP_TXG:
1117 code[0] = 0x00000001;
1118 code[1] = 0x70000000;
1119 code[1] |= i->tex.r << 15;
1120 break;
1121 default:
1122 code[0] = 0x00000001;
1123 code[1] = 0x60000000;
1124 code[1] |= i->tex.r << 15;
1125 break;
1126 }
1127 }
1128
1129 code[1] |= isNextIndependentTex(i) ? 0x1 : 0x2; // t : p mode
1130
1131 if (i->tex.liveOnly)
1132 code[0] |= 0x80000000;
1133
1134 switch (i->op) {
1135 case OP_TEX: break;
1136 case OP_TXB: code[1] |= 0x2000; break;
1137 case OP_TXL: code[1] |= 0x3000; break;
1138 case OP_TXF: break;
1139 case OP_TXG: break;
1140 case OP_TXD: break;
1141 case OP_TXLQ: break;
1142 default:
1143 assert(!"invalid texture op");
1144 break;
1145 }
1146
1147 if (i->op == OP_TXF) {
1148 if (!i->tex.levelZero)
1149 code[1] |= 0x1000;
1150 } else
1151 if (i->tex.levelZero) {
1152 code[1] |= 0x1000;
1153 }
1154
1155 if (i->op != OP_TXD && i->tex.derivAll)
1156 code[1] |= 0x200;
1157
1158 emitPredicate(i);
1159
1160 code[1] |= i->tex.mask << 2;
1161
1162 const int src1 = (i->predSrc == 1) ? 2 : 1; // if predSrc == 1, !srcExists(2)
1163
1164 defId(i->def(0), 2);
1165 srcId(i->src(0), 10);
1166 srcId(i, src1, 23);
1167
1168 if (i->op == OP_TXG) code[1] |= i->tex.gatherComp << 13;
1169
1170 // texture target:
1171 code[1] |= (i->tex.target.isCube() ? 3 : (i->tex.target.getDim() - 1)) << 7;
1172 if (i->tex.target.isArray())
1173 code[1] |= 0x40;
1174 if (i->tex.target.isShadow())
1175 code[1] |= 0x400;
1176 if (i->tex.target == TEX_TARGET_2D_MS ||
1177 i->tex.target == TEX_TARGET_2D_MS_ARRAY)
1178 code[1] |= 0x800;
1179
1180 if (i->srcExists(src1) && i->src(src1).getFile() == FILE_IMMEDIATE) {
1181 // ?
1182 }
1183
1184 if (i->tex.useOffsets == 1) {
1185 switch (i->op) {
1186 case OP_TXF: code[1] |= 0x200; break;
1187 case OP_TXD: code[1] |= 0x00400000; break;
1188 default: code[1] |= 0x800; break;
1189 }
1190 }
1191 if (i->tex.useOffsets == 4)
1192 code[1] |= 0x1000;
1193 }
1194
1195 void
1196 CodeEmitterGK110::emitTXQ(const TexInstruction *i)
1197 {
1198 code[0] = 0x00000002;
1199 code[1] = 0x75400001;
1200
1201 switch (i->tex.query) {
1202 case TXQ_DIMS: code[0] |= 0x01 << 25; break;
1203 case TXQ_TYPE: code[0] |= 0x02 << 25; break;
1204 case TXQ_SAMPLE_POSITION: code[0] |= 0x05 << 25; break;
1205 case TXQ_FILTER: code[0] |= 0x10 << 25; break;
1206 case TXQ_LOD: code[0] |= 0x12 << 25; break;
1207 case TXQ_BORDER_COLOUR: code[0] |= 0x16 << 25; break;
1208 default:
1209 assert(!"invalid texture query");
1210 break;
1211 }
1212
1213 code[1] |= i->tex.mask << 2;
1214 code[1] |= i->tex.r << 9;
1215 if (/*i->tex.sIndirectSrc >= 0 || */i->tex.rIndirectSrc >= 0)
1216 code[1] |= 0x08000000;
1217
1218 defId(i->def(0), 2);
1219 srcId(i->src(0), 10);
1220
1221 emitPredicate(i);
1222 }
1223
1224 void
1225 CodeEmitterGK110::emitQUADOP(const Instruction *i, uint8_t qOp, uint8_t laneMask)
1226 {
1227 code[0] = 0x00000002 | ((qOp & 1) << 31);
1228 code[1] = 0x7fc00000 | (qOp >> 1) | (laneMask << 12);
1229
1230 defId(i->def(0), 2);
1231 srcId(i->src(0), 10);
1232 srcId(i->srcExists(1) ? i->src(1) : i->src(0), 23);
1233
1234 if (i->op == OP_QUADOP && progType != Program::TYPE_FRAGMENT)
1235 code[1] |= 1 << 9; // dall
1236
1237 emitPredicate(i);
1238 }
1239
1240 void
1241 CodeEmitterGK110::emitPIXLD(const Instruction *i)
1242 {
1243 emitForm_L(i, 0x7f4, 2, Modifier(0));
1244 code[1] |= i->subOp << 2;
1245 code[1] |= 0x00070000;
1246 }
1247
1248 void
1249 CodeEmitterGK110::emitFlow(const Instruction *i)
1250 {
1251 const FlowInstruction *f = i->asFlow();
1252
1253 unsigned mask; // bit 0: predicate, bit 1: target
1254
1255 code[0] = 0x00000000;
1256
1257 switch (i->op) {
1258 case OP_BRA:
1259 code[1] = f->absolute ? 0x10800000 : 0x12000000;
1260 if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
1261 code[0] |= 0x80;
1262 mask = 3;
1263 break;
1264 case OP_CALL:
1265 code[1] = f->absolute ? 0x11000000 : 0x13000000;
1266 if (i->srcExists(0) && i->src(0).getFile() == FILE_MEMORY_CONST)
1267 code[0] |= 0x80;
1268 mask = 2;
1269 break;
1270
1271 case OP_EXIT: code[1] = 0x18000000; mask = 1; break;
1272 case OP_RET: code[1] = 0x19000000; mask = 1; break;
1273 case OP_DISCARD: code[1] = 0x19800000; mask = 1; break;
1274 case OP_BREAK: code[1] = 0x1a000000; mask = 1; break;
1275 case OP_CONT: code[1] = 0x1a800000; mask = 1; break;
1276
1277 case OP_JOINAT: code[1] = 0x14800000; mask = 2; break;
1278 case OP_PREBREAK: code[1] = 0x15000000; mask = 2; break;
1279 case OP_PRECONT: code[1] = 0x15800000; mask = 2; break;
1280 case OP_PRERET: code[1] = 0x13800000; mask = 2; break;
1281
1282 case OP_QUADON: code[1] = 0x1b800000; mask = 0; break;
1283 case OP_QUADPOP: code[1] = 0x1c000000; mask = 0; break;
1284 case OP_BRKPT: code[1] = 0x00000000; mask = 0; break;
1285 default:
1286 assert(!"invalid flow operation");
1287 return;
1288 }
1289
1290 if (mask & 1) {
1291 emitPredicate(i);
1292 if (i->flagsSrc < 0)
1293 code[0] |= 0x3c;
1294 }
1295
1296 if (!f)
1297 return;
1298
1299 if (f->allWarp)
1300 code[0] |= 1 << 9;
1301 if (f->limit)
1302 code[0] |= 1 << 8;
1303
1304 if (f->op == OP_CALL) {
1305 if (f->builtin) {
1306 assert(f->absolute);
1307 uint32_t pcAbs = targNVC0->getBuiltinOffset(f->target.builtin);
1308 addReloc(RelocEntry::TYPE_BUILTIN, 0, pcAbs, 0xff800000, 23);
1309 addReloc(RelocEntry::TYPE_BUILTIN, 1, pcAbs, 0x007fffff, -9);
1310 } else {
1311 assert(!f->absolute);
1312 int32_t pcRel = f->target.fn->binPos - (codeSize + 8);
1313 code[0] |= (pcRel & 0x1ff) << 23;
1314 code[1] |= (pcRel >> 9) & 0x7fff;
1315 }
1316 } else
1317 if (mask & 2) {
1318 int32_t pcRel = f->target.bb->binPos - (codeSize + 8);
1319 // currently we don't want absolute branches
1320 assert(!f->absolute);
1321 code[0] |= (pcRel & 0x1ff) << 23;
1322 code[1] |= (pcRel >> 9) & 0x7fff;
1323 }
1324 }
1325
1326 void
1327 CodeEmitterGK110::emitPFETCH(const Instruction *i)
1328 {
1329 uint32_t prim = i->src(0).get()->reg.data.u32;
1330
1331 code[0] = 0x00000002 | ((prim & 0xff) << 23);
1332 code[1] = 0x7f800000;
1333
1334 emitPredicate(i);
1335
1336 defId(i->def(0), 2);
1337 srcId(i->src(1), 10);
1338 }
1339
1340 void
1341 CodeEmitterGK110::emitVFETCH(const Instruction *i)
1342 {
1343 unsigned int size = typeSizeof(i->dType);
1344 uint32_t offset = i->src(0).get()->reg.data.offset;
1345
1346 code[0] = 0x00000002 | (offset << 23);
1347 code[1] = 0x7ec00000 | (offset >> 9);
1348 code[1] |= (size / 4 - 1) << 18;
1349
1350 if (i->perPatch)
1351 code[1] |= 0x4;
1352 if (i->getSrc(0)->reg.file == FILE_SHADER_OUTPUT)
1353 code[1] |= 0x8; // yes, TCPs can read from *outputs* of other threads
1354
1355 emitPredicate(i);
1356
1357 defId(i->def(0), 2);
1358 srcId(i->src(0).getIndirect(0), 10);
1359 srcId(i->src(0).getIndirect(1), 32 + 10); // vertex address
1360 }
1361
1362 void
1363 CodeEmitterGK110::emitEXPORT(const Instruction *i)
1364 {
1365 unsigned int size = typeSizeof(i->dType);
1366 uint32_t offset = i->src(0).get()->reg.data.offset;
1367
1368 code[0] = 0x00000002 | (offset << 23);
1369 code[1] = 0x7f000000 | (offset >> 9);
1370 code[1] |= (size / 4 - 1) << 18;
1371
1372 if (i->perPatch)
1373 code[1] |= 0x4;
1374
1375 emitPredicate(i);
1376
1377 assert(i->src(1).getFile() == FILE_GPR);
1378
1379 srcId(i->src(0).getIndirect(0), 10);
1380 srcId(i->src(0).getIndirect(1), 32 + 10); // vertex base address
1381 srcId(i->src(1), 2);
1382 }
1383
1384 void
1385 CodeEmitterGK110::emitOUT(const Instruction *i)
1386 {
1387 assert(i->src(0).getFile() == FILE_GPR);
1388
1389 emitForm_21(i, 0x1f0, 0xb70);
1390
1391 if (i->op == OP_EMIT)
1392 code[1] |= 1 << 10;
1393 if (i->op == OP_RESTART || i->subOp == NV50_IR_SUBOP_EMIT_RESTART)
1394 code[1] |= 1 << 11;
1395 }
1396
1397 void
1398 CodeEmitterGK110::emitInterpMode(const Instruction *i)
1399 {
1400 code[1] |= (i->ipa & 0x3) << 21; // TODO: INTERP_SAMPLEID
1401 code[1] |= (i->ipa & 0xc) << (19 - 2);
1402 }
1403
1404 void
1405 CodeEmitterGK110::emitINTERP(const Instruction *i)
1406 {
1407 const uint32_t base = i->getSrc(0)->reg.data.offset;
1408
1409 code[0] = 0x00000002 | (base << 31);
1410 code[1] = 0x74800000 | (base >> 1);
1411
1412 if (i->saturate)
1413 code[1] |= 1 << 18;
1414
1415 if (i->op == OP_PINTERP)
1416 srcId(i->src(1), 23);
1417 else
1418 code[0] |= 0xff << 23;
1419
1420 srcId(i->src(0).getIndirect(0), 10);
1421 emitInterpMode(i);
1422
1423 emitPredicate(i);
1424 defId(i->def(0), 2);
1425
1426 if (i->getSampleMode() == NV50_IR_INTERP_OFFSET)
1427 srcId(i->src(i->op == OP_PINTERP ? 2 : 1), 32 + 10);
1428 else
1429 code[1] |= 0xff << 10;
1430 }
1431
1432 void
1433 CodeEmitterGK110::emitLoadStoreType(DataType ty, const int pos)
1434 {
1435 uint8_t n;
1436
1437 switch (ty) {
1438 case TYPE_U8:
1439 n = 0;
1440 break;
1441 case TYPE_S8:
1442 n = 1;
1443 break;
1444 case TYPE_U16:
1445 n = 2;
1446 break;
1447 case TYPE_S16:
1448 n = 3;
1449 break;
1450 case TYPE_F32:
1451 case TYPE_U32:
1452 case TYPE_S32:
1453 n = 4;
1454 break;
1455 case TYPE_F64:
1456 case TYPE_U64:
1457 case TYPE_S64:
1458 n = 5;
1459 break;
1460 case TYPE_B128:
1461 n = 6;
1462 break;
1463 default:
1464 n = 0;
1465 assert(!"invalid ld/st type");
1466 break;
1467 }
1468 code[pos / 32] |= n << (pos % 32);
1469 }
1470
1471 void
1472 CodeEmitterGK110::emitCachingMode(CacheMode c, const int pos)
1473 {
1474 uint8_t n;
1475
1476 switch (c) {
1477 case CACHE_CA:
1478 // case CACHE_WB:
1479 n = 0;
1480 break;
1481 case CACHE_CG:
1482 n = 1;
1483 break;
1484 case CACHE_CS:
1485 n = 2;
1486 break;
1487 case CACHE_CV:
1488 // case CACHE_WT:
1489 n = 3;
1490 break;
1491 default:
1492 n = 0;
1493 assert(!"invalid caching mode");
1494 break;
1495 }
1496 code[pos / 32] |= n << (pos % 32);
1497 }
1498
1499 void
1500 CodeEmitterGK110::emitSTORE(const Instruction *i)
1501 {
1502 int32_t offset = SDATA(i->src(0)).offset;
1503
1504 switch (i->src(0).getFile()) {
1505 case FILE_MEMORY_GLOBAL: code[1] = 0xe0000000; code[0] = 0x00000000; break;
1506 case FILE_MEMORY_LOCAL: code[1] = 0x7a800000; code[0] = 0x00000002; break;
1507 case FILE_MEMORY_SHARED: code[1] = 0x7ac00000; code[0] = 0x00000002; break;
1508 default:
1509 assert(!"invalid memory file");
1510 break;
1511 }
1512
1513 if (i->src(0).getFile() != FILE_MEMORY_GLOBAL)
1514 offset &= 0xffffff;
1515
1516 if (code[0] & 0x2) {
1517 emitLoadStoreType(i->dType, 0x33);
1518 if (i->src(0).getFile() == FILE_MEMORY_LOCAL)
1519 emitCachingMode(i->cache, 0x2f);
1520 } else {
1521 emitLoadStoreType(i->dType, 0x38);
1522 emitCachingMode(i->cache, 0x3b);
1523 }
1524 code[0] |= offset << 23;
1525 code[1] |= offset >> 9;
1526
1527 emitPredicate(i);
1528
1529 srcId(i->src(1), 2);
1530 srcId(i->src(0).getIndirect(0), 10);
1531 }
1532
1533 void
1534 CodeEmitterGK110::emitLOAD(const Instruction *i)
1535 {
1536 int32_t offset = SDATA(i->src(0)).offset;
1537
1538 switch (i->src(0).getFile()) {
1539 case FILE_MEMORY_GLOBAL: code[1] = 0xc0000000; code[0] = 0x00000000; break;
1540 case FILE_MEMORY_LOCAL: code[1] = 0x7a000000; code[0] = 0x00000002; break;
1541 case FILE_MEMORY_SHARED: code[1] = 0x7ac00000; code[0] = 0x00000002; break;
1542 case FILE_MEMORY_CONST:
1543 if (!i->src(0).isIndirect(0) && typeSizeof(i->dType) == 4) {
1544 emitMOV(i);
1545 return;
1546 }
1547 offset &= 0xffff;
1548 code[0] = 0x00000002;
1549 code[1] = 0x7c800000 | (i->src(0).get()->reg.fileIndex << 7);
1550 code[1] |= i->subOp << 15;
1551 break;
1552 default:
1553 assert(!"invalid memory file");
1554 break;
1555 }
1556
1557 if (code[0] & 0x2) {
1558 offset &= 0xffffff;
1559 emitLoadStoreType(i->dType, 0x33);
1560 if (i->src(0).getFile() == FILE_MEMORY_LOCAL)
1561 emitCachingMode(i->cache, 0x2f);
1562 } else {
1563 emitLoadStoreType(i->dType, 0x38);
1564 emitCachingMode(i->cache, 0x3b);
1565 }
1566 code[0] |= offset << 23;
1567 code[1] |= offset >> 9;
1568
1569 emitPredicate(i);
1570
1571 defId(i->def(0), 2);
1572 srcId(i->src(0).getIndirect(0), 10);
1573 }
1574
1575 uint8_t
1576 CodeEmitterGK110::getSRegEncoding(const ValueRef& ref)
1577 {
1578 switch (SDATA(ref).sv.sv) {
1579 case SV_LANEID: return 0x00;
1580 case SV_PHYSID: return 0x03;
1581 case SV_VERTEX_COUNT: return 0x10;
1582 case SV_INVOCATION_ID: return 0x11;
1583 case SV_YDIR: return 0x12;
1584 case SV_TID: return 0x21 + SDATA(ref).sv.index;
1585 case SV_CTAID: return 0x25 + SDATA(ref).sv.index;
1586 case SV_NTID: return 0x29 + SDATA(ref).sv.index;
1587 case SV_GRIDID: return 0x2c;
1588 case SV_NCTAID: return 0x2d + SDATA(ref).sv.index;
1589 case SV_LBASE: return 0x34;
1590 case SV_SBASE: return 0x30;
1591 case SV_CLOCK: return 0x50 + SDATA(ref).sv.index;
1592 default:
1593 assert(!"no sreg for system value");
1594 return 0;
1595 }
1596 }
1597
1598 void
1599 CodeEmitterGK110::emitMOV(const Instruction *i)
1600 {
1601 if (i->src(0).getFile() == FILE_SYSTEM_VALUE) {
1602 code[0] = 0x00000002 | (getSRegEncoding(i->src(0)) << 23);
1603 code[1] = 0x86400000;
1604 emitPredicate(i);
1605 defId(i->def(0), 2);
1606 } else
1607 if (i->src(0).getFile() == FILE_IMMEDIATE) {
1608 code[0] = 0x00000002 | (i->lanes << 14);
1609 code[1] = 0x74000000;
1610 emitPredicate(i);
1611 defId(i->def(0), 2);
1612 setImmediate32(i, 0, Modifier(0));
1613 } else
1614 if (i->src(0).getFile() == FILE_PREDICATE) {
1615 code[0] = 0x00000002;
1616 code[1] = 0x84401c07;
1617 emitPredicate(i);
1618 defId(i->def(0), 2);
1619 srcId(i->src(0), 14);
1620 } else {
1621 emitForm_C(i, 0x24c, 2);
1622 code[1] |= i->lanes << 10;
1623 }
1624 }
1625
1626 bool
1627 CodeEmitterGK110::emitInstruction(Instruction *insn)
1628 {
1629 const unsigned int size = (writeIssueDelays && !(codeSize & 0x3f)) ? 16 : 8;
1630
1631 if (insn->encSize != 8) {
1632 ERROR("skipping unencodable instruction: ");
1633 insn->print();
1634 return false;
1635 } else
1636 if (codeSize + size > codeSizeLimit) {
1637 ERROR("code emitter output buffer too small\n");
1638 return false;
1639 }
1640
1641 if (writeIssueDelays) {
1642 int id = (codeSize & 0x3f) / 8 - 1;
1643 if (id < 0) {
1644 id += 1;
1645 code[0] = 0x00000000; // cf issue delay "instruction"
1646 code[1] = 0x08000000;
1647 code += 2;
1648 codeSize += 8;
1649 }
1650 uint32_t *data = code - (id * 2 + 2);
1651
1652 switch (id) {
1653 case 0: data[0] |= insn->sched << 2; break;
1654 case 1: data[0] |= insn->sched << 10; break;
1655 case 2: data[0] |= insn->sched << 18; break;
1656 case 3: data[0] |= insn->sched << 26; data[1] |= insn->sched >> 6; break;
1657 case 4: data[1] |= insn->sched << 2; break;
1658 case 5: data[1] |= insn->sched << 10; break;
1659 case 6: data[1] |= insn->sched << 18; break;
1660 default:
1661 assert(0);
1662 break;
1663 }
1664 }
1665
1666 // assert that instructions with multiple defs don't corrupt registers
1667 for (int d = 0; insn->defExists(d); ++d)
1668 assert(insn->asTex() || insn->def(d).rep()->reg.data.id >= 0);
1669
1670 switch (insn->op) {
1671 case OP_MOV:
1672 case OP_RDSV:
1673 emitMOV(insn);
1674 break;
1675 case OP_NOP:
1676 break;
1677 case OP_LOAD:
1678 emitLOAD(insn);
1679 break;
1680 case OP_STORE:
1681 emitSTORE(insn);
1682 break;
1683 case OP_LINTERP:
1684 case OP_PINTERP:
1685 emitINTERP(insn);
1686 break;
1687 case OP_VFETCH:
1688 emitVFETCH(insn);
1689 break;
1690 case OP_EXPORT:
1691 emitEXPORT(insn);
1692 break;
1693 case OP_PFETCH:
1694 emitPFETCH(insn);
1695 break;
1696 case OP_EMIT:
1697 case OP_RESTART:
1698 emitOUT(insn);
1699 break;
1700 case OP_ADD:
1701 case OP_SUB:
1702 if (insn->dType == TYPE_F64)
1703 emitDADD(insn);
1704 else if (isFloatType(insn->dType))
1705 emitFADD(insn);
1706 else
1707 emitUADD(insn);
1708 break;
1709 case OP_MUL:
1710 if (insn->dType == TYPE_F64)
1711 emitDMUL(insn);
1712 else if (isFloatType(insn->dType))
1713 emitFMUL(insn);
1714 else
1715 emitIMUL(insn);
1716 break;
1717 case OP_MAD:
1718 case OP_FMA:
1719 if (insn->dType == TYPE_F64)
1720 emitDMAD(insn);
1721 else if (isFloatType(insn->dType))
1722 emitFMAD(insn);
1723 else
1724 emitIMAD(insn);
1725 break;
1726 case OP_SAD:
1727 emitISAD(insn);
1728 break;
1729 case OP_NOT:
1730 emitNOT(insn);
1731 break;
1732 case OP_AND:
1733 emitLogicOp(insn, 0);
1734 break;
1735 case OP_OR:
1736 emitLogicOp(insn, 1);
1737 break;
1738 case OP_XOR:
1739 emitLogicOp(insn, 2);
1740 break;
1741 case OP_SHL:
1742 case OP_SHR:
1743 emitShift(insn);
1744 break;
1745 case OP_SET:
1746 case OP_SET_AND:
1747 case OP_SET_OR:
1748 case OP_SET_XOR:
1749 emitSET(insn->asCmp());
1750 break;
1751 case OP_SELP:
1752 emitSELP(insn);
1753 break;
1754 case OP_SLCT:
1755 emitSLCT(insn->asCmp());
1756 break;
1757 case OP_MIN:
1758 case OP_MAX:
1759 emitMINMAX(insn);
1760 break;
1761 case OP_ABS:
1762 case OP_NEG:
1763 case OP_CEIL:
1764 case OP_FLOOR:
1765 case OP_TRUNC:
1766 case OP_CVT:
1767 case OP_SAT:
1768 emitCVT(insn);
1769 break;
1770 case OP_RSQ:
1771 emitSFnOp(insn, 5 + 2 * insn->subOp);
1772 break;
1773 case OP_RCP:
1774 emitSFnOp(insn, 4 + 2 * insn->subOp);
1775 break;
1776 case OP_LG2:
1777 emitSFnOp(insn, 3);
1778 break;
1779 case OP_EX2:
1780 emitSFnOp(insn, 2);
1781 break;
1782 case OP_SIN:
1783 emitSFnOp(insn, 1);
1784 break;
1785 case OP_COS:
1786 emitSFnOp(insn, 0);
1787 break;
1788 case OP_PRESIN:
1789 case OP_PREEX2:
1790 emitPreOp(insn);
1791 break;
1792 case OP_TEX:
1793 case OP_TXB:
1794 case OP_TXL:
1795 case OP_TXD:
1796 case OP_TXF:
1797 case OP_TXG:
1798 case OP_TXLQ:
1799 emitTEX(insn->asTex());
1800 break;
1801 case OP_TXQ:
1802 emitTXQ(insn->asTex());
1803 break;
1804 case OP_TEXBAR:
1805 emitTEXBAR(insn);
1806 break;
1807 case OP_PIXLD:
1808 emitPIXLD(insn);
1809 break;
1810 case OP_BRA:
1811 case OP_CALL:
1812 case OP_PRERET:
1813 case OP_RET:
1814 case OP_DISCARD:
1815 case OP_EXIT:
1816 case OP_PRECONT:
1817 case OP_CONT:
1818 case OP_PREBREAK:
1819 case OP_BREAK:
1820 case OP_JOINAT:
1821 case OP_BRKPT:
1822 case OP_QUADON:
1823 case OP_QUADPOP:
1824 emitFlow(insn);
1825 break;
1826 case OP_QUADOP:
1827 emitQUADOP(insn, insn->subOp, insn->lanes);
1828 break;
1829 case OP_DFDX:
1830 emitQUADOP(insn, insn->src(0).mod.neg() ? 0x66 : 0x99, 0x4);
1831 break;
1832 case OP_DFDY:
1833 emitQUADOP(insn, insn->src(0).mod.neg() ? 0x5a : 0xa5, 0x5);
1834 break;
1835 case OP_POPCNT:
1836 emitPOPC(insn);
1837 break;
1838 case OP_INSBF:
1839 emitINSBF(insn);
1840 break;
1841 case OP_EXTBF:
1842 emitEXTBF(insn);
1843 break;
1844 case OP_BFIND:
1845 emitBFIND(insn);
1846 break;
1847 case OP_JOIN:
1848 emitNOP(insn);
1849 insn->join = 1;
1850 break;
1851 case OP_PHI:
1852 case OP_UNION:
1853 case OP_CONSTRAINT:
1854 ERROR("operation should have been eliminated");
1855 return false;
1856 case OP_EXP:
1857 case OP_LOG:
1858 case OP_SQRT:
1859 case OP_POW:
1860 ERROR("operation should have been lowered\n");
1861 return false;
1862 default:
1863 ERROR("unknow op\n");
1864 return false;
1865 }
1866
1867 if (insn->join)
1868 code[0] |= 1 << 22;
1869
1870 code += 2;
1871 codeSize += 8;
1872 return true;
1873 }
1874
1875 uint32_t
1876 CodeEmitterGK110::getMinEncodingSize(const Instruction *i) const
1877 {
1878 // No more short instruction encodings.
1879 return 8;
1880 }
1881
1882 void
1883 CodeEmitterGK110::prepareEmission(Function *func)
1884 {
1885 const Target *targ = func->getProgram()->getTarget();
1886
1887 CodeEmitter::prepareEmission(func);
1888
1889 if (targ->hasSWSched)
1890 calculateSchedDataNVC0(targ, func);
1891 }
1892
1893 CodeEmitterGK110::CodeEmitterGK110(const TargetNVC0 *target)
1894 : CodeEmitter(target),
1895 targNVC0(target),
1896 writeIssueDelays(target->hasSWSched)
1897 {
1898 code = NULL;
1899 codeSize = codeSizeLimit = 0;
1900 relocInfo = NULL;
1901 }
1902
1903 CodeEmitter *
1904 TargetNVC0::createCodeEmitterGK110(Program::Type type)
1905 {
1906 CodeEmitterGK110 *emit = new CodeEmitterGK110(this);
1907 emit->setProgramType(type);
1908 return emit;
1909 }
1910
1911 } // namespace nv50_ir