projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nouveau: codegen: Do not silently fail in handeLOAD / handleSTORE / handleATOM
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_from_tgsi.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "tgsi/tgsi_dump.h"
24 #include "tgsi/tgsi_scan.h"
25 #include "tgsi/tgsi_util.h"
26
27 #include <set>
28
29 #include "codegen/nv50_ir.h"
30 #include "codegen/nv50_ir_util.h"
31 #include "codegen/nv50_ir_build_util.h"
32
33 namespace tgsi {
34
35 class Source;
36
37 static nv50_ir::operation translateOpcode(uint opcode);
38 static nv50_ir::DataFile translateFile(uint file);
39 static nv50_ir::TexTarget translateTexture(uint texTarg);
40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
41 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
42
43 class Instruction
44 {
45 public:
46 Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
47
48 class SrcRegister
49 {
50 public:
51 SrcRegister(const struct tgsi_full_src_register *src)
52 : reg(src->Register),
53 fsr(src)
54 { }
55
56 SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
57
58 SrcRegister(const struct tgsi_ind_register& ind)
59 : reg(tgsi_util_get_src_from_ind(&ind)),
60 fsr(NULL)
61 { }
62
63 struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
64 {
65 struct tgsi_src_register reg;
66 memset(&reg, 0, sizeof(reg));
67 reg.Index = off.Index;
68 reg.File = off.File;
69 reg.SwizzleX = off.SwizzleX;
70 reg.SwizzleY = off.SwizzleY;
71 reg.SwizzleZ = off.SwizzleZ;
72 return reg;
73 }
74
75 SrcRegister(const struct tgsi_texture_offset& off) :
76 reg(offsetToSrc(off)),
77 fsr(NULL)
78 { }
79
80 uint getFile() const { return reg.File; }
81
82 bool is2D() const { return reg.Dimension; }
83
84 bool isIndirect(int dim) const
85 {
86 return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
87 }
88
89 int getIndex(int dim) const
90 {
91 return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
92 }
93
94 int getSwizzle(int chan) const
95 {
96 return tgsi_util_get_src_register_swizzle(&reg, chan);
97 }
98
99 int getArrayId() const
100 {
101 if (isIndirect(0))
102 return fsr->Indirect.ArrayID;
103 return 0;
104 }
105
106 nv50_ir::Modifier getMod(int chan) const;
107
108 SrcRegister getIndirect(int dim) const
109 {
110 assert(fsr && isIndirect(dim));
111 if (dim)
112 return SrcRegister(fsr->DimIndirect);
113 return SrcRegister(fsr->Indirect);
114 }
115
116 uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
117 {
118 assert(reg.File == TGSI_FILE_IMMEDIATE);
119 assert(!reg.Absolute);
120 assert(!reg.Negate);
121 return info->immd.data[reg.Index * 4 + getSwizzle(c)];
122 }
123
124 private:
125 const struct tgsi_src_register reg;
126 const struct tgsi_full_src_register *fsr;
127 };
128
129 class DstRegister
130 {
131 public:
132 DstRegister(const struct tgsi_full_dst_register *dst)
133 : reg(dst->Register),
134 fdr(dst)
135 { }
136
137 DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
138
139 uint getFile() const { return reg.File; }
140
141 bool is2D() const { return reg.Dimension; }
142
143 bool isIndirect(int dim) const
144 {
145 return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
146 }
147
148 int getIndex(int dim) const
149 {
150 return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
151 }
152
153 unsigned int getMask() const { return reg.WriteMask; }
154
155 bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
156
157 SrcRegister getIndirect(int dim) const
158 {
159 assert(fdr && isIndirect(dim));
160 if (dim)
161 return SrcRegister(fdr->DimIndirect);
162 return SrcRegister(fdr->Indirect);
163 }
164
165 int getArrayId() const
166 {
167 if (isIndirect(0))
168 return fdr->Indirect.ArrayID;
169 return 0;
170 }
171
172 private:
173 const struct tgsi_dst_register reg;
174 const struct tgsi_full_dst_register *fdr;
175 };
176
177 inline uint getOpcode() const { return insn->Instruction.Opcode; }
178
179 unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
180 unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
181
182 // mask of used components of source s
183 unsigned int srcMask(unsigned int s) const;
184
185 SrcRegister getSrc(unsigned int s) const
186 {
187 assert(s < srcCount());
188 return SrcRegister(&insn->Src[s]);
189 }
190
191 DstRegister getDst(unsigned int d) const
192 {
193 assert(d < dstCount());
194 return DstRegister(&insn->Dst[d]);
195 }
196
197 SrcRegister getTexOffset(unsigned int i) const
198 {
199 assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
200 return SrcRegister(insn->TexOffsets[i]);
201 }
202
203 unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
204
205 bool checkDstSrcAliasing() const;
206
207 inline nv50_ir::operation getOP() const {
208 return translateOpcode(getOpcode()); }
209
210 nv50_ir::DataType inferSrcType() const;
211 nv50_ir::DataType inferDstType() const;
212
213 nv50_ir::CondCode getSetCond() const;
214
215 nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
216
217 nv50_ir::CacheMode getCacheMode() const {
218 if (!insn->Instruction.Memory)
219 return nv50_ir::CACHE_CA;
220 return translateCacheMode(insn->Memory.Qualifier);
221 }
222
223 inline uint getLabel() { return insn->Label.Label; }
224
225 unsigned getSaturate() const { return insn->Instruction.Saturate; }
226
227 void print() const
228 {
229 tgsi_dump_instruction(insn, 1);
230 }
231
232 private:
233 const struct tgsi_full_instruction *insn;
234 };
235
236 unsigned int Instruction::srcMask(unsigned int s) const
237 {
238 unsigned int mask = insn->Dst[0].Register.WriteMask;
239
240 switch (insn->Instruction.Opcode) {
241 case TGSI_OPCODE_COS:
242 case TGSI_OPCODE_SIN:
243 return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
244 case TGSI_OPCODE_DP2:
245 return 0x3;
246 case TGSI_OPCODE_DP3:
247 return 0x7;
248 case TGSI_OPCODE_DP4:
249 case TGSI_OPCODE_DPH:
250 case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
251 return 0xf;
252 case TGSI_OPCODE_DST:
253 return mask & (s ? 0xa : 0x6);
254 case TGSI_OPCODE_EX2:
255 case TGSI_OPCODE_EXP:
256 case TGSI_OPCODE_LG2:
257 case TGSI_OPCODE_LOG:
258 case TGSI_OPCODE_POW:
259 case TGSI_OPCODE_RCP:
260 case TGSI_OPCODE_RSQ:
261 case TGSI_OPCODE_SCS:
262 return 0x1;
263 case TGSI_OPCODE_IF:
264 case TGSI_OPCODE_UIF:
265 return 0x1;
266 case TGSI_OPCODE_LIT:
267 return 0xb;
268 case TGSI_OPCODE_TEX2:
269 case TGSI_OPCODE_TXB2:
270 case TGSI_OPCODE_TXL2:
271 return (s == 0) ? 0xf : 0x3;
272 case TGSI_OPCODE_TEX:
273 case TGSI_OPCODE_TXB:
274 case TGSI_OPCODE_TXD:
275 case TGSI_OPCODE_TXL:
276 case TGSI_OPCODE_TXP:
277 case TGSI_OPCODE_LODQ:
278 {
279 const struct tgsi_instruction_texture *tex = &insn->Texture;
280
281 assert(insn->Instruction.Texture);
282
283 mask = 0x7;
284 if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
285 insn->Instruction.Opcode != TGSI_OPCODE_TXD)
286 mask |= 0x8; /* bias, lod or proj */
287
288 switch (tex->Texture) {
289 case TGSI_TEXTURE_1D:
290 mask &= 0x9;
291 break;
292 case TGSI_TEXTURE_SHADOW1D:
293 mask &= 0xd;
294 break;
295 case TGSI_TEXTURE_1D_ARRAY:
296 case TGSI_TEXTURE_2D:
297 case TGSI_TEXTURE_RECT:
298 mask &= 0xb;
299 break;
300 case TGSI_TEXTURE_CUBE_ARRAY:
301 case TGSI_TEXTURE_SHADOW2D_ARRAY:
302 case TGSI_TEXTURE_SHADOWCUBE:
303 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
304 mask |= 0x8;
305 break;
306 default:
307 break;
308 }
309 }
310 return mask;
311 case TGSI_OPCODE_XPD:
312 {
313 unsigned int x = 0;
314 if (mask & 1) x |= 0x6;
315 if (mask & 2) x |= 0x5;
316 if (mask & 4) x |= 0x3;
317 return x;
318 }
319 case TGSI_OPCODE_D2I:
320 case TGSI_OPCODE_D2U:
321 case TGSI_OPCODE_D2F:
322 case TGSI_OPCODE_DSLT:
323 case TGSI_OPCODE_DSGE:
324 case TGSI_OPCODE_DSEQ:
325 case TGSI_OPCODE_DSNE:
326 switch (util_bitcount(mask)) {
327 case 1: return 0x3;
328 case 2: return 0xf;
329 default:
330 assert(!"unexpected mask");
331 return 0xf;
332 }
333 case TGSI_OPCODE_I2D:
334 case TGSI_OPCODE_U2D:
335 case TGSI_OPCODE_F2D: {
336 unsigned int x = 0;
337 if ((mask & 0x3) == 0x3)
338 x |= 1;
339 if ((mask & 0xc) == 0xc)
340 x |= 2;
341 return x;
342 }
343 case TGSI_OPCODE_PK2H:
344 return 0x3;
345 case TGSI_OPCODE_UP2H:
346 return 0x1;
347 default:
348 break;
349 }
350
351 return mask;
352 }
353
354 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
355 {
356 nv50_ir::Modifier m(0);
357
358 if (reg.Absolute)
359 m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
360 if (reg.Negate)
361 m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
362 return m;
363 }
364
365 static nv50_ir::DataFile translateFile(uint file)
366 {
367 switch (file) {
368 case TGSI_FILE_CONSTANT: return nv50_ir::FILE_MEMORY_CONST;
369 case TGSI_FILE_INPUT: return nv50_ir::FILE_SHADER_INPUT;
370 case TGSI_FILE_OUTPUT: return nv50_ir::FILE_SHADER_OUTPUT;
371 case TGSI_FILE_TEMPORARY: return nv50_ir::FILE_GPR;
372 case TGSI_FILE_ADDRESS: return nv50_ir::FILE_ADDRESS;
373 case TGSI_FILE_PREDICATE: return nv50_ir::FILE_PREDICATE;
374 case TGSI_FILE_IMMEDIATE: return nv50_ir::FILE_IMMEDIATE;
375 case TGSI_FILE_SYSTEM_VALUE: return nv50_ir::FILE_SYSTEM_VALUE;
376 case TGSI_FILE_BUFFER: return nv50_ir::FILE_MEMORY_GLOBAL;
377 case TGSI_FILE_MEMORY: return nv50_ir::FILE_MEMORY_GLOBAL;
378 case TGSI_FILE_SAMPLER:
379 case TGSI_FILE_NULL:
380 default:
381 return nv50_ir::FILE_NULL;
382 }
383 }
384
385 static nv50_ir::SVSemantic translateSysVal(uint sysval)
386 {
387 switch (sysval) {
388 case TGSI_SEMANTIC_FACE: return nv50_ir::SV_FACE;
389 case TGSI_SEMANTIC_PSIZE: return nv50_ir::SV_POINT_SIZE;
390 case TGSI_SEMANTIC_PRIMID: return nv50_ir::SV_PRIMITIVE_ID;
391 case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
392 case TGSI_SEMANTIC_VERTEXID: return nv50_ir::SV_VERTEX_ID;
393 case TGSI_SEMANTIC_GRID_SIZE: return nv50_ir::SV_NCTAID;
394 case TGSI_SEMANTIC_BLOCK_ID: return nv50_ir::SV_CTAID;
395 case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
396 case TGSI_SEMANTIC_THREAD_ID: return nv50_ir::SV_TID;
397 case TGSI_SEMANTIC_SAMPLEID: return nv50_ir::SV_SAMPLE_INDEX;
398 case TGSI_SEMANTIC_SAMPLEPOS: return nv50_ir::SV_SAMPLE_POS;
399 case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
400 case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
401 case TGSI_SEMANTIC_TESSCOORD: return nv50_ir::SV_TESS_COORD;
402 case TGSI_SEMANTIC_TESSOUTER: return nv50_ir::SV_TESS_OUTER;
403 case TGSI_SEMANTIC_TESSINNER: return nv50_ir::SV_TESS_INNER;
404 case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
405 case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
406 case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
407 case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
408 case TGSI_SEMANTIC_DRAWID: return nv50_ir::SV_DRAWID;
409 default:
410 assert(0);
411 return nv50_ir::SV_CLOCK;
412 }
413 }
414
415 #define NV50_IR_TEX_TARG_CASE(a, b) \
416 case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
417
418 static nv50_ir::TexTarget translateTexture(uint tex)
419 {
420 switch (tex) {
421 NV50_IR_TEX_TARG_CASE(1D, 1D);
422 NV50_IR_TEX_TARG_CASE(2D, 2D);
423 NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
424 NV50_IR_TEX_TARG_CASE(3D, 3D);
425 NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
426 NV50_IR_TEX_TARG_CASE(RECT, RECT);
427 NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
428 NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
429 NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
430 NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
431 NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
432 NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
433 NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
434 NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
435 NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
436 NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
437 NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
438 NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
439
440 case TGSI_TEXTURE_UNKNOWN:
441 default:
442 assert(!"invalid texture target");
443 return nv50_ir::TEX_TARGET_2D;
444 }
445 }
446
447 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
448 {
449 if (qualifier & TGSI_MEMORY_VOLATILE)
450 return nv50_ir::CACHE_CV;
451 if (qualifier & TGSI_MEMORY_COHERENT)
452 return nv50_ir::CACHE_CG;
453 return nv50_ir::CACHE_CA;
454 }
455
456 nv50_ir::DataType Instruction::inferSrcType() const
457 {
458 switch (getOpcode()) {
459 case TGSI_OPCODE_UIF:
460 case TGSI_OPCODE_AND:
461 case TGSI_OPCODE_OR:
462 case TGSI_OPCODE_XOR:
463 case TGSI_OPCODE_NOT:
464 case TGSI_OPCODE_SHL:
465 case TGSI_OPCODE_U2F:
466 case TGSI_OPCODE_U2D:
467 case TGSI_OPCODE_UADD:
468 case TGSI_OPCODE_UDIV:
469 case TGSI_OPCODE_UMOD:
470 case TGSI_OPCODE_UMAD:
471 case TGSI_OPCODE_UMUL:
472 case TGSI_OPCODE_UMUL_HI:
473 case TGSI_OPCODE_UMAX:
474 case TGSI_OPCODE_UMIN:
475 case TGSI_OPCODE_USEQ:
476 case TGSI_OPCODE_USGE:
477 case TGSI_OPCODE_USLT:
478 case TGSI_OPCODE_USNE:
479 case TGSI_OPCODE_USHR:
480 case TGSI_OPCODE_ATOMUADD:
481 case TGSI_OPCODE_ATOMXCHG:
482 case TGSI_OPCODE_ATOMCAS:
483 case TGSI_OPCODE_ATOMAND:
484 case TGSI_OPCODE_ATOMOR:
485 case TGSI_OPCODE_ATOMXOR:
486 case TGSI_OPCODE_ATOMUMIN:
487 case TGSI_OPCODE_ATOMUMAX:
488 case TGSI_OPCODE_UBFE:
489 case TGSI_OPCODE_UMSB:
490 case TGSI_OPCODE_UP2H:
491 return nv50_ir::TYPE_U32;
492 case TGSI_OPCODE_I2F:
493 case TGSI_OPCODE_I2D:
494 case TGSI_OPCODE_IDIV:
495 case TGSI_OPCODE_IMUL_HI:
496 case TGSI_OPCODE_IMAX:
497 case TGSI_OPCODE_IMIN:
498 case TGSI_OPCODE_IABS:
499 case TGSI_OPCODE_INEG:
500 case TGSI_OPCODE_ISGE:
501 case TGSI_OPCODE_ISHR:
502 case TGSI_OPCODE_ISLT:
503 case TGSI_OPCODE_ISSG:
504 case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
505 case TGSI_OPCODE_MOD:
506 case TGSI_OPCODE_UARL:
507 case TGSI_OPCODE_ATOMIMIN:
508 case TGSI_OPCODE_ATOMIMAX:
509 case TGSI_OPCODE_IBFE:
510 case TGSI_OPCODE_IMSB:
511 return nv50_ir::TYPE_S32;
512 case TGSI_OPCODE_D2F:
513 case TGSI_OPCODE_D2I:
514 case TGSI_OPCODE_D2U:
515 case TGSI_OPCODE_DABS:
516 case TGSI_OPCODE_DNEG:
517 case TGSI_OPCODE_DADD:
518 case TGSI_OPCODE_DMUL:
519 case TGSI_OPCODE_DMAX:
520 case TGSI_OPCODE_DMIN:
521 case TGSI_OPCODE_DSLT:
522 case TGSI_OPCODE_DSGE:
523 case TGSI_OPCODE_DSEQ:
524 case TGSI_OPCODE_DSNE:
525 case TGSI_OPCODE_DRCP:
526 case TGSI_OPCODE_DSQRT:
527 case TGSI_OPCODE_DMAD:
528 case TGSI_OPCODE_DFMA:
529 case TGSI_OPCODE_DFRAC:
530 case TGSI_OPCODE_DRSQ:
531 case TGSI_OPCODE_DTRUNC:
532 case TGSI_OPCODE_DCEIL:
533 case TGSI_OPCODE_DFLR:
534 case TGSI_OPCODE_DROUND:
535 return nv50_ir::TYPE_F64;
536 default:
537 return nv50_ir::TYPE_F32;
538 }
539 }
540
541 nv50_ir::DataType Instruction::inferDstType() const
542 {
543 switch (getOpcode()) {
544 case TGSI_OPCODE_D2U:
545 case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
546 case TGSI_OPCODE_D2I:
547 case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
548 case TGSI_OPCODE_FSEQ:
549 case TGSI_OPCODE_FSGE:
550 case TGSI_OPCODE_FSLT:
551 case TGSI_OPCODE_FSNE:
552 case TGSI_OPCODE_DSEQ:
553 case TGSI_OPCODE_DSGE:
554 case TGSI_OPCODE_DSLT:
555 case TGSI_OPCODE_DSNE:
556 case TGSI_OPCODE_PK2H:
557 return nv50_ir::TYPE_U32;
558 case TGSI_OPCODE_I2F:
559 case TGSI_OPCODE_U2F:
560 case TGSI_OPCODE_D2F:
561 case TGSI_OPCODE_UP2H:
562 return nv50_ir::TYPE_F32;
563 case TGSI_OPCODE_I2D:
564 case TGSI_OPCODE_U2D:
565 case TGSI_OPCODE_F2D:
566 return nv50_ir::TYPE_F64;
567 default:
568 return inferSrcType();
569 }
570 }
571
572 nv50_ir::CondCode Instruction::getSetCond() const
573 {
574 using namespace nv50_ir;
575
576 switch (getOpcode()) {
577 case TGSI_OPCODE_SLT:
578 case TGSI_OPCODE_ISLT:
579 case TGSI_OPCODE_USLT:
580 case TGSI_OPCODE_FSLT:
581 case TGSI_OPCODE_DSLT:
582 return CC_LT;
583 case TGSI_OPCODE_SLE:
584 return CC_LE;
585 case TGSI_OPCODE_SGE:
586 case TGSI_OPCODE_ISGE:
587 case TGSI_OPCODE_USGE:
588 case TGSI_OPCODE_FSGE:
589 case TGSI_OPCODE_DSGE:
590 return CC_GE;
591 case TGSI_OPCODE_SGT:
592 return CC_GT;
593 case TGSI_OPCODE_SEQ:
594 case TGSI_OPCODE_USEQ:
595 case TGSI_OPCODE_FSEQ:
596 case TGSI_OPCODE_DSEQ:
597 return CC_EQ;
598 case TGSI_OPCODE_SNE:
599 case TGSI_OPCODE_FSNE:
600 case TGSI_OPCODE_DSNE:
601 return CC_NEU;
602 case TGSI_OPCODE_USNE:
603 return CC_NE;
604 default:
605 return CC_ALWAYS;
606 }
607 }
608
609 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
610
611 static nv50_ir::operation translateOpcode(uint opcode)
612 {
613 switch (opcode) {
614 NV50_IR_OPCODE_CASE(ARL, SHL);
615 NV50_IR_OPCODE_CASE(MOV, MOV);
616
617 NV50_IR_OPCODE_CASE(RCP, RCP);
618 NV50_IR_OPCODE_CASE(RSQ, RSQ);
619 NV50_IR_OPCODE_CASE(SQRT, SQRT);
620
621 NV50_IR_OPCODE_CASE(MUL, MUL);
622 NV50_IR_OPCODE_CASE(ADD, ADD);
623
624 NV50_IR_OPCODE_CASE(MIN, MIN);
625 NV50_IR_OPCODE_CASE(MAX, MAX);
626 NV50_IR_OPCODE_CASE(SLT, SET);
627 NV50_IR_OPCODE_CASE(SGE, SET);
628 NV50_IR_OPCODE_CASE(MAD, MAD);
629 NV50_IR_OPCODE_CASE(FMA, FMA);
630 NV50_IR_OPCODE_CASE(SUB, SUB);
631
632 NV50_IR_OPCODE_CASE(FLR, FLOOR);
633 NV50_IR_OPCODE_CASE(ROUND, CVT);
634 NV50_IR_OPCODE_CASE(EX2, EX2);
635 NV50_IR_OPCODE_CASE(LG2, LG2);
636 NV50_IR_OPCODE_CASE(POW, POW);
637
638 NV50_IR_OPCODE_CASE(ABS, ABS);
639
640 NV50_IR_OPCODE_CASE(COS, COS);
641 NV50_IR_OPCODE_CASE(DDX, DFDX);
642 NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
643 NV50_IR_OPCODE_CASE(DDY, DFDY);
644 NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
645 NV50_IR_OPCODE_CASE(KILL, DISCARD);
646
647 NV50_IR_OPCODE_CASE(SEQ, SET);
648 NV50_IR_OPCODE_CASE(SGT, SET);
649 NV50_IR_OPCODE_CASE(SIN, SIN);
650 NV50_IR_OPCODE_CASE(SLE, SET);
651 NV50_IR_OPCODE_CASE(SNE, SET);
652 NV50_IR_OPCODE_CASE(TEX, TEX);
653 NV50_IR_OPCODE_CASE(TXD, TXD);
654 NV50_IR_OPCODE_CASE(TXP, TEX);
655
656 NV50_IR_OPCODE_CASE(CAL, CALL);
657 NV50_IR_OPCODE_CASE(RET, RET);
658 NV50_IR_OPCODE_CASE(CMP, SLCT);
659
660 NV50_IR_OPCODE_CASE(TXB, TXB);
661
662 NV50_IR_OPCODE_CASE(DIV, DIV);
663
664 NV50_IR_OPCODE_CASE(TXL, TXL);
665
666 NV50_IR_OPCODE_CASE(CEIL, CEIL);
667 NV50_IR_OPCODE_CASE(I2F, CVT);
668 NV50_IR_OPCODE_CASE(NOT, NOT);
669 NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
670 NV50_IR_OPCODE_CASE(SHL, SHL);
671
672 NV50_IR_OPCODE_CASE(AND, AND);
673 NV50_IR_OPCODE_CASE(OR, OR);
674 NV50_IR_OPCODE_CASE(MOD, MOD);
675 NV50_IR_OPCODE_CASE(XOR, XOR);
676 NV50_IR_OPCODE_CASE(SAD, SAD);
677 NV50_IR_OPCODE_CASE(TXF, TXF);
678 NV50_IR_OPCODE_CASE(TXQ, TXQ);
679 NV50_IR_OPCODE_CASE(TXQS, TXQ);
680 NV50_IR_OPCODE_CASE(TG4, TXG);
681 NV50_IR_OPCODE_CASE(LODQ, TXLQ);
682
683 NV50_IR_OPCODE_CASE(EMIT, EMIT);
684 NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
685
686 NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
687
688 NV50_IR_OPCODE_CASE(F2I, CVT);
689 NV50_IR_OPCODE_CASE(FSEQ, SET);
690 NV50_IR_OPCODE_CASE(FSGE, SET);
691 NV50_IR_OPCODE_CASE(FSLT, SET);
692 NV50_IR_OPCODE_CASE(FSNE, SET);
693 NV50_IR_OPCODE_CASE(IDIV, DIV);
694 NV50_IR_OPCODE_CASE(IMAX, MAX);
695 NV50_IR_OPCODE_CASE(IMIN, MIN);
696 NV50_IR_OPCODE_CASE(IABS, ABS);
697 NV50_IR_OPCODE_CASE(INEG, NEG);
698 NV50_IR_OPCODE_CASE(ISGE, SET);
699 NV50_IR_OPCODE_CASE(ISHR, SHR);
700 NV50_IR_OPCODE_CASE(ISLT, SET);
701 NV50_IR_OPCODE_CASE(F2U, CVT);
702 NV50_IR_OPCODE_CASE(U2F, CVT);
703 NV50_IR_OPCODE_CASE(UADD, ADD);
704 NV50_IR_OPCODE_CASE(UDIV, DIV);
705 NV50_IR_OPCODE_CASE(UMAD, MAD);
706 NV50_IR_OPCODE_CASE(UMAX, MAX);
707 NV50_IR_OPCODE_CASE(UMIN, MIN);
708 NV50_IR_OPCODE_CASE(UMOD, MOD);
709 NV50_IR_OPCODE_CASE(UMUL, MUL);
710 NV50_IR_OPCODE_CASE(USEQ, SET);
711 NV50_IR_OPCODE_CASE(USGE, SET);
712 NV50_IR_OPCODE_CASE(USHR, SHR);
713 NV50_IR_OPCODE_CASE(USLT, SET);
714 NV50_IR_OPCODE_CASE(USNE, SET);
715
716 NV50_IR_OPCODE_CASE(DABS, ABS);
717 NV50_IR_OPCODE_CASE(DNEG, NEG);
718 NV50_IR_OPCODE_CASE(DADD, ADD);
719 NV50_IR_OPCODE_CASE(DMUL, MUL);
720 NV50_IR_OPCODE_CASE(DMAX, MAX);
721 NV50_IR_OPCODE_CASE(DMIN, MIN);
722 NV50_IR_OPCODE_CASE(DSLT, SET);
723 NV50_IR_OPCODE_CASE(DSGE, SET);
724 NV50_IR_OPCODE_CASE(DSEQ, SET);
725 NV50_IR_OPCODE_CASE(DSNE, SET);
726 NV50_IR_OPCODE_CASE(DRCP, RCP);
727 NV50_IR_OPCODE_CASE(DSQRT, SQRT);
728 NV50_IR_OPCODE_CASE(DMAD, MAD);
729 NV50_IR_OPCODE_CASE(DFMA, FMA);
730 NV50_IR_OPCODE_CASE(D2I, CVT);
731 NV50_IR_OPCODE_CASE(D2U, CVT);
732 NV50_IR_OPCODE_CASE(I2D, CVT);
733 NV50_IR_OPCODE_CASE(U2D, CVT);
734 NV50_IR_OPCODE_CASE(DRSQ, RSQ);
735 NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
736 NV50_IR_OPCODE_CASE(DCEIL, CEIL);
737 NV50_IR_OPCODE_CASE(DFLR, FLOOR);
738 NV50_IR_OPCODE_CASE(DROUND, CVT);
739
740 NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
741 NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
742
743 NV50_IR_OPCODE_CASE(SAMPLE, TEX);
744 NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
745 NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
746 NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
747 NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
748 NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
749 NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
750 NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
751 NV50_IR_OPCODE_CASE(GATHER4, TXG);
752 NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
753
754 NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
755 NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
756 NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
757 NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
758 NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
759 NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
760 NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
761 NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
762 NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
763 NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
764
765 NV50_IR_OPCODE_CASE(TEX2, TEX);
766 NV50_IR_OPCODE_CASE(TXB2, TXB);
767 NV50_IR_OPCODE_CASE(TXL2, TXL);
768
769 NV50_IR_OPCODE_CASE(IBFE, EXTBF);
770 NV50_IR_OPCODE_CASE(UBFE, EXTBF);
771 NV50_IR_OPCODE_CASE(BFI, INSBF);
772 NV50_IR_OPCODE_CASE(BREV, EXTBF);
773 NV50_IR_OPCODE_CASE(POPC, POPCNT);
774 NV50_IR_OPCODE_CASE(LSB, BFIND);
775 NV50_IR_OPCODE_CASE(IMSB, BFIND);
776 NV50_IR_OPCODE_CASE(UMSB, BFIND);
777
778 NV50_IR_OPCODE_CASE(END, EXIT);
779
780 default:
781 return nv50_ir::OP_NOP;
782 }
783 }
784
785 static uint16_t opcodeToSubOp(uint opcode)
786 {
787 switch (opcode) {
788 case TGSI_OPCODE_LFENCE: return NV50_IR_SUBOP_MEMBAR(L, GL);
789 case TGSI_OPCODE_SFENCE: return NV50_IR_SUBOP_MEMBAR(S, GL);
790 case TGSI_OPCODE_MFENCE: return NV50_IR_SUBOP_MEMBAR(M, GL);
791 case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
792 case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
793 case TGSI_OPCODE_ATOMCAS: return NV50_IR_SUBOP_ATOM_CAS;
794 case TGSI_OPCODE_ATOMAND: return NV50_IR_SUBOP_ATOM_AND;
795 case TGSI_OPCODE_ATOMOR: return NV50_IR_SUBOP_ATOM_OR;
796 case TGSI_OPCODE_ATOMXOR: return NV50_IR_SUBOP_ATOM_XOR;
797 case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
798 case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
799 case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
800 case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
801 case TGSI_OPCODE_IMUL_HI:
802 case TGSI_OPCODE_UMUL_HI:
803 return NV50_IR_SUBOP_MUL_HIGH;
804 default:
805 return 0;
806 }
807 }
808
809 bool Instruction::checkDstSrcAliasing() const
810 {
811 if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
812 return false;
813
814 for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
815 if (insn->Src[s].Register.File == TGSI_FILE_NULL)
816 break;
817 if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
818 insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
819 return true;
820 }
821 return false;
822 }
823
824 class Source
825 {
826 public:
827 Source(struct nv50_ir_prog_info *);
828 ~Source();
829
830 public:
831 bool scanSource();
832 unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
833
834 public:
835 struct tgsi_shader_info scan;
836 struct tgsi_full_instruction *insns;
837 const struct tgsi_token *tokens;
838 struct nv50_ir_prog_info *info;
839
840 nv50_ir::DynArray tempArrays;
841 nv50_ir::DynArray immdArrays;
842
843 typedef nv50_ir::BuildUtil::Location Location;
844 // these registers are per-subroutine, cannot be used for parameter passing
845 std::set<Location> locals;
846
847 std::set<int> indirectTempArrays;
848 std::map<int, int> indirectTempOffsets;
849 std::map<int, std::pair<int, int> > tempArrayInfo;
850 std::vector<int> tempArrayId;
851
852 int clipVertexOutput;
853
854 struct TextureView {
855 uint8_t target; // TGSI_TEXTURE_*
856 };
857 std::vector<TextureView> textureViews;
858
859 /*
860 struct Resource {
861 uint8_t target; // TGSI_TEXTURE_*
862 bool raw;
863 uint8_t slot; // $surface index
864 };
865 std::vector<Resource> resources;
866 */
867
868 struct MemoryFile {
869 uint8_t mem_type; // TGSI_MEMORY_TYPE_*
870 };
871 std::vector<MemoryFile> memoryFiles;
872
873 private:
874 int inferSysValDirection(unsigned sn) const;
875 bool scanDeclaration(const struct tgsi_full_declaration *);
876 bool scanInstruction(const struct tgsi_full_instruction *);
877 void scanProperty(const struct tgsi_full_property *);
878 void scanImmediate(const struct tgsi_full_immediate *);
879
880 inline bool isEdgeFlagPassthrough(const Instruction&) const;
881 };
882
883 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
884 {
885 tokens = (const struct tgsi_token *)info->bin.source;
886
887 if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
888 tgsi_dump(tokens, 0);
889 }
890
891 Source::~Source()
892 {
893 if (insns)
894 FREE(insns);
895
896 if (info->immd.data)
897 FREE(info->immd.data);
898 if (info->immd.type)
899 FREE(info->immd.type);
900 }
901
902 bool Source::scanSource()
903 {
904 unsigned insnCount = 0;
905 struct tgsi_parse_context parse;
906
907 tgsi_scan_shader(tokens, &scan);
908
909 insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
910 sizeof(insns[0]));
911 if (!insns)
912 return false;
913
914 clipVertexOutput = -1;
915
916 textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
917 //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
918 tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
919 memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
920
921 info->immd.bufSize = 0;
922
923 info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
924 info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
925 info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
926
927 if (info->type == PIPE_SHADER_FRAGMENT) {
928 info->prop.fp.writesDepth = scan.writes_z;
929 info->prop.fp.usesDiscard = scan.uses_kill;
930 } else
931 if (info->type == PIPE_SHADER_GEOMETRY) {
932 info->prop.gp.instanceCount = 1; // default value
933 }
934
935 info->io.viewportId = -1;
936
937 info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
938 info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
939
940 tgsi_parse_init(&parse, tokens);
941 while (!tgsi_parse_end_of_tokens(&parse)) {
942 tgsi_parse_token(&parse);
943
944 switch (parse.FullToken.Token.Type) {
945 case TGSI_TOKEN_TYPE_IMMEDIATE:
946 scanImmediate(&parse.FullToken.FullImmediate);
947 break;
948 case TGSI_TOKEN_TYPE_DECLARATION:
949 scanDeclaration(&parse.FullToken.FullDeclaration);
950 break;
951 case TGSI_TOKEN_TYPE_INSTRUCTION:
952 insns[insnCount++] = parse.FullToken.FullInstruction;
953 scanInstruction(&parse.FullToken.FullInstruction);
954 break;
955 case TGSI_TOKEN_TYPE_PROPERTY:
956 scanProperty(&parse.FullToken.FullProperty);
957 break;
958 default:
959 INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
960 break;
961 }
962 }
963 tgsi_parse_free(&parse);
964
965 if (indirectTempArrays.size()) {
966 int tempBase = 0;
967 for (std::set<int>::const_iterator it = indirectTempArrays.begin();
968 it != indirectTempArrays.end(); ++it) {
969 std::pair<int, int>& info = tempArrayInfo[*it];
970 indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
971 tempBase += info.second;
972 }
973 info->bin.tlsSpace += tempBase * 16;
974 }
975
976 if (info->io.genUserClip > 0) {
977 info->io.clipDistances = info->io.genUserClip;
978
979 const unsigned int nOut = (info->io.genUserClip + 3) / 4;
980
981 for (unsigned int n = 0; n < nOut; ++n) {
982 unsigned int i = info->numOutputs++;
983 info->out[i].id = i;
984 info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
985 info->out[i].si = n;
986 info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
987 }
988 }
989
990 return info->assignSlots(info) == 0;
991 }
992
993 void Source::scanProperty(const struct tgsi_full_property *prop)
994 {
995 switch (prop->Property.PropertyName) {
996 case TGSI_PROPERTY_GS_OUTPUT_PRIM:
997 info->prop.gp.outputPrim = prop->u[0].Data;
998 break;
999 case TGSI_PROPERTY_GS_INPUT_PRIM:
1000 info->prop.gp.inputPrim = prop->u[0].Data;
1001 break;
1002 case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
1003 info->prop.gp.maxVertices = prop->u[0].Data;
1004 break;
1005 case TGSI_PROPERTY_GS_INVOCATIONS:
1006 info->prop.gp.instanceCount = prop->u[0].Data;
1007 break;
1008 case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1009 info->prop.fp.separateFragData = true;
1010 break;
1011 case TGSI_PROPERTY_FS_COORD_ORIGIN:
1012 case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1013 // we don't care
1014 break;
1015 case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1016 info->io.genUserClip = -1;
1017 break;
1018 case TGSI_PROPERTY_TCS_VERTICES_OUT:
1019 info->prop.tp.outputPatchSize = prop->u[0].Data;
1020 break;
1021 case TGSI_PROPERTY_TES_PRIM_MODE:
1022 info->prop.tp.domain = prop->u[0].Data;
1023 break;
1024 case TGSI_PROPERTY_TES_SPACING:
1025 info->prop.tp.partitioning = prop->u[0].Data;
1026 break;
1027 case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1028 info->prop.tp.winding = prop->u[0].Data;
1029 break;
1030 case TGSI_PROPERTY_TES_POINT_MODE:
1031 if (prop->u[0].Data)
1032 info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1033 else
1034 info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1035 break;
1036 case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1037 info->io.clipDistances = prop->u[0].Data;
1038 break;
1039 case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1040 info->io.cullDistances = prop->u[0].Data;
1041 break;
1042 default:
1043 INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1044 break;
1045 }
1046 }
1047
1048 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1049 {
1050 const unsigned n = info->immd.count++;
1051
1052 assert(n < scan.immediate_count);
1053
1054 for (int c = 0; c < 4; ++c)
1055 info->immd.data[n * 4 + c] = imm->u[c].Uint;
1056
1057 info->immd.type[n] = imm->Immediate.DataType;
1058 }
1059
1060 int Source::inferSysValDirection(unsigned sn) const
1061 {
1062 switch (sn) {
1063 case TGSI_SEMANTIC_INSTANCEID:
1064 case TGSI_SEMANTIC_VERTEXID:
1065 return 1;
1066 case TGSI_SEMANTIC_LAYER:
1067 #if 0
1068 case TGSI_SEMANTIC_VIEWPORTINDEX:
1069 return 0;
1070 #endif
1071 case TGSI_SEMANTIC_PRIMID:
1072 return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1073 default:
1074 return 0;
1075 }
1076 }
1077
1078 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1079 {
1080 unsigned i, c;
1081 unsigned sn = TGSI_SEMANTIC_GENERIC;
1082 unsigned si = 0;
1083 const unsigned first = decl->Range.First, last = decl->Range.Last;
1084 const int arrayId = decl->Array.ArrayID;
1085
1086 if (decl->Declaration.Semantic) {
1087 sn = decl->Semantic.Name;
1088 si = decl->Semantic.Index;
1089 }
1090
1091 if (decl->Declaration.Local) {
1092 for (i = first; i <= last; ++i) {
1093 for (c = 0; c < 4; ++c) {
1094 locals.insert(
1095 Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1096 }
1097 }
1098 }
1099
1100 switch (decl->Declaration.File) {
1101 case TGSI_FILE_INPUT:
1102 if (info->type == PIPE_SHADER_VERTEX) {
1103 // all vertex attributes are equal
1104 for (i = first; i <= last; ++i) {
1105 info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1106 info->in[i].si = i;
1107 }
1108 } else {
1109 for (i = first; i <= last; ++i, ++si) {
1110 info->in[i].id = i;
1111 info->in[i].sn = sn;
1112 info->in[i].si = si;
1113 if (info->type == PIPE_SHADER_FRAGMENT) {
1114 // translate interpolation mode
1115 switch (decl->Interp.Interpolate) {
1116 case TGSI_INTERPOLATE_CONSTANT:
1117 info->in[i].flat = 1;
1118 break;
1119 case TGSI_INTERPOLATE_COLOR:
1120 info->in[i].sc = 1;
1121 break;
1122 case TGSI_INTERPOLATE_LINEAR:
1123 info->in[i].linear = 1;
1124 break;
1125 default:
1126 break;
1127 }
1128 if (decl->Interp.Location)
1129 info->in[i].centroid = 1;
1130 }
1131
1132 if (sn == TGSI_SEMANTIC_PATCH)
1133 info->in[i].patch = 1;
1134 if (sn == TGSI_SEMANTIC_PATCH)
1135 info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1136 }
1137 }
1138 break;
1139 case TGSI_FILE_OUTPUT:
1140 for (i = first; i <= last; ++i, ++si) {
1141 switch (sn) {
1142 case TGSI_SEMANTIC_POSITION:
1143 if (info->type == PIPE_SHADER_FRAGMENT)
1144 info->io.fragDepth = i;
1145 else
1146 if (clipVertexOutput < 0)
1147 clipVertexOutput = i;
1148 break;
1149 case TGSI_SEMANTIC_COLOR:
1150 if (info->type == PIPE_SHADER_FRAGMENT)
1151 info->prop.fp.numColourResults++;
1152 break;
1153 case TGSI_SEMANTIC_EDGEFLAG:
1154 info->io.edgeFlagOut = i;
1155 break;
1156 case TGSI_SEMANTIC_CLIPVERTEX:
1157 clipVertexOutput = i;
1158 break;
1159 case TGSI_SEMANTIC_CLIPDIST:
1160 info->io.genUserClip = -1;
1161 break;
1162 case TGSI_SEMANTIC_SAMPLEMASK:
1163 info->io.sampleMask = i;
1164 break;
1165 case TGSI_SEMANTIC_VIEWPORT_INDEX:
1166 info->io.viewportId = i;
1167 break;
1168 case TGSI_SEMANTIC_PATCH:
1169 info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1170 /* fallthrough */
1171 case TGSI_SEMANTIC_TESSOUTER:
1172 case TGSI_SEMANTIC_TESSINNER:
1173 info->out[i].patch = 1;
1174 break;
1175 default:
1176 break;
1177 }
1178 info->out[i].id = i;
1179 info->out[i].sn = sn;
1180 info->out[i].si = si;
1181 }
1182 break;
1183 case TGSI_FILE_SYSTEM_VALUE:
1184 switch (sn) {
1185 case TGSI_SEMANTIC_INSTANCEID:
1186 info->io.instanceId = first;
1187 break;
1188 case TGSI_SEMANTIC_VERTEXID:
1189 info->io.vertexId = first;
1190 break;
1191 case TGSI_SEMANTIC_BASEVERTEX:
1192 case TGSI_SEMANTIC_BASEINSTANCE:
1193 case TGSI_SEMANTIC_DRAWID:
1194 info->prop.vp.usesDrawParameters = true;
1195 break;
1196 default:
1197 break;
1198 }
1199 for (i = first; i <= last; ++i, ++si) {
1200 info->sv[i].sn = sn;
1201 info->sv[i].si = si;
1202 info->sv[i].input = inferSysValDirection(sn);
1203
1204 switch (sn) {
1205 case TGSI_SEMANTIC_TESSOUTER:
1206 case TGSI_SEMANTIC_TESSINNER:
1207 info->sv[i].patch = 1;
1208 break;
1209 }
1210 }
1211 break;
1212 /*
1213 case TGSI_FILE_RESOURCE:
1214 for (i = first; i <= last; ++i) {
1215 resources[i].target = decl->Resource.Resource;
1216 resources[i].raw = decl->Resource.Raw;
1217 resources[i].slot = i;
1218 }
1219 break;
1220 */
1221 case TGSI_FILE_SAMPLER_VIEW:
1222 for (i = first; i <= last; ++i)
1223 textureViews[i].target = decl->SamplerView.Resource;
1224 break;
1225 case TGSI_FILE_MEMORY:
1226 for (i = first; i <= last; ++i)
1227 memoryFiles[i].mem_type = decl->Declaration.MemType;
1228 break;
1229 case TGSI_FILE_NULL:
1230 case TGSI_FILE_TEMPORARY:
1231 for (i = first; i <= last; ++i)
1232 tempArrayId[i] = arrayId;
1233 if (arrayId)
1234 tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1235 first, last - first + 1)));
1236 break;
1237 case TGSI_FILE_ADDRESS:
1238 case TGSI_FILE_CONSTANT:
1239 case TGSI_FILE_IMMEDIATE:
1240 case TGSI_FILE_PREDICATE:
1241 case TGSI_FILE_SAMPLER:
1242 case TGSI_FILE_BUFFER:
1243 break;
1244 default:
1245 ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1246 return false;
1247 }
1248 return true;
1249 }
1250
1251 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1252 {
1253 return insn.getOpcode() == TGSI_OPCODE_MOV &&
1254 insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1255 insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1256 }
1257
1258 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1259 {
1260 Instruction insn(inst);
1261
1262 if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1263 info->numBarriers = 1;
1264
1265 if (insn.dstCount()) {
1266 Instruction::DstRegister dst = insn.getDst(0);
1267
1268 if (dst.getFile() == TGSI_FILE_OUTPUT) {
1269 if (dst.isIndirect(0))
1270 for (unsigned i = 0; i < info->numOutputs; ++i)
1271 info->out[i].mask = 0xf;
1272 else
1273 info->out[dst.getIndex(0)].mask |= dst.getMask();
1274
1275 if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1276 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1277 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1278 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1279 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1280 info->out[dst.getIndex(0)].mask &= 1;
1281
1282 if (isEdgeFlagPassthrough(insn))
1283 info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1284 } else
1285 if (dst.getFile() == TGSI_FILE_TEMPORARY) {
1286 if (dst.isIndirect(0))
1287 indirectTempArrays.insert(dst.getArrayId());
1288 } else
1289 if (dst.getFile() == TGSI_FILE_BUFFER) {
1290 info->io.globalAccess |= 0x2;
1291 }
1292 }
1293
1294 for (unsigned s = 0; s < insn.srcCount(); ++s) {
1295 Instruction::SrcRegister src = insn.getSrc(s);
1296 if (src.getFile() == TGSI_FILE_TEMPORARY) {
1297 if (src.isIndirect(0))
1298 indirectTempArrays.insert(src.getArrayId());
1299 } else
1300 if (src.getFile() == TGSI_FILE_BUFFER) {
1301 info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1302 0x1 : 0x2;
1303 } else
1304 if (src.getFile() == TGSI_FILE_OUTPUT) {
1305 if (src.isIndirect(0)) {
1306 // We don't know which one is accessed, just mark everything for
1307 // reading. This is an extremely unlikely occurrence.
1308 for (unsigned i = 0; i < info->numOutputs; ++i)
1309 info->out[i].oread = 1;
1310 } else {
1311 info->out[src.getIndex(0)].oread = 1;
1312 }
1313 }
1314 if (src.getFile() != TGSI_FILE_INPUT)
1315 continue;
1316 unsigned mask = insn.srcMask(s);
1317
1318 if (src.isIndirect(0)) {
1319 for (unsigned i = 0; i < info->numInputs; ++i)
1320 info->in[i].mask = 0xf;
1321 } else {
1322 const int i = src.getIndex(0);
1323 for (unsigned c = 0; c < 4; ++c) {
1324 if (!(mask & (1 << c)))
1325 continue;
1326 int k = src.getSwizzle(c);
1327 if (k <= TGSI_SWIZZLE_W)
1328 info->in[i].mask |= 1 << k;
1329 }
1330 switch (info->in[i].sn) {
1331 case TGSI_SEMANTIC_PSIZE:
1332 case TGSI_SEMANTIC_PRIMID:
1333 case TGSI_SEMANTIC_FOG:
1334 info->in[i].mask &= 0x1;
1335 break;
1336 case TGSI_SEMANTIC_PCOORD:
1337 info->in[i].mask &= 0x3;
1338 break;
1339 default:
1340 break;
1341 }
1342 }
1343 }
1344 return true;
1345 }
1346
1347 nv50_ir::TexInstruction::Target
1348 Instruction::getTexture(const tgsi::Source *code, int s) const
1349 {
1350 // XXX: indirect access
1351 unsigned int r;
1352
1353 switch (getSrc(s).getFile()) {
1354 /*
1355 case TGSI_FILE_RESOURCE:
1356 r = getSrc(s).getIndex(0);
1357 return translateTexture(code->resources.at(r).target);
1358 */
1359 case TGSI_FILE_SAMPLER_VIEW:
1360 r = getSrc(s).getIndex(0);
1361 return translateTexture(code->textureViews.at(r).target);
1362 default:
1363 return translateTexture(insn->Texture.Texture);
1364 }
1365 }
1366
1367 } // namespace tgsi
1368
1369 namespace {
1370
1371 using namespace nv50_ir;
1372
1373 class Converter : public BuildUtil
1374 {
1375 public:
1376 Converter(Program *, const tgsi::Source *);
1377 ~Converter();
1378
1379 bool run();
1380
1381 private:
1382 struct Subroutine
1383 {
1384 Subroutine(Function *f) : f(f) { }
1385 Function *f;
1386 ValueMap values;
1387 };
1388
1389 Value *shiftAddress(Value *);
1390 Value *getVertexBase(int s);
1391 Value *getOutputBase(int s);
1392 DataArray *getArrayForFile(unsigned file, int idx);
1393 Value *fetchSrc(int s, int c);
1394 Value *acquireDst(int d, int c);
1395 void storeDst(int d, int c, Value *);
1396
1397 Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1398 void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1399 Value *val, Value *ptr);
1400
1401 void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1402 Value *applySrcMod(Value *, int s, int c);
1403
1404 Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1405 Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1406 Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1407
1408 bool handleInstruction(const struct tgsi_full_instruction *);
1409 void exportOutputs();
1410 inline Subroutine *getSubroutine(unsigned ip);
1411 inline Subroutine *getSubroutine(Function *);
1412 inline bool isEndOfSubroutine(uint ip);
1413
1414 void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1415
1416 // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1417 void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1418 void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1419 void handleTXF(Value *dst0[4], int R, int L_M);
1420 void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1421 void handleLIT(Value *dst0[4]);
1422 void handleUserClipPlanes();
1423
1424 // Symbol *getResourceBase(int r);
1425 // void getResourceCoords(std::vector<Value *>&, int r, int s);
1426
1427 void handleLOAD(Value *dst0[4]);
1428 void handleSTORE();
1429 void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1430
1431 void handleINTERP(Value *dst0[4]);
1432
1433 uint8_t translateInterpMode(const struct nv50_ir_varying *var,
1434 operation& op);
1435 Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1436
1437 void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1438
1439 Value *buildDot(int dim);
1440
1441 class BindArgumentsPass : public Pass {
1442 public:
1443 BindArgumentsPass(Converter &conv) : conv(conv) { }
1444
1445 private:
1446 Converter &conv;
1447 Subroutine *sub;
1448
1449 inline const Location *getValueLocation(Subroutine *, Value *);
1450
1451 template<typename T> inline void
1452 updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1453 T (Function::*proto));
1454
1455 template<typename T> inline void
1456 updatePrototype(BitSet *set, void (Function::*updateSet)(),
1457 T (Function::*proto));
1458
1459 protected:
1460 bool visit(Function *);
1461 bool visit(BasicBlock *bb) { return false; }
1462 };
1463
1464 private:
1465 const tgsi::Source *code;
1466 const struct nv50_ir_prog_info *info;
1467
1468 struct {
1469 std::map<unsigned, Subroutine> map;
1470 Subroutine *cur;
1471 } sub;
1472
1473 uint ip; // instruction pointer
1474
1475 tgsi::Instruction tgsi;
1476
1477 DataType dstTy;
1478 DataType srcTy;
1479
1480 DataArray tData; // TGSI_FILE_TEMPORARY
1481 DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1482 DataArray aData; // TGSI_FILE_ADDRESS
1483 DataArray pData; // TGSI_FILE_PREDICATE
1484 DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1485
1486 Value *zero;
1487 Value *fragCoord[4];
1488 Value *clipVtx[4];
1489
1490 Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1491 uint8_t vtxBaseValid;
1492
1493 Value *outBase; // base address of vertex out patch (for TCP)
1494
1495 Stack condBBs; // fork BB, then else clause BB
1496 Stack joinBBs; // fork BB, for inserting join ops on ENDIF
1497 Stack loopBBs; // loop headers
1498 Stack breakBBs; // end of / after loop
1499
1500 Value *viewport;
1501 };
1502
1503 Symbol *
1504 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1505 {
1506 const int swz = src.getSwizzle(c);
1507
1508 /* TODO: Use Array ID when it's available for the index */
1509 return makeSym(src.getFile(),
1510 src.is2D() ? src.getIndex(1) : 0,
1511 src.getIndex(0), swz,
1512 src.getIndex(0) * 16 + swz * 4);
1513 }
1514
1515 Symbol *
1516 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1517 {
1518 /* TODO: Use Array ID when it's available for the index */
1519 return makeSym(dst.getFile(),
1520 dst.is2D() ? dst.getIndex(1) : 0,
1521 dst.getIndex(0), c,
1522 dst.getIndex(0) * 16 + c * 4);
1523 }
1524
1525 Symbol *
1526 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1527 {
1528 Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1529
1530 sym->reg.fileIndex = fileIdx;
1531
1532 if (tgsiFile == TGSI_FILE_MEMORY) {
1533 switch (code->memoryFiles[fileIdx].mem_type) {
1534 case TGSI_MEMORY_TYPE_SHARED:
1535 sym->setFile(FILE_MEMORY_SHARED);
1536 break;
1537 case TGSI_MEMORY_TYPE_INPUT:
1538 assert(prog->getType() == Program::TYPE_COMPUTE);
1539 assert(idx == -1);
1540 sym->setFile(FILE_SHADER_INPUT);
1541 address += info->prop.cp.inputOffset;
1542 break;
1543 default:
1544 assert(0); /* TODO: Add support for global and private memory */
1545 }
1546 }
1547
1548 if (idx >= 0) {
1549 if (sym->reg.file == FILE_SHADER_INPUT)
1550 sym->setOffset(info->in[idx].slot[c] * 4);
1551 else
1552 if (sym->reg.file == FILE_SHADER_OUTPUT)
1553 sym->setOffset(info->out[idx].slot[c] * 4);
1554 else
1555 if (sym->reg.file == FILE_SYSTEM_VALUE)
1556 sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1557 else
1558 sym->setOffset(address);
1559 } else {
1560 sym->setOffset(address);
1561 }
1562 return sym;
1563 }
1564
1565 uint8_t
1566 Converter::translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1567 {
1568 uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1569
1570 if (var->flat)
1571 mode = NV50_IR_INTERP_FLAT;
1572 else
1573 if (var->linear)
1574 mode = NV50_IR_INTERP_LINEAR;
1575 else
1576 if (var->sc)
1577 mode = NV50_IR_INTERP_SC;
1578
1579 op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1580 ? OP_PINTERP : OP_LINTERP;
1581
1582 if (var->centroid)
1583 mode |= NV50_IR_INTERP_CENTROID;
1584
1585 return mode;
1586 }
1587
1588 Value *
1589 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1590 {
1591 operation op;
1592
1593 // XXX: no way to know interpolation mode if we don't know what's accessed
1594 const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1595 src.getIndex(0)], op);
1596
1597 Instruction *insn = new_Instruction(func, op, TYPE_F32);
1598
1599 insn->setDef(0, getScratch());
1600 insn->setSrc(0, srcToSym(src, c));
1601 if (op == OP_PINTERP)
1602 insn->setSrc(1, fragCoord[3]);
1603 if (ptr)
1604 insn->setIndirect(0, 0, ptr);
1605
1606 insn->setInterpolate(mode);
1607
1608 bb->insertTail(insn);
1609 return insn->getDef(0);
1610 }
1611
1612 Value *
1613 Converter::applySrcMod(Value *val, int s, int c)
1614 {
1615 Modifier m = tgsi.getSrc(s).getMod(c);
1616 DataType ty = tgsi.inferSrcType();
1617
1618 if (m & Modifier(NV50_IR_MOD_ABS))
1619 val = mkOp1v(OP_ABS, ty, getScratch(), val);
1620
1621 if (m & Modifier(NV50_IR_MOD_NEG))
1622 val = mkOp1v(OP_NEG, ty, getScratch(), val);
1623
1624 return val;
1625 }
1626
1627 Value *
1628 Converter::getVertexBase(int s)
1629 {
1630 assert(s < 5);
1631 if (!(vtxBaseValid & (1 << s))) {
1632 const int index = tgsi.getSrc(s).getIndex(1);
1633 Value *rel = NULL;
1634 if (tgsi.getSrc(s).isIndirect(1))
1635 rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1636 vtxBaseValid |= 1 << s;
1637 vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1638 mkImm(index), rel);
1639 }
1640 return vtxBase[s];
1641 }
1642
1643 Value *
1644 Converter::getOutputBase(int s)
1645 {
1646 assert(s < 5);
1647 if (!(vtxBaseValid & (1 << s))) {
1648 Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1649 if (tgsi.getSrc(s).isIndirect(1))
1650 offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1651 fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1652 offset);
1653 vtxBaseValid |= 1 << s;
1654 vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1655 }
1656 return vtxBase[s];
1657 }
1658
1659 Value *
1660 Converter::fetchSrc(int s, int c)
1661 {
1662 Value *res;
1663 Value *ptr = NULL, *dimRel = NULL;
1664
1665 tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1666
1667 if (src.isIndirect(0))
1668 ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1669
1670 if (src.is2D()) {
1671 switch (src.getFile()) {
1672 case TGSI_FILE_OUTPUT:
1673 dimRel = getOutputBase(s);
1674 break;
1675 case TGSI_FILE_INPUT:
1676 dimRel = getVertexBase(s);
1677 break;
1678 case TGSI_FILE_CONSTANT:
1679 // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1680 if (src.isIndirect(1))
1681 dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1682 break;
1683 default:
1684 break;
1685 }
1686 }
1687
1688 res = fetchSrc(src, c, ptr);
1689
1690 if (dimRel)
1691 res->getInsn()->setIndirect(0, 1, dimRel);
1692
1693 return applySrcMod(res, s, c);
1694 }
1695
1696 Converter::DataArray *
1697 Converter::getArrayForFile(unsigned file, int idx)
1698 {
1699 switch (file) {
1700 case TGSI_FILE_TEMPORARY:
1701 return idx == 0 ? &tData : &lData;
1702 case TGSI_FILE_PREDICATE:
1703 return &pData;
1704 case TGSI_FILE_ADDRESS:
1705 return &aData;
1706 case TGSI_FILE_OUTPUT:
1707 assert(prog->getType() == Program::TYPE_FRAGMENT);
1708 return &oData;
1709 default:
1710 assert(!"invalid/unhandled TGSI source file");
1711 return NULL;
1712 }
1713 }
1714
1715 Value *
1716 Converter::shiftAddress(Value *index)
1717 {
1718 if (!index)
1719 return NULL;
1720 return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1721 }
1722
1723 void
1724 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1725 {
1726 std::map<int, int>::const_iterator it =
1727 code->indirectTempOffsets.find(arrayId);
1728 if (it == code->indirectTempOffsets.end())
1729 return;
1730
1731 idx2d = 1;
1732 idx += it->second;
1733 }
1734
1735 Value *
1736 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1737 {
1738 int idx2d = src.is2D() ? src.getIndex(1) : 0;
1739 int idx = src.getIndex(0);
1740 const int swz = src.getSwizzle(c);
1741 Instruction *ld;
1742
1743 switch (src.getFile()) {
1744 case TGSI_FILE_IMMEDIATE:
1745 assert(!ptr);
1746 return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1747 case TGSI_FILE_CONSTANT:
1748 return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1749 case TGSI_FILE_INPUT:
1750 if (prog->getType() == Program::TYPE_FRAGMENT) {
1751 // don't load masked inputs, won't be assigned a slot
1752 if (!ptr && !(info->in[idx].mask & (1 << swz)))
1753 return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1754 return interpolate(src, c, shiftAddress(ptr));
1755 } else
1756 if (prog->getType() == Program::TYPE_GEOMETRY) {
1757 if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1758 return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1759 // XXX: This is going to be a problem with scalar arrays, i.e. when
1760 // we cannot assume that the address is given in units of vec4.
1761 //
1762 // nv50 and nvc0 need different things here, so let the lowering
1763 // passes decide what to do with the address
1764 if (ptr)
1765 return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1766 }
1767 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1768 ld->perPatch = info->in[idx].patch;
1769 return ld->getDef(0);
1770 case TGSI_FILE_OUTPUT:
1771 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1772 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1773 ld->perPatch = info->out[idx].patch;
1774 return ld->getDef(0);
1775 case TGSI_FILE_SYSTEM_VALUE:
1776 assert(!ptr);
1777 ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1778 ld->perPatch = info->sv[idx].patch;
1779 return ld->getDef(0);
1780 case TGSI_FILE_TEMPORARY: {
1781 int arrayid = src.getArrayId();
1782 if (!arrayid)
1783 arrayid = code->tempArrayId[idx];
1784 adjustTempIndex(arrayid, idx, idx2d);
1785 }
1786 /* fallthrough */
1787 default:
1788 return getArrayForFile(src.getFile(), idx2d)->load(
1789 sub.cur->values, idx, swz, shiftAddress(ptr));
1790 }
1791 }
1792
1793 Value *
1794 Converter::acquireDst(int d, int c)
1795 {
1796 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1797 const unsigned f = dst.getFile();
1798 int idx = dst.getIndex(0);
1799 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1800
1801 if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY)
1802 return NULL;
1803
1804 if (dst.isIndirect(0) ||
1805 f == TGSI_FILE_SYSTEM_VALUE ||
1806 (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1807 return getScratch();
1808
1809 if (f == TGSI_FILE_TEMPORARY) {
1810 int arrayid = dst.getArrayId();
1811 if (!arrayid)
1812 arrayid = code->tempArrayId[idx];
1813 adjustTempIndex(arrayid, idx, idx2d);
1814 }
1815
1816 return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1817 }
1818
1819 void
1820 Converter::storeDst(int d, int c, Value *val)
1821 {
1822 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1823
1824 if (tgsi.getSaturate()) {
1825 mkOp1(OP_SAT, dstTy, val, val);
1826 }
1827
1828 Value *ptr = NULL;
1829 if (dst.isIndirect(0))
1830 ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1831
1832 if (info->io.genUserClip > 0 &&
1833 dst.getFile() == TGSI_FILE_OUTPUT &&
1834 !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1835 mkMov(clipVtx[c], val);
1836 val = clipVtx[c];
1837 }
1838
1839 storeDst(dst, c, val, ptr);
1840 }
1841
1842 void
1843 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1844 Value *val, Value *ptr)
1845 {
1846 const unsigned f = dst.getFile();
1847 int idx = dst.getIndex(0);
1848 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1849
1850 if (f == TGSI_FILE_SYSTEM_VALUE) {
1851 assert(!ptr);
1852 mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1853 } else
1854 if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1855
1856 if (ptr || (info->out[idx].mask & (1 << c))) {
1857 /* Save the viewport index into a scratch register so that it can be
1858 exported at EMIT time */
1859 if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1860 viewport != NULL)
1861 mkOp1(OP_MOV, TYPE_U32, viewport, val);
1862 else
1863 mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1864 info->out[idx].patch;
1865 }
1866 } else
1867 if (f == TGSI_FILE_TEMPORARY ||
1868 f == TGSI_FILE_PREDICATE ||
1869 f == TGSI_FILE_ADDRESS ||
1870 f == TGSI_FILE_OUTPUT) {
1871 if (f == TGSI_FILE_TEMPORARY) {
1872 int arrayid = dst.getArrayId();
1873 if (!arrayid)
1874 arrayid = code->tempArrayId[idx];
1875 adjustTempIndex(arrayid, idx, idx2d);
1876 }
1877
1878 getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1879 } else {
1880 assert(!"invalid dst file");
1881 }
1882 }
1883
1884 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1885 for (chan = 0; chan < 4; ++chan) \
1886 if (!inst.getDst(d).isMasked(chan))
1887
1888 Value *
1889 Converter::buildDot(int dim)
1890 {
1891 assert(dim > 0);
1892
1893 Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1894 Value *dotp = getScratch();
1895
1896 mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1897
1898 for (int c = 1; c < dim; ++c) {
1899 src0 = fetchSrc(0, c);
1900 src1 = fetchSrc(1, c);
1901 mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1902 }
1903 return dotp;
1904 }
1905
1906 void
1907 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1908 {
1909 FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1910 join->fixed = 1;
1911 conv->insertHead(join);
1912
1913 assert(!fork->joinAt);
1914 fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1915 fork->insertBefore(fork->getExit(), fork->joinAt);
1916 }
1917
1918 void
1919 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1920 {
1921 unsigned rIdx = 0, sIdx = 0;
1922
1923 if (R >= 0)
1924 rIdx = tgsi.getSrc(R).getIndex(0);
1925 if (S >= 0)
1926 sIdx = tgsi.getSrc(S).getIndex(0);
1927
1928 tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1929
1930 if (tgsi.getSrc(R).isIndirect(0)) {
1931 tex->tex.rIndirectSrc = s;
1932 tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1933 }
1934 if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1935 tex->tex.sIndirectSrc = s;
1936 tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1937 }
1938 }
1939
1940 void
1941 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1942 {
1943 TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1944 tex->tex.query = query;
1945 unsigned int c, d;
1946
1947 for (d = 0, c = 0; c < 4; ++c) {
1948 if (!dst0[c])
1949 continue;
1950 tex->tex.mask |= 1 << c;
1951 tex->setDef(d++, dst0[c]);
1952 }
1953 if (query == TXQ_DIMS)
1954 tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1955 else
1956 tex->setSrc((c = 0), zero);
1957
1958 setTexRS(tex, ++c, R, -1);
1959
1960 bb->insertTail(tex);
1961 }
1962
1963 void
1964 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1965 {
1966 Value *proj = fetchSrc(0, 3);
1967 Instruction *insn = proj->getUniqueInsn();
1968 int c;
1969
1970 if (insn->op == OP_PINTERP) {
1971 bb->insertTail(insn = cloneForward(func, insn));
1972 insn->op = OP_LINTERP;
1973 insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1974 insn->setSrc(1, NULL);
1975 proj = insn->getDef(0);
1976 }
1977 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1978
1979 for (c = 0; c < 4; ++c) {
1980 if (!(mask & (1 << c)))
1981 continue;
1982 if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1983 continue;
1984 mask &= ~(1 << c);
1985
1986 bb->insertTail(insn = cloneForward(func, insn));
1987 insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1988 insn->setSrc(1, proj);
1989 dst[c] = insn->getDef(0);
1990 }
1991 if (!mask)
1992 return;
1993
1994 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1995
1996 for (c = 0; c < 4; ++c)
1997 if (mask & (1 << c))
1998 dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1999 }
2000
2001 // order of nv50 ir sources: x y z layer lod/bias shadow
2002 // order of TGSI TEX sources: x y z layer shadow lod/bias
2003 // lowering will finally set the hw specific order (like array first on nvc0)
2004 void
2005 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
2006 {
2007 Value *arg[4], *src[8];
2008 Value *lod = NULL, *shd = NULL;
2009 unsigned int s, c, d;
2010 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2011
2012 TexInstruction::Target tgt = tgsi.getTexture(code, R);
2013
2014 for (s = 0; s < tgt.getArgCount(); ++s)
2015 arg[s] = src[s] = fetchSrc(0, s);
2016
2017 if (texi->op == OP_TXL || texi->op == OP_TXB)
2018 lod = fetchSrc(L >> 4, L & 3);
2019
2020 if (C == 0x0f)
2021 C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2022
2023 if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
2024 tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
2025 shd = fetchSrc(1, 0);
2026 else if (tgt.isShadow())
2027 shd = fetchSrc(C >> 4, C & 3);
2028
2029 if (texi->op == OP_TXD) {
2030 for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2031 texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2032 texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2033 }
2034 }
2035
2036 // cube textures don't care about projection value, it's divided out
2037 if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2038 unsigned int n = tgt.getDim();
2039 if (shd) {
2040 arg[n] = shd;
2041 ++n;
2042 assert(tgt.getDim() == tgt.getArgCount());
2043 }
2044 loadProjTexCoords(src, arg, (1 << n) - 1);
2045 if (shd)
2046 shd = src[n - 1];
2047 }
2048
2049 for (c = 0, d = 0; c < 4; ++c) {
2050 if (dst[c]) {
2051 texi->setDef(d++, dst[c]);
2052 texi->tex.mask |= 1 << c;
2053 } else {
2054 // NOTE: maybe hook up def too, for CSE
2055 }
2056 }
2057 for (s = 0; s < tgt.getArgCount(); ++s)
2058 texi->setSrc(s, src[s]);
2059 if (lod)
2060 texi->setSrc(s++, lod);
2061 if (shd)
2062 texi->setSrc(s++, shd);
2063
2064 setTexRS(texi, s, R, S);
2065
2066 if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2067 texi->tex.levelZero = true;
2068 if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2069 texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2070
2071 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2072 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2073 for (c = 0; c < 3; ++c) {
2074 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2075 texi->offset[s][c].setInsn(texi);
2076 }
2077 }
2078
2079 bb->insertTail(texi);
2080 }
2081
2082 // 1st source: xyz = coordinates, w = lod/sample
2083 // 2nd source: offset
2084 void
2085 Converter::handleTXF(Value *dst[4], int R, int L_M)
2086 {
2087 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2088 int ms;
2089 unsigned int c, d, s;
2090
2091 texi->tex.target = tgsi.getTexture(code, R);
2092
2093 ms = texi->tex.target.isMS() ? 1 : 0;
2094 texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2095
2096 for (c = 0, d = 0; c < 4; ++c) {
2097 if (dst[c]) {
2098 texi->setDef(d++, dst[c]);
2099 texi->tex.mask |= 1 << c;
2100 }
2101 }
2102 for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2103 texi->setSrc(c, fetchSrc(0, c));
2104 texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2105
2106 setTexRS(texi, c, R, -1);
2107
2108 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2109 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2110 for (c = 0; c < 3; ++c) {
2111 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2112 texi->offset[s][c].setInsn(texi);
2113 }
2114 }
2115
2116 bb->insertTail(texi);
2117 }
2118
2119 void
2120 Converter::handleLIT(Value *dst0[4])
2121 {
2122 Value *val0 = NULL;
2123 unsigned int mask = tgsi.getDst(0).getMask();
2124
2125 if (mask & (1 << 0))
2126 loadImm(dst0[0], 1.0f);
2127
2128 if (mask & (1 << 3))
2129 loadImm(dst0[3], 1.0f);
2130
2131 if (mask & (3 << 1)) {
2132 val0 = getScratch();
2133 mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2134 if (mask & (1 << 1))
2135 mkMov(dst0[1], val0);
2136 }
2137
2138 if (mask & (1 << 2)) {
2139 Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2140 Value *val1 = getScratch(), *val3 = getScratch();
2141
2142 Value *pos128 = loadImm(NULL, +127.999999f);
2143 Value *neg128 = loadImm(NULL, -127.999999f);
2144
2145 mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2146 mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2147 mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2148 mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2149
2150 mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2151 }
2152 }
2153
2154 /* Keep this around for now as reference when adding img support
2155 static inline bool
2156 isResourceSpecial(const int r)
2157 {
2158 return (r == TGSI_RESOURCE_GLOBAL ||
2159 r == TGSI_RESOURCE_LOCAL ||
2160 r == TGSI_RESOURCE_PRIVATE ||
2161 r == TGSI_RESOURCE_INPUT);
2162 }
2163
2164 static inline bool
2165 isResourceRaw(const tgsi::Source *code, const int r)
2166 {
2167 return isResourceSpecial(r) || code->resources[r].raw;
2168 }
2169
2170 static inline nv50_ir::TexTarget
2171 getResourceTarget(const tgsi::Source *code, int r)
2172 {
2173 if (isResourceSpecial(r))
2174 return nv50_ir::TEX_TARGET_BUFFER;
2175 return tgsi::translateTexture(code->resources.at(r).target);
2176 }
2177
2178 Symbol *
2179 Converter::getResourceBase(const int r)
2180 {
2181 Symbol *sym = NULL;
2182
2183 switch (r) {
2184 case TGSI_RESOURCE_GLOBAL:
2185 sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL,
2186 info->io.auxCBSlot);
2187 break;
2188 case TGSI_RESOURCE_LOCAL:
2189 assert(prog->getType() == Program::TYPE_COMPUTE);
2190 sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2191 info->prop.cp.sharedOffset);
2192 break;
2193 case TGSI_RESOURCE_PRIVATE:
2194 sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2195 info->bin.tlsSpace);
2196 break;
2197 case TGSI_RESOURCE_INPUT:
2198 assert(prog->getType() == Program::TYPE_COMPUTE);
2199 sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2200 info->prop.cp.inputOffset);
2201 break;
2202 default:
2203 sym = new_Symbol(prog,
2204 nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2205 break;
2206 }
2207 return sym;
2208 }
2209
2210 void
2211 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2212 {
2213 const int arg =
2214 TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2215
2216 for (int c = 0; c < arg; ++c)
2217 coords.push_back(fetchSrc(s, c));
2218
2219 // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2220 if (r == TGSI_RESOURCE_LOCAL ||
2221 r == TGSI_RESOURCE_PRIVATE ||
2222 r == TGSI_RESOURCE_INPUT)
2223 coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2224 coords[0]);
2225 }
2226
2227 static inline int
2228 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2229 {
2230 int n = 0;
2231
2232 while (mask) {
2233 if (mask & 1) {
2234 size[n]++;
2235 } else {
2236 if (size[n])
2237 comp[n = 1] = size[0] + 1;
2238 else
2239 comp[n]++;
2240 }
2241 mask >>= 1;
2242 }
2243 if (size[0] == 3) {
2244 n = 1;
2245 size[0] = (comp[0] == 1) ? 1 : 2;
2246 size[1] = 3 - size[0];
2247 comp[1] = comp[0] + size[0];
2248 }
2249 return n + 1;
2250 }
2251 */
2252
2253 // For raw loads, granularity is 4 byte.
2254 // Usage of the texture read mask on OP_SULDP is not allowed.
2255 void
2256 Converter::handleLOAD(Value *dst0[4])
2257 {
2258 const int r = tgsi.getSrc(0).getIndex(0);
2259 int c;
2260 std::vector<Value *> off, src, ldv, def;
2261
2262 switch (tgsi.getSrc(0).getFile()) {
2263 case TGSI_FILE_BUFFER:
2264 case TGSI_FILE_MEMORY:
2265 for (c = 0; c < 4; ++c) {
2266 if (!dst0[c])
2267 continue;
2268
2269 Value *off = fetchSrc(1, c);
2270 Symbol *sym;
2271 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2272 off = NULL;
2273 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2274 tgsi.getSrc(1).getValueU32(0, info) + 4 * c);
2275 } else {
2276 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 4 * c);
2277 }
2278
2279 Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2280 ld->cache = tgsi.getCacheMode();
2281 if (tgsi.getSrc(0).isIndirect(0))
2282 ld->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
2283 }
2284 break;
2285 default:
2286 assert(!"Unsupported srcFile for LOAD");
2287 }
2288
2289 /* Keep this around for now as reference when adding img support
2290 getResourceCoords(off, r, 1);
2291
2292 if (isResourceRaw(code, r)) {
2293 uint8_t mask = 0;
2294 uint8_t comp[2] = { 0, 0 };
2295 uint8_t size[2] = { 0, 0 };
2296
2297 Symbol *base = getResourceBase(r);
2298
2299 // determine the base and size of the at most 2 load ops
2300 for (c = 0; c < 4; ++c)
2301 if (!tgsi.getDst(0).isMasked(c))
2302 mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2303
2304 int n = partitionLoadStore(comp, size, mask);
2305
2306 src = off;
2307
2308 def.resize(4); // index by component, the ones we need will be non-NULL
2309 for (c = 0; c < 4; ++c) {
2310 if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2311 def[c] = dst0[c];
2312 else
2313 if (mask & (1 << c))
2314 def[c] = getScratch();
2315 }
2316
2317 const bool useLd = isResourceSpecial(r) ||
2318 (info->io.nv50styleSurfaces &&
2319 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2320
2321 for (int i = 0; i < n; ++i) {
2322 ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2323
2324 if (comp[i]) // adjust x component of source address if necessary
2325 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2326 off[0], mkImm(comp[i] * 4));
2327 else
2328 src[0] = off[0];
2329
2330 if (useLd) {
2331 Instruction *ld =
2332 mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2333 for (size_t c = 1; c < ldv.size(); ++c)
2334 ld->setDef(c, ldv[c]);
2335 } else {
2336 mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2337 0, ldv, src)->dType = typeOfSize(size[i] * 4);
2338 }
2339 }
2340 } else {
2341 def.resize(4);
2342 for (c = 0; c < 4; ++c) {
2343 if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2344 def[c] = getScratch();
2345 else
2346 def[c] = dst0[c];
2347 }
2348
2349 mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2350 def, off);
2351 }
2352 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2353 if (dst0[c] != def[c])
2354 mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2355 */
2356 }
2357
2358 // For formatted stores, the write mask on OP_SUSTP can be used.
2359 // Raw stores have to be split.
2360 void
2361 Converter::handleSTORE()
2362 {
2363 const int r = tgsi.getDst(0).getIndex(0);
2364 int c;
2365 std::vector<Value *> off, src, dummy;
2366
2367 switch (tgsi.getDst(0).getFile()) {
2368 case TGSI_FILE_BUFFER:
2369 case TGSI_FILE_MEMORY:
2370 for (c = 0; c < 4; ++c) {
2371 if (!(tgsi.getDst(0).getMask() & (1 << c)))
2372 continue;
2373
2374 Symbol *sym;
2375 Value *off;
2376 if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2377 off = NULL;
2378 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2379 tgsi.getSrc(0).getValueU32(0, info) + 4 * c);
2380 } else {
2381 off = fetchSrc(0, 0);
2382 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2383 }
2384
2385 Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2386 st->cache = tgsi.getCacheMode();
2387 if (tgsi.getDst(0).isIndirect(0))
2388 st->setIndirect(0, 1, fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0));
2389 }
2390 break;
2391 default:
2392 assert(!"Unsupported dstFile for STORE");
2393 }
2394
2395 /* Keep this around for now as reference when adding img support
2396 getResourceCoords(off, r, 0);
2397 src = off;
2398 const int s = src.size();
2399
2400 if (isResourceRaw(code, r)) {
2401 uint8_t comp[2] = { 0, 0 };
2402 uint8_t size[2] = { 0, 0 };
2403
2404 int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2405
2406 Symbol *base = getResourceBase(r);
2407
2408 const bool useSt = isResourceSpecial(r) ||
2409 (info->io.nv50styleSurfaces &&
2410 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2411
2412 for (int i = 0; i < n; ++i) {
2413 if (comp[i]) // adjust x component of source address if necessary
2414 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2415 off[0], mkImm(comp[i] * 4));
2416 else
2417 src[0] = off[0];
2418
2419 const DataType stTy = typeOfSize(size[i] * 4);
2420
2421 if (useSt) {
2422 Instruction *st =
2423 mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2424 for (c = 1; c < size[i]; ++c)
2425 st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2426 st->setIndirect(0, 0, src[0]);
2427 } else {
2428 // attach values to be stored
2429 src.resize(s + size[i]);
2430 for (c = 0; c < size[i]; ++c)
2431 src[s + c] = fetchSrc(1, comp[i] + c);
2432 mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2433 0, dummy, src)->setType(stTy);
2434 }
2435 }
2436 } else {
2437 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2438 src.push_back(fetchSrc(1, c));
2439
2440 mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2441 dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2442 }
2443 */
2444 }
2445
2446 // XXX: These only work on resources with the single-component u32/s32 formats.
2447 // Therefore the result is replicated. This might not be intended by TGSI, but
2448 // operating on more than 1 component would produce undefined results because
2449 // they do not exist.
2450 void
2451 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2452 {
2453 const int r = tgsi.getSrc(0).getIndex(0);
2454 std::vector<Value *> srcv;
2455 std::vector<Value *> defv;
2456 LValue *dst = getScratch();
2457
2458 switch (tgsi.getSrc(0).getFile()) {
2459 case TGSI_FILE_BUFFER:
2460 case TGSI_FILE_MEMORY:
2461 for (int c = 0; c < 4; ++c) {
2462 if (!dst0[c])
2463 continue;
2464
2465 Instruction *insn;
2466 Value *off = fetchSrc(1, c), *off2 = NULL;
2467 Value *sym;
2468 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2469 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2470 tgsi.getSrc(1).getValueU32(c, info));
2471 else
2472 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2473 if (tgsi.getSrc(0).isIndirect(0))
2474 off2 = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2475 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2476 insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2477 else
2478 insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2479 if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2480 insn->setIndirect(0, 0, off);
2481 if (off2)
2482 insn->setIndirect(0, 1, off2);
2483 insn->subOp = subOp;
2484 }
2485 for (int c = 0; c < 4; ++c)
2486 if (dst0[c])
2487 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2488 break;
2489 default:
2490 assert(!"Unsupported srcFile for ATOM");
2491 }
2492
2493 /* Keep this around for now as reference when adding img support
2494 getResourceCoords(srcv, r, 1);
2495
2496 if (isResourceSpecial(r)) {
2497 assert(r != TGSI_RESOURCE_INPUT);
2498 Instruction *insn;
2499 insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2500 insn->subOp = subOp;
2501 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2502 insn->setSrc(2, fetchSrc(3, 0));
2503 insn->setIndirect(0, 0, srcv.at(0));
2504 } else {
2505 operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2506 TexTarget targ = getResourceTarget(code, r);
2507 int idx = code->resources[r].slot;
2508 defv.push_back(dst);
2509 srcv.push_back(fetchSrc(2, 0));
2510 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2511 srcv.push_back(fetchSrc(3, 0));
2512 TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2513 tex->subOp = subOp;
2514 tex->tex.mask = 1;
2515 tex->setType(ty);
2516 }
2517
2518 for (int c = 0; c < 4; ++c)
2519 if (dst0[c])
2520 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2521 */
2522 }
2523
2524 void
2525 Converter::handleINTERP(Value *dst[4])
2526 {
2527 // Check whether the input is linear. All other attributes ignored.
2528 Instruction *insn;
2529 Value *offset = NULL, *ptr = NULL, *w = NULL;
2530 bool linear;
2531 operation op;
2532 int c, mode;
2533
2534 tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2535 assert(src.getFile() == TGSI_FILE_INPUT);
2536
2537 if (src.isIndirect(0))
2538 ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2539
2540 // XXX: no way to know interp mode if we don't know the index
2541 linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2542 if (linear) {
2543 op = OP_LINTERP;
2544 mode = NV50_IR_INTERP_LINEAR;
2545 } else {
2546 op = OP_PINTERP;
2547 mode = NV50_IR_INTERP_PERSPECTIVE;
2548 }
2549
2550 switch (tgsi.getOpcode()) {
2551 case TGSI_OPCODE_INTERP_CENTROID:
2552 mode |= NV50_IR_INTERP_CENTROID;
2553 break;
2554 case TGSI_OPCODE_INTERP_SAMPLE:
2555 insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2556 insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2557 mode |= NV50_IR_INTERP_OFFSET;
2558 break;
2559 case TGSI_OPCODE_INTERP_OFFSET: {
2560 // The input in src1.xy is float, but we need a single 32-bit value
2561 // where the upper and lower 16 bits are encoded in S0.12 format. We need
2562 // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2563 // and then convert to s32.
2564 Value *offs[2];
2565 for (c = 0; c < 2; c++) {
2566 offs[c] = fetchSrc(1, c);
2567 mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2568 mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2569 mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2570 mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2571 }
2572 offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2573 offs[1], mkImm(0x1010), offs[0]);
2574 mode |= NV50_IR_INTERP_OFFSET;
2575 break;
2576 }
2577 }
2578
2579 if (op == OP_PINTERP) {
2580 if (offset) {
2581 w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2582 mkOp1(OP_RCP, TYPE_F32, w, w);
2583 } else {
2584 w = fragCoord[3];
2585 }
2586 }
2587
2588
2589 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2590 insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2591 if (op == OP_PINTERP)
2592 insn->setSrc(1, w);
2593 if (ptr)
2594 insn->setIndirect(0, 0, ptr);
2595 if (offset)
2596 insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2597
2598 insn->setInterpolate(mode);
2599 }
2600 }
2601
2602 Converter::Subroutine *
2603 Converter::getSubroutine(unsigned ip)
2604 {
2605 std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2606
2607 if (it == sub.map.end())
2608 it = sub.map.insert(std::make_pair(
2609 ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2610
2611 return &it->second;
2612 }
2613
2614 Converter::Subroutine *
2615 Converter::getSubroutine(Function *f)
2616 {
2617 unsigned ip = f->getLabel();
2618 std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2619
2620 if (it == sub.map.end())
2621 it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2622
2623 return &it->second;
2624 }
2625
2626 bool
2627 Converter::isEndOfSubroutine(uint ip)
2628 {
2629 assert(ip < code->scan.num_instructions);
2630 tgsi::Instruction insn(&code->insns[ip]);
2631 return (insn.getOpcode() == TGSI_OPCODE_END ||
2632 insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2633 // does END occur at end of main or the very end ?
2634 insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2635 }
2636
2637 bool
2638 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2639 {
2640 Instruction *geni;
2641
2642 Value *dst0[4], *rDst0[4];
2643 Value *src0, *src1, *src2, *src3;
2644 Value *val0, *val1;
2645 int c;
2646
2647 tgsi = tgsi::Instruction(insn);
2648
2649 bool useScratchDst = tgsi.checkDstSrcAliasing();
2650
2651 operation op = tgsi.getOP();
2652 dstTy = tgsi.inferDstType();
2653 srcTy = tgsi.inferSrcType();
2654
2655 unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2656
2657 if (tgsi.dstCount()) {
2658 for (c = 0; c < 4; ++c) {
2659 rDst0[c] = acquireDst(0, c);
2660 dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2661 }
2662 }
2663
2664 switch (tgsi.getOpcode()) {
2665 case TGSI_OPCODE_ADD:
2666 case TGSI_OPCODE_UADD:
2667 case TGSI_OPCODE_AND:
2668 case TGSI_OPCODE_DIV:
2669 case TGSI_OPCODE_IDIV:
2670 case TGSI_OPCODE_UDIV:
2671 case TGSI_OPCODE_MAX:
2672 case TGSI_OPCODE_MIN:
2673 case TGSI_OPCODE_IMAX:
2674 case TGSI_OPCODE_IMIN:
2675 case TGSI_OPCODE_UMAX:
2676 case TGSI_OPCODE_UMIN:
2677 case TGSI_OPCODE_MOD:
2678 case TGSI_OPCODE_UMOD:
2679 case TGSI_OPCODE_MUL:
2680 case TGSI_OPCODE_UMUL:
2681 case TGSI_OPCODE_IMUL_HI:
2682 case TGSI_OPCODE_UMUL_HI:
2683 case TGSI_OPCODE_OR:
2684 case TGSI_OPCODE_SHL:
2685 case TGSI_OPCODE_ISHR:
2686 case TGSI_OPCODE_USHR:
2687 case TGSI_OPCODE_SUB:
2688 case TGSI_OPCODE_XOR:
2689 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2690 src0 = fetchSrc(0, c);
2691 src1 = fetchSrc(1, c);
2692 geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2693 geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2694 }
2695 break;
2696 case TGSI_OPCODE_MAD:
2697 case TGSI_OPCODE_UMAD:
2698 case TGSI_OPCODE_SAD:
2699 case TGSI_OPCODE_FMA:
2700 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2701 src0 = fetchSrc(0, c);
2702 src1 = fetchSrc(1, c);
2703 src2 = fetchSrc(2, c);
2704 mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2705 }
2706 break;
2707 case TGSI_OPCODE_MOV:
2708 case TGSI_OPCODE_ABS:
2709 case TGSI_OPCODE_CEIL:
2710 case TGSI_OPCODE_FLR:
2711 case TGSI_OPCODE_TRUNC:
2712 case TGSI_OPCODE_RCP:
2713 case TGSI_OPCODE_SQRT:
2714 case TGSI_OPCODE_IABS:
2715 case TGSI_OPCODE_INEG:
2716 case TGSI_OPCODE_NOT:
2717 case TGSI_OPCODE_DDX:
2718 case TGSI_OPCODE_DDY:
2719 case TGSI_OPCODE_DDX_FINE:
2720 case TGSI_OPCODE_DDY_FINE:
2721 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2722 mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2723 break;
2724 case TGSI_OPCODE_RSQ:
2725 src0 = fetchSrc(0, 0);
2726 val0 = getScratch();
2727 mkOp1(OP_ABS, TYPE_F32, val0, src0);
2728 mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2729 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2730 mkMov(dst0[c], val0);
2731 break;
2732 case TGSI_OPCODE_ARL:
2733 case TGSI_OPCODE_ARR:
2734 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2735 const RoundMode rnd =
2736 tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2737 src0 = fetchSrc(0, c);
2738 mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2739 }
2740 break;
2741 case TGSI_OPCODE_UARL:
2742 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2743 mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2744 break;
2745 case TGSI_OPCODE_POW:
2746 val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2747 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2748 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2749 break;
2750 case TGSI_OPCODE_EX2:
2751 case TGSI_OPCODE_LG2:
2752 val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2753 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2754 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2755 break;
2756 case TGSI_OPCODE_COS:
2757 case TGSI_OPCODE_SIN:
2758 val0 = getScratch();
2759 if (mask & 7) {
2760 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2761 mkOp1(op, TYPE_F32, val0, val0);
2762 for (c = 0; c < 3; ++c)
2763 if (dst0[c])
2764 mkMov(dst0[c], val0);
2765 }
2766 if (dst0[3]) {
2767 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2768 mkOp1(op, TYPE_F32, dst0[3], val0);
2769 }
2770 break;
2771 case TGSI_OPCODE_SCS:
2772 if (mask & 3) {
2773 val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2774 if (dst0[0])
2775 mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2776 if (dst0[1])
2777 mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2778 }
2779 if (dst0[2])
2780 loadImm(dst0[2], 0.0f);
2781 if (dst0[3])
2782 loadImm(dst0[3], 1.0f);
2783 break;
2784 case TGSI_OPCODE_EXP:
2785 src0 = fetchSrc(0, 0);
2786 val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2787 if (dst0[1])
2788 mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2789 if (dst0[0])
2790 mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2791 if (dst0[2])
2792 mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2793 if (dst0[3])
2794 loadImm(dst0[3], 1.0f);
2795 break;
2796 case TGSI_OPCODE_LOG:
2797 src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2798 val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2799 if (dst0[0] || dst0[1])
2800 val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2801 if (dst0[1]) {
2802 mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2803 mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2804 mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2805 }
2806 if (dst0[3])
2807 loadImm(dst0[3], 1.0f);
2808 break;
2809 case TGSI_OPCODE_DP2:
2810 val0 = buildDot(2);
2811 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2812 mkMov(dst0[c], val0);
2813 break;
2814 case TGSI_OPCODE_DP3:
2815 val0 = buildDot(3);
2816 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2817 mkMov(dst0[c], val0);
2818 break;
2819 case TGSI_OPCODE_DP4:
2820 val0 = buildDot(4);
2821 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2822 mkMov(dst0[c], val0);
2823 break;
2824 case TGSI_OPCODE_DPH:
2825 val0 = buildDot(3);
2826 src1 = fetchSrc(1, 3);
2827 mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2828 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2829 mkMov(dst0[c], val0);
2830 break;
2831 case TGSI_OPCODE_DST:
2832 if (dst0[0])
2833 loadImm(dst0[0], 1.0f);
2834 if (dst0[1]) {
2835 src0 = fetchSrc(0, 1);
2836 src1 = fetchSrc(1, 1);
2837 mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2838 }
2839 if (dst0[2])
2840 mkMov(dst0[2], fetchSrc(0, 2));
2841 if (dst0[3])
2842 mkMov(dst0[3], fetchSrc(1, 3));
2843 break;
2844 case TGSI_OPCODE_LRP:
2845 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2846 src0 = fetchSrc(0, c);
2847 src1 = fetchSrc(1, c);
2848 src2 = fetchSrc(2, c);
2849 mkOp3(OP_MAD, TYPE_F32, dst0[c],
2850 mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2851 }
2852 break;
2853 case TGSI_OPCODE_LIT:
2854 handleLIT(dst0);
2855 break;
2856 case TGSI_OPCODE_XPD:
2857 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2858 if (c < 3) {
2859 val0 = getSSA();
2860 src0 = fetchSrc(1, (c + 1) % 3);
2861 src1 = fetchSrc(0, (c + 2) % 3);
2862 mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2863 mkOp1(OP_NEG, TYPE_F32, val0, val0);
2864
2865 src0 = fetchSrc(0, (c + 1) % 3);
2866 src1 = fetchSrc(1, (c + 2) % 3);
2867 mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2868 } else {
2869 loadImm(dst0[c], 1.0f);
2870 }
2871 }
2872 break;
2873 case TGSI_OPCODE_ISSG:
2874 case TGSI_OPCODE_SSG:
2875 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2876 src0 = fetchSrc(0, c);
2877 val0 = getScratch();
2878 val1 = getScratch();
2879 mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2880 mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2881 if (srcTy == TYPE_F32)
2882 mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2883 else
2884 mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2885 }
2886 break;
2887 case TGSI_OPCODE_UCMP:
2888 srcTy = TYPE_U32;
2889 /* fallthrough */
2890 case TGSI_OPCODE_CMP:
2891 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2892 src0 = fetchSrc(0, c);
2893 src1 = fetchSrc(1, c);
2894 src2 = fetchSrc(2, c);
2895 if (src1 == src2)
2896 mkMov(dst0[c], src1);
2897 else
2898 mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2899 srcTy, dst0[c], srcTy, src1, src2, src0);
2900 }
2901 break;
2902 case TGSI_OPCODE_FRC:
2903 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2904 src0 = fetchSrc(0, c);
2905 val0 = getScratch();
2906 mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2907 mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2908 }
2909 break;
2910 case TGSI_OPCODE_ROUND:
2911 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2912 mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2913 ->rnd = ROUND_NI;
2914 break;
2915 case TGSI_OPCODE_CLAMP:
2916 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2917 src0 = fetchSrc(0, c);
2918 src1 = fetchSrc(1, c);
2919 src2 = fetchSrc(2, c);
2920 val0 = getScratch();
2921 mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2922 mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2923 }
2924 break;
2925 case TGSI_OPCODE_SLT:
2926 case TGSI_OPCODE_SGE:
2927 case TGSI_OPCODE_SEQ:
2928 case TGSI_OPCODE_SGT:
2929 case TGSI_OPCODE_SLE:
2930 case TGSI_OPCODE_SNE:
2931 case TGSI_OPCODE_FSEQ:
2932 case TGSI_OPCODE_FSGE:
2933 case TGSI_OPCODE_FSLT:
2934 case TGSI_OPCODE_FSNE:
2935 case TGSI_OPCODE_ISGE:
2936 case TGSI_OPCODE_ISLT:
2937 case TGSI_OPCODE_USEQ:
2938 case TGSI_OPCODE_USGE:
2939 case TGSI_OPCODE_USLT:
2940 case TGSI_OPCODE_USNE:
2941 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2942 src0 = fetchSrc(0, c);
2943 src1 = fetchSrc(1, c);
2944 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2945 }
2946 break;
2947 case TGSI_OPCODE_KILL_IF:
2948 val0 = new_LValue(func, FILE_PREDICATE);
2949 mask = 0;
2950 for (c = 0; c < 4; ++c) {
2951 const int s = tgsi.getSrc(0).getSwizzle(c);
2952 if (mask & (1 << s))
2953 continue;
2954 mask |= 1 << s;
2955 mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2956 mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2957 }
2958 break;
2959 case TGSI_OPCODE_KILL:
2960 mkOp(OP_DISCARD, TYPE_NONE, NULL);
2961 break;
2962 case TGSI_OPCODE_TEX:
2963 case TGSI_OPCODE_TXB:
2964 case TGSI_OPCODE_TXL:
2965 case TGSI_OPCODE_TXP:
2966 case TGSI_OPCODE_LODQ:
2967 // R S L C Dx Dy
2968 handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2969 break;
2970 case TGSI_OPCODE_TXD:
2971 handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2972 break;
2973 case TGSI_OPCODE_TG4:
2974 handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2975 break;
2976 case TGSI_OPCODE_TEX2:
2977 handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2978 break;
2979 case TGSI_OPCODE_TXB2:
2980 case TGSI_OPCODE_TXL2:
2981 handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2982 break;
2983 case TGSI_OPCODE_SAMPLE:
2984 case TGSI_OPCODE_SAMPLE_B:
2985 case TGSI_OPCODE_SAMPLE_D:
2986 case TGSI_OPCODE_SAMPLE_L:
2987 case TGSI_OPCODE_SAMPLE_C:
2988 case TGSI_OPCODE_SAMPLE_C_LZ:
2989 handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2990 break;
2991 case TGSI_OPCODE_TXF:
2992 handleTXF(dst0, 1, 0x03);
2993 break;
2994 case TGSI_OPCODE_SAMPLE_I:
2995 handleTXF(dst0, 1, 0x03);
2996 break;
2997 case TGSI_OPCODE_SAMPLE_I_MS:
2998 handleTXF(dst0, 1, 0x20);
2999 break;
3000 case TGSI_OPCODE_TXQ:
3001 case TGSI_OPCODE_SVIEWINFO:
3002 handleTXQ(dst0, TXQ_DIMS, 1);
3003 break;
3004 case TGSI_OPCODE_TXQS:
3005 // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
3006 // be in .x
3007 dst0[1] = dst0[2] = dst0[3] = NULL;
3008 std::swap(dst0[0], dst0[2]);
3009 handleTXQ(dst0, TXQ_TYPE, 0);
3010 std::swap(dst0[0], dst0[2]);
3011 break;
3012 case TGSI_OPCODE_F2I:
3013 case TGSI_OPCODE_F2U:
3014 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3015 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
3016 break;
3017 case TGSI_OPCODE_I2F:
3018 case TGSI_OPCODE_U2F:
3019 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3020 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
3021 break;
3022 case TGSI_OPCODE_PK2H:
3023 val0 = getScratch();
3024 val1 = getScratch();
3025 mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
3026 mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
3027 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
3028 mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
3029 break;
3030 case TGSI_OPCODE_UP2H:
3031 src0 = fetchSrc(0, 0);
3032 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3033 geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3034 geni->subOp = c & 1;
3035 }
3036 break;
3037 case TGSI_OPCODE_EMIT:
3038 /* export the saved viewport index */
3039 if (viewport != NULL) {
3040 Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3041 info->out[info->io.viewportId].slot[0] * 4);
3042 mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3043 }
3044 /* fallthrough */
3045 case TGSI_OPCODE_ENDPRIM:
3046 {
3047 // get vertex stream (must be immediate)
3048 unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
3049 if (stream && op == OP_RESTART)
3050 break;
3051 src0 = mkImm(stream);
3052 mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3053 break;
3054 }
3055 case TGSI_OPCODE_IF:
3056 case TGSI_OPCODE_UIF:
3057 {
3058 BasicBlock *ifBB = new BasicBlock(func);
3059
3060 bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3061 condBBs.push(bb);
3062 joinBBs.push(bb);
3063
3064 mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3065
3066 setPosition(ifBB, true);
3067 }
3068 break;
3069 case TGSI_OPCODE_ELSE:
3070 {
3071 BasicBlock *elseBB = new BasicBlock(func);
3072 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3073
3074 forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3075 condBBs.push(bb);
3076
3077 forkBB->getExit()->asFlow()->target.bb = elseBB;
3078 if (!bb->isTerminated())
3079 mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3080
3081 setPosition(elseBB, true);
3082 }
3083 break;
3084 case TGSI_OPCODE_ENDIF:
3085 {
3086 BasicBlock *convBB = new BasicBlock(func);
3087 BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3088 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3089
3090 if (!bb->isTerminated()) {
3091 // we only want join if none of the clauses ended with CONT/BREAK/RET
3092 if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3093 insertConvergenceOps(convBB, forkBB);
3094 mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3095 bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3096 }
3097
3098 if (prevBB->getExit()->op == OP_BRA) {
3099 prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3100 prevBB->getExit()->asFlow()->target.bb = convBB;
3101 }
3102 setPosition(convBB, true);
3103 }
3104 break;
3105 case TGSI_OPCODE_BGNLOOP:
3106 {
3107 BasicBlock *lbgnBB = new BasicBlock(func);
3108 BasicBlock *lbrkBB = new BasicBlock(func);
3109
3110 loopBBs.push(lbgnBB);
3111 breakBBs.push(lbrkBB);
3112 if (loopBBs.getSize() > func->loopNestingBound)
3113 func->loopNestingBound++;
3114
3115 mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3116
3117 bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3118 setPosition(lbgnBB, true);
3119 mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3120 }
3121 break;
3122 case TGSI_OPCODE_ENDLOOP:
3123 {
3124 BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3125
3126 if (!bb->isTerminated()) {
3127 mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3128 bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3129 }
3130 setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3131
3132 // If the loop never breaks (e.g. only has RET's inside), then there
3133 // will be no way to get to the break bb. However BGNLOOP will have
3134 // already made a PREBREAK to it, so it must be in the CFG.
3135 if (getBB()->cfg.incidentCount() == 0)
3136 loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3137 }
3138 break;
3139 case TGSI_OPCODE_BRK:
3140 {
3141 if (bb->isTerminated())
3142 break;
3143 BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3144 mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3145 bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3146 }
3147 break;
3148 case TGSI_OPCODE_CONT:
3149 {
3150 if (bb->isTerminated())
3151 break;
3152 BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3153 mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3154 contBB->explicitCont = true;
3155 bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3156 }
3157 break;
3158 case TGSI_OPCODE_BGNSUB:
3159 {
3160 Subroutine *s = getSubroutine(ip);
3161 BasicBlock *entry = new BasicBlock(s->f);
3162 BasicBlock *leave = new BasicBlock(s->f);
3163
3164 // multiple entrypoints possible, keep the graph connected
3165 if (prog->getType() == Program::TYPE_COMPUTE)
3166 prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3167
3168 sub.cur = s;
3169 s->f->setEntry(entry);
3170 s->f->setExit(leave);
3171 setPosition(entry, true);
3172 return true;
3173 }
3174 case TGSI_OPCODE_ENDSUB:
3175 {
3176 sub.cur = getSubroutine(prog->main);
3177 setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3178 return true;
3179 }
3180 case TGSI_OPCODE_CAL:
3181 {
3182 Subroutine *s = getSubroutine(tgsi.getLabel());
3183 mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3184 func->call.attach(&s->f->call, Graph::Edge::TREE);
3185 return true;
3186 }
3187 case TGSI_OPCODE_RET:
3188 {
3189 if (bb->isTerminated())
3190 return true;
3191 BasicBlock *leave = BasicBlock::get(func->cfgExit);
3192
3193 if (!isEndOfSubroutine(ip + 1)) {
3194 // insert a PRERET at the entry if this is an early return
3195 // (only needed for sharing code in the epilogue)
3196 BasicBlock *pos = getBB();
3197 setPosition(BasicBlock::get(func->cfg.getRoot()), false);
3198 mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3199 setPosition(pos, true);
3200 }
3201 mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3202 bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3203 }
3204 break;
3205 case TGSI_OPCODE_END:
3206 {
3207 // attach and generate epilogue code
3208 BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3209 bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3210 setPosition(epilogue, true);
3211 if (prog->getType() == Program::TYPE_FRAGMENT)
3212 exportOutputs();
3213 if (info->io.genUserClip > 0)
3214 handleUserClipPlanes();
3215 mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3216 }
3217 break;
3218 case TGSI_OPCODE_SWITCH:
3219 case TGSI_OPCODE_CASE:
3220 ERROR("switch/case opcode encountered, should have been lowered\n");
3221 abort();
3222 break;
3223 case TGSI_OPCODE_LOAD:
3224 handleLOAD(dst0);
3225 break;
3226 case TGSI_OPCODE_STORE:
3227 handleSTORE();
3228 break;
3229 case TGSI_OPCODE_BARRIER:
3230 geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3231 geni->fixed = 1;
3232 geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3233 break;
3234 case TGSI_OPCODE_MFENCE:
3235 case TGSI_OPCODE_LFENCE:
3236 case TGSI_OPCODE_SFENCE:
3237 geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3238 geni->fixed = 1;
3239 geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3240 break;
3241 case TGSI_OPCODE_MEMBAR:
3242 geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3243 geni->fixed = 1;
3244 if (tgsi.getSrc(0).getValueU32(0, info) & TGSI_MEMBAR_THREAD_GROUP)
3245 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3246 else
3247 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3248 break;
3249 case TGSI_OPCODE_ATOMUADD:
3250 case TGSI_OPCODE_ATOMXCHG:
3251 case TGSI_OPCODE_ATOMCAS:
3252 case TGSI_OPCODE_ATOMAND:
3253 case TGSI_OPCODE_ATOMOR:
3254 case TGSI_OPCODE_ATOMXOR:
3255 case TGSI_OPCODE_ATOMUMIN:
3256 case TGSI_OPCODE_ATOMIMIN:
3257 case TGSI_OPCODE_ATOMUMAX:
3258 case TGSI_OPCODE_ATOMIMAX:
3259 handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3260 break;
3261 case TGSI_OPCODE_RESQ:
3262 geni = mkOp1(OP_SUQ, TYPE_U32, dst0[0],
3263 makeSym(TGSI_FILE_BUFFER, tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3264 if (tgsi.getSrc(0).isIndirect(0))
3265 geni->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
3266 break;
3267 case TGSI_OPCODE_IBFE:
3268 case TGSI_OPCODE_UBFE:
3269 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3270 src0 = fetchSrc(0, c);
3271 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3272 tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3273 src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3274 tgsi.getSrc(1).getValueU32(c, info));
3275 } else {
3276 src1 = fetchSrc(1, c);
3277 src2 = fetchSrc(2, c);
3278 mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3279 }
3280 mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3281 }
3282 break;
3283 case TGSI_OPCODE_BFI:
3284 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3285 src0 = fetchSrc(0, c);
3286 src1 = fetchSrc(1, c);
3287 src2 = fetchSrc(2, c);
3288 src3 = fetchSrc(3, c);
3289 mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3290 mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3291 }
3292 break;
3293 case TGSI_OPCODE_LSB:
3294 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3295 src0 = fetchSrc(0, c);
3296 geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3297 geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3298 geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3299 geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3300 }
3301 break;
3302 case TGSI_OPCODE_IMSB:
3303 case TGSI_OPCODE_UMSB:
3304 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3305 src0 = fetchSrc(0, c);
3306 mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3307 }
3308 break;
3309 case TGSI_OPCODE_BREV:
3310 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3311 src0 = fetchSrc(0, c);
3312 geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3313 geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3314 }
3315 break;
3316 case TGSI_OPCODE_POPC:
3317 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3318 src0 = fetchSrc(0, c);
3319 mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3320 }
3321 break;
3322 case TGSI_OPCODE_INTERP_CENTROID:
3323 case TGSI_OPCODE_INTERP_SAMPLE:
3324 case TGSI_OPCODE_INTERP_OFFSET:
3325 handleINTERP(dst0);
3326 break;
3327 case TGSI_OPCODE_D2I:
3328 case TGSI_OPCODE_D2U:
3329 case TGSI_OPCODE_D2F: {
3330 int pos = 0;
3331 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3332 Value *dreg = getSSA(8);
3333 src0 = fetchSrc(0, pos);
3334 src1 = fetchSrc(0, pos + 1);
3335 mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3336 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3337 pos += 2;
3338 }
3339 break;
3340 }
3341 case TGSI_OPCODE_I2D:
3342 case TGSI_OPCODE_U2D:
3343 case TGSI_OPCODE_F2D:
3344 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3345 Value *dreg = getSSA(8);
3346 mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3347 mkSplit(&dst0[c], 4, dreg);
3348 c++;
3349 }
3350 break;
3351 case TGSI_OPCODE_DABS:
3352 case TGSI_OPCODE_DNEG:
3353 case TGSI_OPCODE_DRCP:
3354 case TGSI_OPCODE_DSQRT:
3355 case TGSI_OPCODE_DRSQ:
3356 case TGSI_OPCODE_DTRUNC:
3357 case TGSI_OPCODE_DCEIL:
3358 case TGSI_OPCODE_DFLR:
3359 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3360 src0 = getSSA(8);
3361 Value *dst = getSSA(8), *tmp[2];
3362 tmp[0] = fetchSrc(0, c);
3363 tmp[1] = fetchSrc(0, c + 1);
3364 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3365 mkOp1(op, dstTy, dst, src0);
3366 mkSplit(&dst0[c], 4, dst);
3367 c++;
3368 }
3369 break;
3370 case TGSI_OPCODE_DFRAC:
3371 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3372 src0 = getSSA(8);
3373 Value *dst = getSSA(8), *tmp[2];
3374 tmp[0] = fetchSrc(0, c);
3375 tmp[1] = fetchSrc(0, c + 1);
3376 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3377 mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3378 mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3379 mkSplit(&dst0[c], 4, dst);
3380 c++;
3381 }
3382 break;
3383 case TGSI_OPCODE_DSLT:
3384 case TGSI_OPCODE_DSGE:
3385 case TGSI_OPCODE_DSEQ:
3386 case TGSI_OPCODE_DSNE: {
3387 int pos = 0;
3388 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3389 Value *tmp[2];
3390
3391 src0 = getSSA(8);
3392 src1 = getSSA(8);
3393 tmp[0] = fetchSrc(0, pos);
3394 tmp[1] = fetchSrc(0, pos + 1);
3395 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3396 tmp[0] = fetchSrc(1, pos);
3397 tmp[1] = fetchSrc(1, pos + 1);
3398 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3399 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3400 pos += 2;
3401 }
3402 break;
3403 }
3404 case TGSI_OPCODE_DADD:
3405 case TGSI_OPCODE_DMUL:
3406 case TGSI_OPCODE_DMAX:
3407 case TGSI_OPCODE_DMIN:
3408 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3409 src0 = getSSA(8);
3410 src1 = getSSA(8);
3411 Value *dst = getSSA(8), *tmp[2];
3412 tmp[0] = fetchSrc(0, c);
3413 tmp[1] = fetchSrc(0, c + 1);
3414 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3415 tmp[0] = fetchSrc(1, c);
3416 tmp[1] = fetchSrc(1, c + 1);
3417 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3418 mkOp2(op, dstTy, dst, src0, src1);
3419 mkSplit(&dst0[c], 4, dst);
3420 c++;
3421 }
3422 break;
3423 case TGSI_OPCODE_DMAD:
3424 case TGSI_OPCODE_DFMA:
3425 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3426 src0 = getSSA(8);
3427 src1 = getSSA(8);
3428 src2 = getSSA(8);
3429 Value *dst = getSSA(8), *tmp[2];
3430 tmp[0] = fetchSrc(0, c);
3431 tmp[1] = fetchSrc(0, c + 1);
3432 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3433 tmp[0] = fetchSrc(1, c);
3434 tmp[1] = fetchSrc(1, c + 1);
3435 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3436 tmp[0] = fetchSrc(2, c);
3437 tmp[1] = fetchSrc(2, c + 1);
3438 mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3439 mkOp3(op, dstTy, dst, src0, src1, src2);
3440 mkSplit(&dst0[c], 4, dst);
3441 c++;
3442 }
3443 break;
3444 case TGSI_OPCODE_DROUND:
3445 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3446 src0 = getSSA(8);
3447 Value *dst = getSSA(8), *tmp[2];
3448 tmp[0] = fetchSrc(0, c);
3449 tmp[1] = fetchSrc(0, c + 1);
3450 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3451 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3452 ->rnd = ROUND_NI;
3453 mkSplit(&dst0[c], 4, dst);
3454 c++;
3455 }
3456 break;
3457 case TGSI_OPCODE_DSSG:
3458 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3459 src0 = getSSA(8);
3460 Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3461 tmp[0] = fetchSrc(0, c);
3462 tmp[1] = fetchSrc(0, c + 1);
3463 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3464
3465 val0 = getScratch();
3466 val1 = getScratch();
3467 // The zero is wrong here since it's only 32-bit, but it works out in
3468 // the end since it gets replaced with $r63.
3469 mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3470 mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3471 mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3472 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3473 mkSplit(&dst0[c], 4, dst);
3474 c++;
3475 }
3476 break;
3477 default:
3478 ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3479 assert(0);
3480 break;
3481 }
3482
3483 if (tgsi.dstCount()) {
3484 for (c = 0; c < 4; ++c) {
3485 if (!dst0[c])
3486 continue;
3487 if (dst0[c] != rDst0[c])
3488 mkMov(rDst0[c], dst0[c]);
3489 storeDst(0, c, rDst0[c]);
3490 }
3491 }
3492 vtxBaseValid = 0;
3493
3494 return true;
3495 }
3496
3497 void
3498 Converter::handleUserClipPlanes()
3499 {
3500 Value *res[8];
3501 int n, i, c;
3502
3503 for (c = 0; c < 4; ++c) {
3504 for (i = 0; i < info->io.genUserClip; ++i) {
3505 Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
3506 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3507 Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3508 if (c == 0)
3509 res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3510 else
3511 mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3512 }
3513 }
3514
3515 const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3516
3517 for (i = 0; i < info->io.genUserClip; ++i) {
3518 n = i / 4 + first;
3519 c = i % 4;
3520 Symbol *sym =
3521 mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3522 mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3523 }
3524 }
3525
3526 void
3527 Converter::exportOutputs()
3528 {
3529 for (unsigned int i = 0; i < info->numOutputs; ++i) {
3530 for (unsigned int c = 0; c < 4; ++c) {
3531 if (!oData.exists(sub.cur->values, i, c))
3532 continue;
3533 Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3534 info->out[i].slot[c] * 4);
3535 Value *val = oData.load(sub.cur->values, i, c, NULL);
3536 if (val)
3537 mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3538 }
3539 }
3540 }
3541
3542 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3543 code(code),
3544 tgsi(NULL),
3545 tData(this), lData(this), aData(this), pData(this), oData(this)
3546 {
3547 info = code->info;
3548
3549 const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3550 const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3551 const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3552 const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3553
3554 tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
3555 lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
3556 pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3557 aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3558 oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3559
3560 zero = mkImm((uint32_t)0);
3561
3562 vtxBaseValid = 0;
3563 }
3564
3565 Converter::~Converter()
3566 {
3567 }
3568
3569 inline const Converter::Location *
3570 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3571 {
3572 ValueMap::l_iterator it = s->values.l.find(v);
3573 return it == s->values.l.end() ? NULL : &it->second;
3574 }
3575
3576 template<typename T> inline void
3577 Converter::BindArgumentsPass::updateCallArgs(
3578 Instruction *i, void (Instruction::*setArg)(int, Value *),
3579 T (Function::*proto))
3580 {
3581 Function *g = i->asFlow()->target.fn;
3582 Subroutine *subg = conv.getSubroutine(g);
3583
3584 for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3585 Value *v = (g->*proto)[a].get();
3586 const Converter::Location &l = *getValueLocation(subg, v);
3587 Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3588
3589 (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3590 }
3591 }
3592
3593 template<typename T> inline void
3594 Converter::BindArgumentsPass::updatePrototype(
3595 BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3596 {
3597 (func->*updateSet)();
3598
3599 for (unsigned i = 0; i < set->getSize(); ++i) {
3600 Value *v = func->getLValue(i);
3601 const Converter::Location *l = getValueLocation(sub, v);
3602
3603 // only include values with a matching TGSI register
3604 if (set->test(i) && l && !conv.code->locals.count(*l))
3605 (func->*proto).push_back(v);
3606 }
3607 }
3608
3609 bool
3610 Converter::BindArgumentsPass::visit(Function *f)
3611 {
3612 sub = conv.getSubroutine(f);
3613
3614 for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3615 !bi.end(); bi.next()) {
3616 for (Instruction *i = BasicBlock::get(bi)->getFirst();
3617 i; i = i->next) {
3618 if (i->op == OP_CALL && !i->asFlow()->builtin) {
3619 updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3620 updateCallArgs(i, &Instruction::setDef, &Function::outs);
3621 }
3622 }
3623 }
3624
3625 if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3626 return true;
3627 updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3628 &Function::buildLiveSets, &Function::ins);
3629 updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3630 &Function::buildDefSets, &Function::outs);
3631
3632 return true;
3633 }
3634
3635 bool
3636 Converter::run()
3637 {
3638 BasicBlock *entry = new BasicBlock(prog->main);
3639 BasicBlock *leave = new BasicBlock(prog->main);
3640
3641 prog->main->setEntry(entry);
3642 prog->main->setExit(leave);
3643
3644 setPosition(entry, true);
3645 sub.cur = getSubroutine(prog->main);
3646
3647 if (info->io.genUserClip > 0) {
3648 for (int c = 0; c < 4; ++c)
3649 clipVtx[c] = getScratch();
3650 }
3651
3652 switch (prog->getType()) {
3653 case Program::TYPE_TESSELLATION_CONTROL:
3654 outBase = mkOp2v(
3655 OP_SUB, TYPE_U32, getSSA(),
3656 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3657 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3658 break;
3659 case Program::TYPE_FRAGMENT: {
3660 Symbol *sv = mkSysVal(SV_POSITION, 3);
3661 fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3662 mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3663 break;
3664 }
3665 default:
3666 break;
3667 }
3668
3669 if (info->io.viewportId >= 0)
3670 viewport = getScratch();
3671 else
3672 viewport = NULL;
3673
3674 for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3675 if (!handleInstruction(&code->insns[ip]))
3676 return false;
3677 }
3678
3679 if (!BindArgumentsPass(*this).run(prog))
3680 return false;
3681
3682 return true;
3683 }
3684
3685 } // unnamed namespace
3686
3687 namespace nv50_ir {
3688
3689 bool
3690 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3691 {
3692 tgsi::Source src(info);
3693 if (!src.scanSource())
3694 return false;
3695 tlsSize = info->bin.tlsSpace;
3696
3697 Converter builder(this, &src);
3698 return builder.run();
3699 }
3700
3701 } // namespace nv50_ir