projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
tgsi: Add support for global / private / input MEMORY
[mesa.git] / src / gallium / drivers / nouveau / codegen / nv50_ir_from_tgsi.cpp
1 /*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include "tgsi/tgsi_dump.h"
24 #include "tgsi/tgsi_scan.h"
25 #include "tgsi/tgsi_util.h"
26
27 #include <set>
28
29 #include "codegen/nv50_ir.h"
30 #include "codegen/nv50_ir_util.h"
31 #include "codegen/nv50_ir_build_util.h"
32
33 namespace tgsi {
34
35 class Source;
36
37 static nv50_ir::operation translateOpcode(uint opcode);
38 static nv50_ir::DataFile translateFile(uint file);
39 static nv50_ir::TexTarget translateTexture(uint texTarg);
40 static nv50_ir::SVSemantic translateSysVal(uint sysval);
41 static nv50_ir::CacheMode translateCacheMode(uint qualifier);
42
43 class Instruction
44 {
45 public:
46 Instruction(const struct tgsi_full_instruction *inst) : insn(inst) { }
47
48 class SrcRegister
49 {
50 public:
51 SrcRegister(const struct tgsi_full_src_register *src)
52 : reg(src->Register),
53 fsr(src)
54 { }
55
56 SrcRegister(const struct tgsi_src_register& src) : reg(src), fsr(NULL) { }
57
58 SrcRegister(const struct tgsi_ind_register& ind)
59 : reg(tgsi_util_get_src_from_ind(&ind)),
60 fsr(NULL)
61 { }
62
63 struct tgsi_src_register offsetToSrc(struct tgsi_texture_offset off)
64 {
65 struct tgsi_src_register reg;
66 memset(&reg, 0, sizeof(reg));
67 reg.Index = off.Index;
68 reg.File = off.File;
69 reg.SwizzleX = off.SwizzleX;
70 reg.SwizzleY = off.SwizzleY;
71 reg.SwizzleZ = off.SwizzleZ;
72 return reg;
73 }
74
75 SrcRegister(const struct tgsi_texture_offset& off) :
76 reg(offsetToSrc(off)),
77 fsr(NULL)
78 { }
79
80 uint getFile() const { return reg.File; }
81
82 bool is2D() const { return reg.Dimension; }
83
84 bool isIndirect(int dim) const
85 {
86 return (dim && fsr) ? fsr->Dimension.Indirect : reg.Indirect;
87 }
88
89 int getIndex(int dim) const
90 {
91 return (dim && fsr) ? fsr->Dimension.Index : reg.Index;
92 }
93
94 int getSwizzle(int chan) const
95 {
96 return tgsi_util_get_src_register_swizzle(&reg, chan);
97 }
98
99 int getArrayId() const
100 {
101 if (isIndirect(0))
102 return fsr->Indirect.ArrayID;
103 return 0;
104 }
105
106 nv50_ir::Modifier getMod(int chan) const;
107
108 SrcRegister getIndirect(int dim) const
109 {
110 assert(fsr && isIndirect(dim));
111 if (dim)
112 return SrcRegister(fsr->DimIndirect);
113 return SrcRegister(fsr->Indirect);
114 }
115
116 uint32_t getValueU32(int c, const struct nv50_ir_prog_info *info) const
117 {
118 assert(reg.File == TGSI_FILE_IMMEDIATE);
119 assert(!reg.Absolute);
120 assert(!reg.Negate);
121 return info->immd.data[reg.Index * 4 + getSwizzle(c)];
122 }
123
124 private:
125 const struct tgsi_src_register reg;
126 const struct tgsi_full_src_register *fsr;
127 };
128
129 class DstRegister
130 {
131 public:
132 DstRegister(const struct tgsi_full_dst_register *dst)
133 : reg(dst->Register),
134 fdr(dst)
135 { }
136
137 DstRegister(const struct tgsi_dst_register& dst) : reg(dst), fdr(NULL) { }
138
139 uint getFile() const { return reg.File; }
140
141 bool is2D() const { return reg.Dimension; }
142
143 bool isIndirect(int dim) const
144 {
145 return (dim && fdr) ? fdr->Dimension.Indirect : reg.Indirect;
146 }
147
148 int getIndex(int dim) const
149 {
150 return (dim && fdr) ? fdr->Dimension.Dimension : reg.Index;
151 }
152
153 unsigned int getMask() const { return reg.WriteMask; }
154
155 bool isMasked(int chan) const { return !(getMask() & (1 << chan)); }
156
157 SrcRegister getIndirect(int dim) const
158 {
159 assert(fdr && isIndirect(dim));
160 if (dim)
161 return SrcRegister(fdr->DimIndirect);
162 return SrcRegister(fdr->Indirect);
163 }
164
165 int getArrayId() const
166 {
167 if (isIndirect(0))
168 return fdr->Indirect.ArrayID;
169 return 0;
170 }
171
172 private:
173 const struct tgsi_dst_register reg;
174 const struct tgsi_full_dst_register *fdr;
175 };
176
177 inline uint getOpcode() const { return insn->Instruction.Opcode; }
178
179 unsigned int srcCount() const { return insn->Instruction.NumSrcRegs; }
180 unsigned int dstCount() const { return insn->Instruction.NumDstRegs; }
181
182 // mask of used components of source s
183 unsigned int srcMask(unsigned int s) const;
184
185 SrcRegister getSrc(unsigned int s) const
186 {
187 assert(s < srcCount());
188 return SrcRegister(&insn->Src[s]);
189 }
190
191 DstRegister getDst(unsigned int d) const
192 {
193 assert(d < dstCount());
194 return DstRegister(&insn->Dst[d]);
195 }
196
197 SrcRegister getTexOffset(unsigned int i) const
198 {
199 assert(i < TGSI_FULL_MAX_TEX_OFFSETS);
200 return SrcRegister(insn->TexOffsets[i]);
201 }
202
203 unsigned int getNumTexOffsets() const { return insn->Texture.NumOffsets; }
204
205 bool checkDstSrcAliasing() const;
206
207 inline nv50_ir::operation getOP() const {
208 return translateOpcode(getOpcode()); }
209
210 nv50_ir::DataType inferSrcType() const;
211 nv50_ir::DataType inferDstType() const;
212
213 nv50_ir::CondCode getSetCond() const;
214
215 nv50_ir::TexInstruction::Target getTexture(const Source *, int s) const;
216
217 nv50_ir::CacheMode getCacheMode() const {
218 if (!insn->Instruction.Memory)
219 return nv50_ir::CACHE_CA;
220 return translateCacheMode(insn->Memory.Qualifier);
221 }
222
223 inline uint getLabel() { return insn->Label.Label; }
224
225 unsigned getSaturate() const { return insn->Instruction.Saturate; }
226
227 void print() const
228 {
229 tgsi_dump_instruction(insn, 1);
230 }
231
232 private:
233 const struct tgsi_full_instruction *insn;
234 };
235
236 unsigned int Instruction::srcMask(unsigned int s) const
237 {
238 unsigned int mask = insn->Dst[0].Register.WriteMask;
239
240 switch (insn->Instruction.Opcode) {
241 case TGSI_OPCODE_COS:
242 case TGSI_OPCODE_SIN:
243 return (mask & 0x8) | ((mask & 0x7) ? 0x1 : 0x0);
244 case TGSI_OPCODE_DP2:
245 return 0x3;
246 case TGSI_OPCODE_DP3:
247 return 0x7;
248 case TGSI_OPCODE_DP4:
249 case TGSI_OPCODE_DPH:
250 case TGSI_OPCODE_KILL_IF: /* WriteMask ignored */
251 return 0xf;
252 case TGSI_OPCODE_DST:
253 return mask & (s ? 0xa : 0x6);
254 case TGSI_OPCODE_EX2:
255 case TGSI_OPCODE_EXP:
256 case TGSI_OPCODE_LG2:
257 case TGSI_OPCODE_LOG:
258 case TGSI_OPCODE_POW:
259 case TGSI_OPCODE_RCP:
260 case TGSI_OPCODE_RSQ:
261 case TGSI_OPCODE_SCS:
262 return 0x1;
263 case TGSI_OPCODE_IF:
264 case TGSI_OPCODE_UIF:
265 return 0x1;
266 case TGSI_OPCODE_LIT:
267 return 0xb;
268 case TGSI_OPCODE_TEX2:
269 case TGSI_OPCODE_TXB2:
270 case TGSI_OPCODE_TXL2:
271 return (s == 0) ? 0xf : 0x3;
272 case TGSI_OPCODE_TEX:
273 case TGSI_OPCODE_TXB:
274 case TGSI_OPCODE_TXD:
275 case TGSI_OPCODE_TXL:
276 case TGSI_OPCODE_TXP:
277 case TGSI_OPCODE_LODQ:
278 {
279 const struct tgsi_instruction_texture *tex = &insn->Texture;
280
281 assert(insn->Instruction.Texture);
282
283 mask = 0x7;
284 if (insn->Instruction.Opcode != TGSI_OPCODE_TEX &&
285 insn->Instruction.Opcode != TGSI_OPCODE_TXD)
286 mask |= 0x8; /* bias, lod or proj */
287
288 switch (tex->Texture) {
289 case TGSI_TEXTURE_1D:
290 mask &= 0x9;
291 break;
292 case TGSI_TEXTURE_SHADOW1D:
293 mask &= 0xd;
294 break;
295 case TGSI_TEXTURE_1D_ARRAY:
296 case TGSI_TEXTURE_2D:
297 case TGSI_TEXTURE_RECT:
298 mask &= 0xb;
299 break;
300 case TGSI_TEXTURE_CUBE_ARRAY:
301 case TGSI_TEXTURE_SHADOW2D_ARRAY:
302 case TGSI_TEXTURE_SHADOWCUBE:
303 case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
304 mask |= 0x8;
305 break;
306 default:
307 break;
308 }
309 }
310 return mask;
311 case TGSI_OPCODE_XPD:
312 {
313 unsigned int x = 0;
314 if (mask & 1) x |= 0x6;
315 if (mask & 2) x |= 0x5;
316 if (mask & 4) x |= 0x3;
317 return x;
318 }
319 case TGSI_OPCODE_D2I:
320 case TGSI_OPCODE_D2U:
321 case TGSI_OPCODE_D2F:
322 case TGSI_OPCODE_DSLT:
323 case TGSI_OPCODE_DSGE:
324 case TGSI_OPCODE_DSEQ:
325 case TGSI_OPCODE_DSNE:
326 switch (util_bitcount(mask)) {
327 case 1: return 0x3;
328 case 2: return 0xf;
329 default:
330 assert(!"unexpected mask");
331 return 0xf;
332 }
333 case TGSI_OPCODE_I2D:
334 case TGSI_OPCODE_U2D:
335 case TGSI_OPCODE_F2D: {
336 unsigned int x = 0;
337 if ((mask & 0x3) == 0x3)
338 x |= 1;
339 if ((mask & 0xc) == 0xc)
340 x |= 2;
341 return x;
342 }
343 case TGSI_OPCODE_PK2H:
344 return 0x3;
345 case TGSI_OPCODE_UP2H:
346 return 0x1;
347 default:
348 break;
349 }
350
351 return mask;
352 }
353
354 nv50_ir::Modifier Instruction::SrcRegister::getMod(int chan) const
355 {
356 nv50_ir::Modifier m(0);
357
358 if (reg.Absolute)
359 m = m | nv50_ir::Modifier(NV50_IR_MOD_ABS);
360 if (reg.Negate)
361 m = m | nv50_ir::Modifier(NV50_IR_MOD_NEG);
362 return m;
363 }
364
365 static nv50_ir::DataFile translateFile(uint file)
366 {
367 switch (file) {
368 case TGSI_FILE_CONSTANT: return nv50_ir::FILE_MEMORY_CONST;
369 case TGSI_FILE_INPUT: return nv50_ir::FILE_SHADER_INPUT;
370 case TGSI_FILE_OUTPUT: return nv50_ir::FILE_SHADER_OUTPUT;
371 case TGSI_FILE_TEMPORARY: return nv50_ir::FILE_GPR;
372 case TGSI_FILE_ADDRESS: return nv50_ir::FILE_ADDRESS;
373 case TGSI_FILE_PREDICATE: return nv50_ir::FILE_PREDICATE;
374 case TGSI_FILE_IMMEDIATE: return nv50_ir::FILE_IMMEDIATE;
375 case TGSI_FILE_SYSTEM_VALUE: return nv50_ir::FILE_SYSTEM_VALUE;
376 case TGSI_FILE_BUFFER: return nv50_ir::FILE_MEMORY_GLOBAL;
377 case TGSI_FILE_MEMORY: return nv50_ir::FILE_MEMORY_GLOBAL;
378 case TGSI_FILE_SAMPLER:
379 case TGSI_FILE_NULL:
380 default:
381 return nv50_ir::FILE_NULL;
382 }
383 }
384
385 static nv50_ir::SVSemantic translateSysVal(uint sysval)
386 {
387 switch (sysval) {
388 case TGSI_SEMANTIC_FACE: return nv50_ir::SV_FACE;
389 case TGSI_SEMANTIC_PSIZE: return nv50_ir::SV_POINT_SIZE;
390 case TGSI_SEMANTIC_PRIMID: return nv50_ir::SV_PRIMITIVE_ID;
391 case TGSI_SEMANTIC_INSTANCEID: return nv50_ir::SV_INSTANCE_ID;
392 case TGSI_SEMANTIC_VERTEXID: return nv50_ir::SV_VERTEX_ID;
393 case TGSI_SEMANTIC_GRID_SIZE: return nv50_ir::SV_NCTAID;
394 case TGSI_SEMANTIC_BLOCK_ID: return nv50_ir::SV_CTAID;
395 case TGSI_SEMANTIC_BLOCK_SIZE: return nv50_ir::SV_NTID;
396 case TGSI_SEMANTIC_THREAD_ID: return nv50_ir::SV_TID;
397 case TGSI_SEMANTIC_SAMPLEID: return nv50_ir::SV_SAMPLE_INDEX;
398 case TGSI_SEMANTIC_SAMPLEPOS: return nv50_ir::SV_SAMPLE_POS;
399 case TGSI_SEMANTIC_SAMPLEMASK: return nv50_ir::SV_SAMPLE_MASK;
400 case TGSI_SEMANTIC_INVOCATIONID: return nv50_ir::SV_INVOCATION_ID;
401 case TGSI_SEMANTIC_TESSCOORD: return nv50_ir::SV_TESS_COORD;
402 case TGSI_SEMANTIC_TESSOUTER: return nv50_ir::SV_TESS_OUTER;
403 case TGSI_SEMANTIC_TESSINNER: return nv50_ir::SV_TESS_INNER;
404 case TGSI_SEMANTIC_VERTICESIN: return nv50_ir::SV_VERTEX_COUNT;
405 case TGSI_SEMANTIC_HELPER_INVOCATION: return nv50_ir::SV_THREAD_KILL;
406 case TGSI_SEMANTIC_BASEVERTEX: return nv50_ir::SV_BASEVERTEX;
407 case TGSI_SEMANTIC_BASEINSTANCE: return nv50_ir::SV_BASEINSTANCE;
408 case TGSI_SEMANTIC_DRAWID: return nv50_ir::SV_DRAWID;
409 default:
410 assert(0);
411 return nv50_ir::SV_CLOCK;
412 }
413 }
414
415 #define NV50_IR_TEX_TARG_CASE(a, b) \
416 case TGSI_TEXTURE_##a: return nv50_ir::TEX_TARGET_##b;
417
418 static nv50_ir::TexTarget translateTexture(uint tex)
419 {
420 switch (tex) {
421 NV50_IR_TEX_TARG_CASE(1D, 1D);
422 NV50_IR_TEX_TARG_CASE(2D, 2D);
423 NV50_IR_TEX_TARG_CASE(2D_MSAA, 2D_MS);
424 NV50_IR_TEX_TARG_CASE(3D, 3D);
425 NV50_IR_TEX_TARG_CASE(CUBE, CUBE);
426 NV50_IR_TEX_TARG_CASE(RECT, RECT);
427 NV50_IR_TEX_TARG_CASE(1D_ARRAY, 1D_ARRAY);
428 NV50_IR_TEX_TARG_CASE(2D_ARRAY, 2D_ARRAY);
429 NV50_IR_TEX_TARG_CASE(2D_ARRAY_MSAA, 2D_MS_ARRAY);
430 NV50_IR_TEX_TARG_CASE(CUBE_ARRAY, CUBE_ARRAY);
431 NV50_IR_TEX_TARG_CASE(SHADOW1D, 1D_SHADOW);
432 NV50_IR_TEX_TARG_CASE(SHADOW2D, 2D_SHADOW);
433 NV50_IR_TEX_TARG_CASE(SHADOWCUBE, CUBE_SHADOW);
434 NV50_IR_TEX_TARG_CASE(SHADOWRECT, RECT_SHADOW);
435 NV50_IR_TEX_TARG_CASE(SHADOW1D_ARRAY, 1D_ARRAY_SHADOW);
436 NV50_IR_TEX_TARG_CASE(SHADOW2D_ARRAY, 2D_ARRAY_SHADOW);
437 NV50_IR_TEX_TARG_CASE(SHADOWCUBE_ARRAY, CUBE_ARRAY_SHADOW);
438 NV50_IR_TEX_TARG_CASE(BUFFER, BUFFER);
439
440 case TGSI_TEXTURE_UNKNOWN:
441 default:
442 assert(!"invalid texture target");
443 return nv50_ir::TEX_TARGET_2D;
444 }
445 }
446
447 static nv50_ir::CacheMode translateCacheMode(uint qualifier)
448 {
449 if (qualifier & TGSI_MEMORY_VOLATILE)
450 return nv50_ir::CACHE_CV;
451 if (qualifier & TGSI_MEMORY_COHERENT)
452 return nv50_ir::CACHE_CG;
453 return nv50_ir::CACHE_CA;
454 }
455
456 nv50_ir::DataType Instruction::inferSrcType() const
457 {
458 switch (getOpcode()) {
459 case TGSI_OPCODE_UIF:
460 case TGSI_OPCODE_AND:
461 case TGSI_OPCODE_OR:
462 case TGSI_OPCODE_XOR:
463 case TGSI_OPCODE_NOT:
464 case TGSI_OPCODE_SHL:
465 case TGSI_OPCODE_U2F:
466 case TGSI_OPCODE_U2D:
467 case TGSI_OPCODE_UADD:
468 case TGSI_OPCODE_UDIV:
469 case TGSI_OPCODE_UMOD:
470 case TGSI_OPCODE_UMAD:
471 case TGSI_OPCODE_UMUL:
472 case TGSI_OPCODE_UMUL_HI:
473 case TGSI_OPCODE_UMAX:
474 case TGSI_OPCODE_UMIN:
475 case TGSI_OPCODE_USEQ:
476 case TGSI_OPCODE_USGE:
477 case TGSI_OPCODE_USLT:
478 case TGSI_OPCODE_USNE:
479 case TGSI_OPCODE_USHR:
480 case TGSI_OPCODE_ATOMUADD:
481 case TGSI_OPCODE_ATOMXCHG:
482 case TGSI_OPCODE_ATOMCAS:
483 case TGSI_OPCODE_ATOMAND:
484 case TGSI_OPCODE_ATOMOR:
485 case TGSI_OPCODE_ATOMXOR:
486 case TGSI_OPCODE_ATOMUMIN:
487 case TGSI_OPCODE_ATOMUMAX:
488 case TGSI_OPCODE_UBFE:
489 case TGSI_OPCODE_UMSB:
490 case TGSI_OPCODE_UP2H:
491 return nv50_ir::TYPE_U32;
492 case TGSI_OPCODE_I2F:
493 case TGSI_OPCODE_I2D:
494 case TGSI_OPCODE_IDIV:
495 case TGSI_OPCODE_IMUL_HI:
496 case TGSI_OPCODE_IMAX:
497 case TGSI_OPCODE_IMIN:
498 case TGSI_OPCODE_IABS:
499 case TGSI_OPCODE_INEG:
500 case TGSI_OPCODE_ISGE:
501 case TGSI_OPCODE_ISHR:
502 case TGSI_OPCODE_ISLT:
503 case TGSI_OPCODE_ISSG:
504 case TGSI_OPCODE_SAD: // not sure about SAD, but no one has a float version
505 case TGSI_OPCODE_MOD:
506 case TGSI_OPCODE_UARL:
507 case TGSI_OPCODE_ATOMIMIN:
508 case TGSI_OPCODE_ATOMIMAX:
509 case TGSI_OPCODE_IBFE:
510 case TGSI_OPCODE_IMSB:
511 return nv50_ir::TYPE_S32;
512 case TGSI_OPCODE_D2F:
513 case TGSI_OPCODE_D2I:
514 case TGSI_OPCODE_D2U:
515 case TGSI_OPCODE_DABS:
516 case TGSI_OPCODE_DNEG:
517 case TGSI_OPCODE_DADD:
518 case TGSI_OPCODE_DMUL:
519 case TGSI_OPCODE_DMAX:
520 case TGSI_OPCODE_DMIN:
521 case TGSI_OPCODE_DSLT:
522 case TGSI_OPCODE_DSGE:
523 case TGSI_OPCODE_DSEQ:
524 case TGSI_OPCODE_DSNE:
525 case TGSI_OPCODE_DRCP:
526 case TGSI_OPCODE_DSQRT:
527 case TGSI_OPCODE_DMAD:
528 case TGSI_OPCODE_DFMA:
529 case TGSI_OPCODE_DFRAC:
530 case TGSI_OPCODE_DRSQ:
531 case TGSI_OPCODE_DTRUNC:
532 case TGSI_OPCODE_DCEIL:
533 case TGSI_OPCODE_DFLR:
534 case TGSI_OPCODE_DROUND:
535 return nv50_ir::TYPE_F64;
536 default:
537 return nv50_ir::TYPE_F32;
538 }
539 }
540
541 nv50_ir::DataType Instruction::inferDstType() const
542 {
543 switch (getOpcode()) {
544 case TGSI_OPCODE_D2U:
545 case TGSI_OPCODE_F2U: return nv50_ir::TYPE_U32;
546 case TGSI_OPCODE_D2I:
547 case TGSI_OPCODE_F2I: return nv50_ir::TYPE_S32;
548 case TGSI_OPCODE_FSEQ:
549 case TGSI_OPCODE_FSGE:
550 case TGSI_OPCODE_FSLT:
551 case TGSI_OPCODE_FSNE:
552 case TGSI_OPCODE_DSEQ:
553 case TGSI_OPCODE_DSGE:
554 case TGSI_OPCODE_DSLT:
555 case TGSI_OPCODE_DSNE:
556 case TGSI_OPCODE_PK2H:
557 return nv50_ir::TYPE_U32;
558 case TGSI_OPCODE_I2F:
559 case TGSI_OPCODE_U2F:
560 case TGSI_OPCODE_D2F:
561 case TGSI_OPCODE_UP2H:
562 return nv50_ir::TYPE_F32;
563 case TGSI_OPCODE_I2D:
564 case TGSI_OPCODE_U2D:
565 case TGSI_OPCODE_F2D:
566 return nv50_ir::TYPE_F64;
567 default:
568 return inferSrcType();
569 }
570 }
571
572 nv50_ir::CondCode Instruction::getSetCond() const
573 {
574 using namespace nv50_ir;
575
576 switch (getOpcode()) {
577 case TGSI_OPCODE_SLT:
578 case TGSI_OPCODE_ISLT:
579 case TGSI_OPCODE_USLT:
580 case TGSI_OPCODE_FSLT:
581 case TGSI_OPCODE_DSLT:
582 return CC_LT;
583 case TGSI_OPCODE_SLE:
584 return CC_LE;
585 case TGSI_OPCODE_SGE:
586 case TGSI_OPCODE_ISGE:
587 case TGSI_OPCODE_USGE:
588 case TGSI_OPCODE_FSGE:
589 case TGSI_OPCODE_DSGE:
590 return CC_GE;
591 case TGSI_OPCODE_SGT:
592 return CC_GT;
593 case TGSI_OPCODE_SEQ:
594 case TGSI_OPCODE_USEQ:
595 case TGSI_OPCODE_FSEQ:
596 case TGSI_OPCODE_DSEQ:
597 return CC_EQ;
598 case TGSI_OPCODE_SNE:
599 case TGSI_OPCODE_FSNE:
600 case TGSI_OPCODE_DSNE:
601 return CC_NEU;
602 case TGSI_OPCODE_USNE:
603 return CC_NE;
604 default:
605 return CC_ALWAYS;
606 }
607 }
608
609 #define NV50_IR_OPCODE_CASE(a, b) case TGSI_OPCODE_##a: return nv50_ir::OP_##b
610
611 static nv50_ir::operation translateOpcode(uint opcode)
612 {
613 switch (opcode) {
614 NV50_IR_OPCODE_CASE(ARL, SHL);
615 NV50_IR_OPCODE_CASE(MOV, MOV);
616
617 NV50_IR_OPCODE_CASE(RCP, RCP);
618 NV50_IR_OPCODE_CASE(RSQ, RSQ);
619 NV50_IR_OPCODE_CASE(SQRT, SQRT);
620
621 NV50_IR_OPCODE_CASE(MUL, MUL);
622 NV50_IR_OPCODE_CASE(ADD, ADD);
623
624 NV50_IR_OPCODE_CASE(MIN, MIN);
625 NV50_IR_OPCODE_CASE(MAX, MAX);
626 NV50_IR_OPCODE_CASE(SLT, SET);
627 NV50_IR_OPCODE_CASE(SGE, SET);
628 NV50_IR_OPCODE_CASE(MAD, MAD);
629 NV50_IR_OPCODE_CASE(FMA, FMA);
630 NV50_IR_OPCODE_CASE(SUB, SUB);
631
632 NV50_IR_OPCODE_CASE(FLR, FLOOR);
633 NV50_IR_OPCODE_CASE(ROUND, CVT);
634 NV50_IR_OPCODE_CASE(EX2, EX2);
635 NV50_IR_OPCODE_CASE(LG2, LG2);
636 NV50_IR_OPCODE_CASE(POW, POW);
637
638 NV50_IR_OPCODE_CASE(ABS, ABS);
639
640 NV50_IR_OPCODE_CASE(COS, COS);
641 NV50_IR_OPCODE_CASE(DDX, DFDX);
642 NV50_IR_OPCODE_CASE(DDX_FINE, DFDX);
643 NV50_IR_OPCODE_CASE(DDY, DFDY);
644 NV50_IR_OPCODE_CASE(DDY_FINE, DFDY);
645 NV50_IR_OPCODE_CASE(KILL, DISCARD);
646
647 NV50_IR_OPCODE_CASE(SEQ, SET);
648 NV50_IR_OPCODE_CASE(SGT, SET);
649 NV50_IR_OPCODE_CASE(SIN, SIN);
650 NV50_IR_OPCODE_CASE(SLE, SET);
651 NV50_IR_OPCODE_CASE(SNE, SET);
652 NV50_IR_OPCODE_CASE(TEX, TEX);
653 NV50_IR_OPCODE_CASE(TXD, TXD);
654 NV50_IR_OPCODE_CASE(TXP, TEX);
655
656 NV50_IR_OPCODE_CASE(CAL, CALL);
657 NV50_IR_OPCODE_CASE(RET, RET);
658 NV50_IR_OPCODE_CASE(CMP, SLCT);
659
660 NV50_IR_OPCODE_CASE(TXB, TXB);
661
662 NV50_IR_OPCODE_CASE(DIV, DIV);
663
664 NV50_IR_OPCODE_CASE(TXL, TXL);
665
666 NV50_IR_OPCODE_CASE(CEIL, CEIL);
667 NV50_IR_OPCODE_CASE(I2F, CVT);
668 NV50_IR_OPCODE_CASE(NOT, NOT);
669 NV50_IR_OPCODE_CASE(TRUNC, TRUNC);
670 NV50_IR_OPCODE_CASE(SHL, SHL);
671
672 NV50_IR_OPCODE_CASE(AND, AND);
673 NV50_IR_OPCODE_CASE(OR, OR);
674 NV50_IR_OPCODE_CASE(MOD, MOD);
675 NV50_IR_OPCODE_CASE(XOR, XOR);
676 NV50_IR_OPCODE_CASE(SAD, SAD);
677 NV50_IR_OPCODE_CASE(TXF, TXF);
678 NV50_IR_OPCODE_CASE(TXQ, TXQ);
679 NV50_IR_OPCODE_CASE(TXQS, TXQ);
680 NV50_IR_OPCODE_CASE(TG4, TXG);
681 NV50_IR_OPCODE_CASE(LODQ, TXLQ);
682
683 NV50_IR_OPCODE_CASE(EMIT, EMIT);
684 NV50_IR_OPCODE_CASE(ENDPRIM, RESTART);
685
686 NV50_IR_OPCODE_CASE(KILL_IF, DISCARD);
687
688 NV50_IR_OPCODE_CASE(F2I, CVT);
689 NV50_IR_OPCODE_CASE(FSEQ, SET);
690 NV50_IR_OPCODE_CASE(FSGE, SET);
691 NV50_IR_OPCODE_CASE(FSLT, SET);
692 NV50_IR_OPCODE_CASE(FSNE, SET);
693 NV50_IR_OPCODE_CASE(IDIV, DIV);
694 NV50_IR_OPCODE_CASE(IMAX, MAX);
695 NV50_IR_OPCODE_CASE(IMIN, MIN);
696 NV50_IR_OPCODE_CASE(IABS, ABS);
697 NV50_IR_OPCODE_CASE(INEG, NEG);
698 NV50_IR_OPCODE_CASE(ISGE, SET);
699 NV50_IR_OPCODE_CASE(ISHR, SHR);
700 NV50_IR_OPCODE_CASE(ISLT, SET);
701 NV50_IR_OPCODE_CASE(F2U, CVT);
702 NV50_IR_OPCODE_CASE(U2F, CVT);
703 NV50_IR_OPCODE_CASE(UADD, ADD);
704 NV50_IR_OPCODE_CASE(UDIV, DIV);
705 NV50_IR_OPCODE_CASE(UMAD, MAD);
706 NV50_IR_OPCODE_CASE(UMAX, MAX);
707 NV50_IR_OPCODE_CASE(UMIN, MIN);
708 NV50_IR_OPCODE_CASE(UMOD, MOD);
709 NV50_IR_OPCODE_CASE(UMUL, MUL);
710 NV50_IR_OPCODE_CASE(USEQ, SET);
711 NV50_IR_OPCODE_CASE(USGE, SET);
712 NV50_IR_OPCODE_CASE(USHR, SHR);
713 NV50_IR_OPCODE_CASE(USLT, SET);
714 NV50_IR_OPCODE_CASE(USNE, SET);
715
716 NV50_IR_OPCODE_CASE(DABS, ABS);
717 NV50_IR_OPCODE_CASE(DNEG, NEG);
718 NV50_IR_OPCODE_CASE(DADD, ADD);
719 NV50_IR_OPCODE_CASE(DMUL, MUL);
720 NV50_IR_OPCODE_CASE(DMAX, MAX);
721 NV50_IR_OPCODE_CASE(DMIN, MIN);
722 NV50_IR_OPCODE_CASE(DSLT, SET);
723 NV50_IR_OPCODE_CASE(DSGE, SET);
724 NV50_IR_OPCODE_CASE(DSEQ, SET);
725 NV50_IR_OPCODE_CASE(DSNE, SET);
726 NV50_IR_OPCODE_CASE(DRCP, RCP);
727 NV50_IR_OPCODE_CASE(DSQRT, SQRT);
728 NV50_IR_OPCODE_CASE(DMAD, MAD);
729 NV50_IR_OPCODE_CASE(DFMA, FMA);
730 NV50_IR_OPCODE_CASE(D2I, CVT);
731 NV50_IR_OPCODE_CASE(D2U, CVT);
732 NV50_IR_OPCODE_CASE(I2D, CVT);
733 NV50_IR_OPCODE_CASE(U2D, CVT);
734 NV50_IR_OPCODE_CASE(DRSQ, RSQ);
735 NV50_IR_OPCODE_CASE(DTRUNC, TRUNC);
736 NV50_IR_OPCODE_CASE(DCEIL, CEIL);
737 NV50_IR_OPCODE_CASE(DFLR, FLOOR);
738 NV50_IR_OPCODE_CASE(DROUND, CVT);
739
740 NV50_IR_OPCODE_CASE(IMUL_HI, MUL);
741 NV50_IR_OPCODE_CASE(UMUL_HI, MUL);
742
743 NV50_IR_OPCODE_CASE(SAMPLE, TEX);
744 NV50_IR_OPCODE_CASE(SAMPLE_B, TXB);
745 NV50_IR_OPCODE_CASE(SAMPLE_C, TEX);
746 NV50_IR_OPCODE_CASE(SAMPLE_C_LZ, TEX);
747 NV50_IR_OPCODE_CASE(SAMPLE_D, TXD);
748 NV50_IR_OPCODE_CASE(SAMPLE_L, TXL);
749 NV50_IR_OPCODE_CASE(SAMPLE_I, TXF);
750 NV50_IR_OPCODE_CASE(SAMPLE_I_MS, TXF);
751 NV50_IR_OPCODE_CASE(GATHER4, TXG);
752 NV50_IR_OPCODE_CASE(SVIEWINFO, TXQ);
753
754 NV50_IR_OPCODE_CASE(ATOMUADD, ATOM);
755 NV50_IR_OPCODE_CASE(ATOMXCHG, ATOM);
756 NV50_IR_OPCODE_CASE(ATOMCAS, ATOM);
757 NV50_IR_OPCODE_CASE(ATOMAND, ATOM);
758 NV50_IR_OPCODE_CASE(ATOMOR, ATOM);
759 NV50_IR_OPCODE_CASE(ATOMXOR, ATOM);
760 NV50_IR_OPCODE_CASE(ATOMUMIN, ATOM);
761 NV50_IR_OPCODE_CASE(ATOMUMAX, ATOM);
762 NV50_IR_OPCODE_CASE(ATOMIMIN, ATOM);
763 NV50_IR_OPCODE_CASE(ATOMIMAX, ATOM);
764
765 NV50_IR_OPCODE_CASE(TEX2, TEX);
766 NV50_IR_OPCODE_CASE(TXB2, TXB);
767 NV50_IR_OPCODE_CASE(TXL2, TXL);
768
769 NV50_IR_OPCODE_CASE(IBFE, EXTBF);
770 NV50_IR_OPCODE_CASE(UBFE, EXTBF);
771 NV50_IR_OPCODE_CASE(BFI, INSBF);
772 NV50_IR_OPCODE_CASE(BREV, EXTBF);
773 NV50_IR_OPCODE_CASE(POPC, POPCNT);
774 NV50_IR_OPCODE_CASE(LSB, BFIND);
775 NV50_IR_OPCODE_CASE(IMSB, BFIND);
776 NV50_IR_OPCODE_CASE(UMSB, BFIND);
777
778 NV50_IR_OPCODE_CASE(END, EXIT);
779
780 default:
781 return nv50_ir::OP_NOP;
782 }
783 }
784
785 static uint16_t opcodeToSubOp(uint opcode)
786 {
787 switch (opcode) {
788 case TGSI_OPCODE_LFENCE: return NV50_IR_SUBOP_MEMBAR(L, GL);
789 case TGSI_OPCODE_SFENCE: return NV50_IR_SUBOP_MEMBAR(S, GL);
790 case TGSI_OPCODE_MFENCE: return NV50_IR_SUBOP_MEMBAR(M, GL);
791 case TGSI_OPCODE_ATOMUADD: return NV50_IR_SUBOP_ATOM_ADD;
792 case TGSI_OPCODE_ATOMXCHG: return NV50_IR_SUBOP_ATOM_EXCH;
793 case TGSI_OPCODE_ATOMCAS: return NV50_IR_SUBOP_ATOM_CAS;
794 case TGSI_OPCODE_ATOMAND: return NV50_IR_SUBOP_ATOM_AND;
795 case TGSI_OPCODE_ATOMOR: return NV50_IR_SUBOP_ATOM_OR;
796 case TGSI_OPCODE_ATOMXOR: return NV50_IR_SUBOP_ATOM_XOR;
797 case TGSI_OPCODE_ATOMUMIN: return NV50_IR_SUBOP_ATOM_MIN;
798 case TGSI_OPCODE_ATOMIMIN: return NV50_IR_SUBOP_ATOM_MIN;
799 case TGSI_OPCODE_ATOMUMAX: return NV50_IR_SUBOP_ATOM_MAX;
800 case TGSI_OPCODE_ATOMIMAX: return NV50_IR_SUBOP_ATOM_MAX;
801 case TGSI_OPCODE_IMUL_HI:
802 case TGSI_OPCODE_UMUL_HI:
803 return NV50_IR_SUBOP_MUL_HIGH;
804 default:
805 return 0;
806 }
807 }
808
809 bool Instruction::checkDstSrcAliasing() const
810 {
811 if (insn->Dst[0].Register.Indirect) // no danger if indirect, using memory
812 return false;
813
814 for (int s = 0; s < TGSI_FULL_MAX_SRC_REGISTERS; ++s) {
815 if (insn->Src[s].Register.File == TGSI_FILE_NULL)
816 break;
817 if (insn->Src[s].Register.File == insn->Dst[0].Register.File &&
818 insn->Src[s].Register.Index == insn->Dst[0].Register.Index)
819 return true;
820 }
821 return false;
822 }
823
824 class Source
825 {
826 public:
827 Source(struct nv50_ir_prog_info *);
828 ~Source();
829
830 public:
831 bool scanSource();
832 unsigned fileSize(unsigned file) const { return scan.file_max[file] + 1; }
833
834 public:
835 struct tgsi_shader_info scan;
836 struct tgsi_full_instruction *insns;
837 const struct tgsi_token *tokens;
838 struct nv50_ir_prog_info *info;
839
840 nv50_ir::DynArray tempArrays;
841 nv50_ir::DynArray immdArrays;
842
843 typedef nv50_ir::BuildUtil::Location Location;
844 // these registers are per-subroutine, cannot be used for parameter passing
845 std::set<Location> locals;
846
847 std::set<int> indirectTempArrays;
848 std::map<int, int> indirectTempOffsets;
849 std::map<int, std::pair<int, int> > tempArrayInfo;
850 std::vector<int> tempArrayId;
851
852 int clipVertexOutput;
853
854 struct TextureView {
855 uint8_t target; // TGSI_TEXTURE_*
856 };
857 std::vector<TextureView> textureViews;
858
859 struct Resource {
860 uint8_t target; // TGSI_TEXTURE_*
861 bool raw;
862 uint8_t slot; // $surface index
863 };
864 std::vector<Resource> resources;
865
866 struct MemoryFile {
867 uint8_t mem_type; // TGSI_MEMORY_TYPE_*
868 };
869 std::vector<MemoryFile> memoryFiles;
870
871 private:
872 int inferSysValDirection(unsigned sn) const;
873 bool scanDeclaration(const struct tgsi_full_declaration *);
874 bool scanInstruction(const struct tgsi_full_instruction *);
875 void scanProperty(const struct tgsi_full_property *);
876 void scanImmediate(const struct tgsi_full_immediate *);
877
878 inline bool isEdgeFlagPassthrough(const Instruction&) const;
879 };
880
881 Source::Source(struct nv50_ir_prog_info *prog) : info(prog)
882 {
883 tokens = (const struct tgsi_token *)info->bin.source;
884
885 if (prog->dbgFlags & NV50_IR_DEBUG_BASIC)
886 tgsi_dump(tokens, 0);
887 }
888
889 Source::~Source()
890 {
891 if (insns)
892 FREE(insns);
893
894 if (info->immd.data)
895 FREE(info->immd.data);
896 if (info->immd.type)
897 FREE(info->immd.type);
898 }
899
900 bool Source::scanSource()
901 {
902 unsigned insnCount = 0;
903 struct tgsi_parse_context parse;
904
905 tgsi_scan_shader(tokens, &scan);
906
907 insns = (struct tgsi_full_instruction *)MALLOC(scan.num_instructions *
908 sizeof(insns[0]));
909 if (!insns)
910 return false;
911
912 clipVertexOutput = -1;
913
914 textureViews.resize(scan.file_max[TGSI_FILE_SAMPLER_VIEW] + 1);
915 //resources.resize(scan.file_max[TGSI_FILE_RESOURCE] + 1);
916 tempArrayId.resize(scan.file_max[TGSI_FILE_TEMPORARY] + 1);
917 memoryFiles.resize(scan.file_max[TGSI_FILE_MEMORY] + 1);
918
919 info->immd.bufSize = 0;
920
921 info->numInputs = scan.file_max[TGSI_FILE_INPUT] + 1;
922 info->numOutputs = scan.file_max[TGSI_FILE_OUTPUT] + 1;
923 info->numSysVals = scan.file_max[TGSI_FILE_SYSTEM_VALUE] + 1;
924
925 if (info->type == PIPE_SHADER_FRAGMENT) {
926 info->prop.fp.writesDepth = scan.writes_z;
927 info->prop.fp.usesDiscard = scan.uses_kill;
928 } else
929 if (info->type == PIPE_SHADER_GEOMETRY) {
930 info->prop.gp.instanceCount = 1; // default value
931 }
932
933 info->io.viewportId = -1;
934
935 info->immd.data = (uint32_t *)MALLOC(scan.immediate_count * 16);
936 info->immd.type = (ubyte *)MALLOC(scan.immediate_count * sizeof(ubyte));
937
938 tgsi_parse_init(&parse, tokens);
939 while (!tgsi_parse_end_of_tokens(&parse)) {
940 tgsi_parse_token(&parse);
941
942 switch (parse.FullToken.Token.Type) {
943 case TGSI_TOKEN_TYPE_IMMEDIATE:
944 scanImmediate(&parse.FullToken.FullImmediate);
945 break;
946 case TGSI_TOKEN_TYPE_DECLARATION:
947 scanDeclaration(&parse.FullToken.FullDeclaration);
948 break;
949 case TGSI_TOKEN_TYPE_INSTRUCTION:
950 insns[insnCount++] = parse.FullToken.FullInstruction;
951 scanInstruction(&parse.FullToken.FullInstruction);
952 break;
953 case TGSI_TOKEN_TYPE_PROPERTY:
954 scanProperty(&parse.FullToken.FullProperty);
955 break;
956 default:
957 INFO("unknown TGSI token type: %d\n", parse.FullToken.Token.Type);
958 break;
959 }
960 }
961 tgsi_parse_free(&parse);
962
963 if (indirectTempArrays.size()) {
964 int tempBase = 0;
965 for (std::set<int>::const_iterator it = indirectTempArrays.begin();
966 it != indirectTempArrays.end(); ++it) {
967 std::pair<int, int>& info = tempArrayInfo[*it];
968 indirectTempOffsets.insert(std::make_pair(*it, tempBase - info.first));
969 tempBase += info.second;
970 }
971 info->bin.tlsSpace += tempBase * 16;
972 }
973
974 if (info->io.genUserClip > 0) {
975 info->io.clipDistances = info->io.genUserClip;
976
977 const unsigned int nOut = (info->io.genUserClip + 3) / 4;
978
979 for (unsigned int n = 0; n < nOut; ++n) {
980 unsigned int i = info->numOutputs++;
981 info->out[i].id = i;
982 info->out[i].sn = TGSI_SEMANTIC_CLIPDIST;
983 info->out[i].si = n;
984 info->out[i].mask = ((1 << info->io.clipDistances) - 1) >> (n * 4);
985 }
986 }
987
988 return info->assignSlots(info) == 0;
989 }
990
991 void Source::scanProperty(const struct tgsi_full_property *prop)
992 {
993 switch (prop->Property.PropertyName) {
994 case TGSI_PROPERTY_GS_OUTPUT_PRIM:
995 info->prop.gp.outputPrim = prop->u[0].Data;
996 break;
997 case TGSI_PROPERTY_GS_INPUT_PRIM:
998 info->prop.gp.inputPrim = prop->u[0].Data;
999 break;
1000 case TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES:
1001 info->prop.gp.maxVertices = prop->u[0].Data;
1002 break;
1003 case TGSI_PROPERTY_GS_INVOCATIONS:
1004 info->prop.gp.instanceCount = prop->u[0].Data;
1005 break;
1006 case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
1007 info->prop.fp.separateFragData = true;
1008 break;
1009 case TGSI_PROPERTY_FS_COORD_ORIGIN:
1010 case TGSI_PROPERTY_FS_COORD_PIXEL_CENTER:
1011 // we don't care
1012 break;
1013 case TGSI_PROPERTY_VS_PROHIBIT_UCPS:
1014 info->io.genUserClip = -1;
1015 break;
1016 case TGSI_PROPERTY_TCS_VERTICES_OUT:
1017 info->prop.tp.outputPatchSize = prop->u[0].Data;
1018 break;
1019 case TGSI_PROPERTY_TES_PRIM_MODE:
1020 info->prop.tp.domain = prop->u[0].Data;
1021 break;
1022 case TGSI_PROPERTY_TES_SPACING:
1023 info->prop.tp.partitioning = prop->u[0].Data;
1024 break;
1025 case TGSI_PROPERTY_TES_VERTEX_ORDER_CW:
1026 info->prop.tp.winding = prop->u[0].Data;
1027 break;
1028 case TGSI_PROPERTY_TES_POINT_MODE:
1029 if (prop->u[0].Data)
1030 info->prop.tp.outputPrim = PIPE_PRIM_POINTS;
1031 else
1032 info->prop.tp.outputPrim = PIPE_PRIM_TRIANGLES; /* anything but points */
1033 break;
1034 case TGSI_PROPERTY_NUM_CLIPDIST_ENABLED:
1035 info->io.clipDistances = prop->u[0].Data;
1036 break;
1037 case TGSI_PROPERTY_NUM_CULLDIST_ENABLED:
1038 info->io.cullDistances = prop->u[0].Data;
1039 break;
1040 default:
1041 INFO("unhandled TGSI property %d\n", prop->Property.PropertyName);
1042 break;
1043 }
1044 }
1045
1046 void Source::scanImmediate(const struct tgsi_full_immediate *imm)
1047 {
1048 const unsigned n = info->immd.count++;
1049
1050 assert(n < scan.immediate_count);
1051
1052 for (int c = 0; c < 4; ++c)
1053 info->immd.data[n * 4 + c] = imm->u[c].Uint;
1054
1055 info->immd.type[n] = imm->Immediate.DataType;
1056 }
1057
1058 int Source::inferSysValDirection(unsigned sn) const
1059 {
1060 switch (sn) {
1061 case TGSI_SEMANTIC_INSTANCEID:
1062 case TGSI_SEMANTIC_VERTEXID:
1063 return 1;
1064 case TGSI_SEMANTIC_LAYER:
1065 #if 0
1066 case TGSI_SEMANTIC_VIEWPORTINDEX:
1067 return 0;
1068 #endif
1069 case TGSI_SEMANTIC_PRIMID:
1070 return (info->type == PIPE_SHADER_FRAGMENT) ? 1 : 0;
1071 default:
1072 return 0;
1073 }
1074 }
1075
1076 bool Source::scanDeclaration(const struct tgsi_full_declaration *decl)
1077 {
1078 unsigned i, c;
1079 unsigned sn = TGSI_SEMANTIC_GENERIC;
1080 unsigned si = 0;
1081 const unsigned first = decl->Range.First, last = decl->Range.Last;
1082 const int arrayId = decl->Array.ArrayID;
1083
1084 if (decl->Declaration.Semantic) {
1085 sn = decl->Semantic.Name;
1086 si = decl->Semantic.Index;
1087 }
1088
1089 if (decl->Declaration.Local) {
1090 for (i = first; i <= last; ++i) {
1091 for (c = 0; c < 4; ++c) {
1092 locals.insert(
1093 Location(decl->Declaration.File, decl->Dim.Index2D, i, c));
1094 }
1095 }
1096 }
1097
1098 switch (decl->Declaration.File) {
1099 case TGSI_FILE_INPUT:
1100 if (info->type == PIPE_SHADER_VERTEX) {
1101 // all vertex attributes are equal
1102 for (i = first; i <= last; ++i) {
1103 info->in[i].sn = TGSI_SEMANTIC_GENERIC;
1104 info->in[i].si = i;
1105 }
1106 } else {
1107 for (i = first; i <= last; ++i, ++si) {
1108 info->in[i].id = i;
1109 info->in[i].sn = sn;
1110 info->in[i].si = si;
1111 if (info->type == PIPE_SHADER_FRAGMENT) {
1112 // translate interpolation mode
1113 switch (decl->Interp.Interpolate) {
1114 case TGSI_INTERPOLATE_CONSTANT:
1115 info->in[i].flat = 1;
1116 break;
1117 case TGSI_INTERPOLATE_COLOR:
1118 info->in[i].sc = 1;
1119 break;
1120 case TGSI_INTERPOLATE_LINEAR:
1121 info->in[i].linear = 1;
1122 break;
1123 default:
1124 break;
1125 }
1126 if (decl->Interp.Location)
1127 info->in[i].centroid = 1;
1128 }
1129
1130 if (sn == TGSI_SEMANTIC_PATCH)
1131 info->in[i].patch = 1;
1132 if (sn == TGSI_SEMANTIC_PATCH)
1133 info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1134 }
1135 }
1136 break;
1137 case TGSI_FILE_OUTPUT:
1138 for (i = first; i <= last; ++i, ++si) {
1139 switch (sn) {
1140 case TGSI_SEMANTIC_POSITION:
1141 if (info->type == PIPE_SHADER_FRAGMENT)
1142 info->io.fragDepth = i;
1143 else
1144 if (clipVertexOutput < 0)
1145 clipVertexOutput = i;
1146 break;
1147 case TGSI_SEMANTIC_COLOR:
1148 if (info->type == PIPE_SHADER_FRAGMENT)
1149 info->prop.fp.numColourResults++;
1150 break;
1151 case TGSI_SEMANTIC_EDGEFLAG:
1152 info->io.edgeFlagOut = i;
1153 break;
1154 case TGSI_SEMANTIC_CLIPVERTEX:
1155 clipVertexOutput = i;
1156 break;
1157 case TGSI_SEMANTIC_CLIPDIST:
1158 info->io.genUserClip = -1;
1159 break;
1160 case TGSI_SEMANTIC_SAMPLEMASK:
1161 info->io.sampleMask = i;
1162 break;
1163 case TGSI_SEMANTIC_VIEWPORT_INDEX:
1164 info->io.viewportId = i;
1165 break;
1166 case TGSI_SEMANTIC_PATCH:
1167 info->numPatchConstants = MAX2(info->numPatchConstants, si + 1);
1168 /* fallthrough */
1169 case TGSI_SEMANTIC_TESSOUTER:
1170 case TGSI_SEMANTIC_TESSINNER:
1171 info->out[i].patch = 1;
1172 break;
1173 default:
1174 break;
1175 }
1176 info->out[i].id = i;
1177 info->out[i].sn = sn;
1178 info->out[i].si = si;
1179 }
1180 break;
1181 case TGSI_FILE_SYSTEM_VALUE:
1182 switch (sn) {
1183 case TGSI_SEMANTIC_INSTANCEID:
1184 info->io.instanceId = first;
1185 break;
1186 case TGSI_SEMANTIC_VERTEXID:
1187 info->io.vertexId = first;
1188 break;
1189 case TGSI_SEMANTIC_BASEVERTEX:
1190 case TGSI_SEMANTIC_BASEINSTANCE:
1191 case TGSI_SEMANTIC_DRAWID:
1192 info->prop.vp.usesDrawParameters = true;
1193 break;
1194 default:
1195 break;
1196 }
1197 for (i = first; i <= last; ++i, ++si) {
1198 info->sv[i].sn = sn;
1199 info->sv[i].si = si;
1200 info->sv[i].input = inferSysValDirection(sn);
1201
1202 switch (sn) {
1203 case TGSI_SEMANTIC_TESSOUTER:
1204 case TGSI_SEMANTIC_TESSINNER:
1205 info->sv[i].patch = 1;
1206 break;
1207 }
1208 }
1209 break;
1210 /*
1211 case TGSI_FILE_RESOURCE:
1212 for (i = first; i <= last; ++i) {
1213 resources[i].target = decl->Resource.Resource;
1214 resources[i].raw = decl->Resource.Raw;
1215 resources[i].slot = i;
1216 }
1217 break;
1218 */
1219 case TGSI_FILE_SAMPLER_VIEW:
1220 for (i = first; i <= last; ++i)
1221 textureViews[i].target = decl->SamplerView.Resource;
1222 break;
1223 case TGSI_FILE_MEMORY:
1224 for (i = first; i <= last; ++i)
1225 memoryFiles[i].mem_type = decl->Declaration.MemType;
1226 break;
1227 case TGSI_FILE_NULL:
1228 case TGSI_FILE_TEMPORARY:
1229 for (i = first; i <= last; ++i)
1230 tempArrayId[i] = arrayId;
1231 if (arrayId)
1232 tempArrayInfo.insert(std::make_pair(arrayId, std::make_pair(
1233 first, last - first + 1)));
1234 break;
1235 case TGSI_FILE_ADDRESS:
1236 case TGSI_FILE_CONSTANT:
1237 case TGSI_FILE_IMMEDIATE:
1238 case TGSI_FILE_PREDICATE:
1239 case TGSI_FILE_SAMPLER:
1240 case TGSI_FILE_BUFFER:
1241 break;
1242 default:
1243 ERROR("unhandled TGSI_FILE %d\n", decl->Declaration.File);
1244 return false;
1245 }
1246 return true;
1247 }
1248
1249 inline bool Source::isEdgeFlagPassthrough(const Instruction& insn) const
1250 {
1251 return insn.getOpcode() == TGSI_OPCODE_MOV &&
1252 insn.getDst(0).getIndex(0) == info->io.edgeFlagOut &&
1253 insn.getSrc(0).getFile() == TGSI_FILE_INPUT;
1254 }
1255
1256 bool Source::scanInstruction(const struct tgsi_full_instruction *inst)
1257 {
1258 Instruction insn(inst);
1259
1260 if (insn.getOpcode() == TGSI_OPCODE_BARRIER)
1261 info->numBarriers = 1;
1262
1263 if (insn.dstCount()) {
1264 if (insn.getDst(0).getFile() == TGSI_FILE_OUTPUT) {
1265 Instruction::DstRegister dst = insn.getDst(0);
1266
1267 if (dst.isIndirect(0))
1268 for (unsigned i = 0; i < info->numOutputs; ++i)
1269 info->out[i].mask = 0xf;
1270 else
1271 info->out[dst.getIndex(0)].mask |= dst.getMask();
1272
1273 if (info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PSIZE ||
1274 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_PRIMID ||
1275 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_LAYER ||
1276 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_VIEWPORT_INDEX ||
1277 info->out[dst.getIndex(0)].sn == TGSI_SEMANTIC_FOG)
1278 info->out[dst.getIndex(0)].mask &= 1;
1279
1280 if (isEdgeFlagPassthrough(insn))
1281 info->io.edgeFlagIn = insn.getSrc(0).getIndex(0);
1282 } else
1283 if (insn.getDst(0).getFile() == TGSI_FILE_TEMPORARY) {
1284 if (insn.getDst(0).isIndirect(0))
1285 indirectTempArrays.insert(insn.getDst(0).getArrayId());
1286 } else
1287 if (insn.getDst(0).getFile() == TGSI_FILE_BUFFER) {
1288 info->io.globalAccess |= 0x2;
1289 }
1290 }
1291
1292 for (unsigned s = 0; s < insn.srcCount(); ++s) {
1293 Instruction::SrcRegister src = insn.getSrc(s);
1294 if (src.getFile() == TGSI_FILE_TEMPORARY) {
1295 if (src.isIndirect(0))
1296 indirectTempArrays.insert(src.getArrayId());
1297 } else
1298 if (src.getFile() == TGSI_FILE_BUFFER) {
1299 info->io.globalAccess |= (insn.getOpcode() == TGSI_OPCODE_LOAD) ?
1300 0x1 : 0x2;
1301 } else
1302 if (src.getFile() == TGSI_FILE_OUTPUT) {
1303 if (src.isIndirect(0)) {
1304 // We don't know which one is accessed, just mark everything for
1305 // reading. This is an extremely unlikely occurrence.
1306 for (unsigned i = 0; i < info->numOutputs; ++i)
1307 info->out[i].oread = 1;
1308 } else {
1309 info->out[src.getIndex(0)].oread = 1;
1310 }
1311 }
1312 if (src.getFile() != TGSI_FILE_INPUT)
1313 continue;
1314 unsigned mask = insn.srcMask(s);
1315
1316 if (src.isIndirect(0)) {
1317 for (unsigned i = 0; i < info->numInputs; ++i)
1318 info->in[i].mask = 0xf;
1319 } else {
1320 const int i = src.getIndex(0);
1321 for (unsigned c = 0; c < 4; ++c) {
1322 if (!(mask & (1 << c)))
1323 continue;
1324 int k = src.getSwizzle(c);
1325 if (k <= TGSI_SWIZZLE_W)
1326 info->in[i].mask |= 1 << k;
1327 }
1328 switch (info->in[i].sn) {
1329 case TGSI_SEMANTIC_PSIZE:
1330 case TGSI_SEMANTIC_PRIMID:
1331 case TGSI_SEMANTIC_FOG:
1332 info->in[i].mask &= 0x1;
1333 break;
1334 case TGSI_SEMANTIC_PCOORD:
1335 info->in[i].mask &= 0x3;
1336 break;
1337 default:
1338 break;
1339 }
1340 }
1341 }
1342 return true;
1343 }
1344
1345 nv50_ir::TexInstruction::Target
1346 Instruction::getTexture(const tgsi::Source *code, int s) const
1347 {
1348 // XXX: indirect access
1349 unsigned int r;
1350
1351 switch (getSrc(s).getFile()) {
1352 /*
1353 case TGSI_FILE_RESOURCE:
1354 r = getSrc(s).getIndex(0);
1355 return translateTexture(code->resources.at(r).target);
1356 */
1357 case TGSI_FILE_SAMPLER_VIEW:
1358 r = getSrc(s).getIndex(0);
1359 return translateTexture(code->textureViews.at(r).target);
1360 default:
1361 return translateTexture(insn->Texture.Texture);
1362 }
1363 }
1364
1365 } // namespace tgsi
1366
1367 namespace {
1368
1369 using namespace nv50_ir;
1370
1371 class Converter : public BuildUtil
1372 {
1373 public:
1374 Converter(Program *, const tgsi::Source *);
1375 ~Converter();
1376
1377 bool run();
1378
1379 private:
1380 struct Subroutine
1381 {
1382 Subroutine(Function *f) : f(f) { }
1383 Function *f;
1384 ValueMap values;
1385 };
1386
1387 Value *shiftAddress(Value *);
1388 Value *getVertexBase(int s);
1389 Value *getOutputBase(int s);
1390 DataArray *getArrayForFile(unsigned file, int idx);
1391 Value *fetchSrc(int s, int c);
1392 Value *acquireDst(int d, int c);
1393 void storeDst(int d, int c, Value *);
1394
1395 Value *fetchSrc(const tgsi::Instruction::SrcRegister src, int c, Value *ptr);
1396 void storeDst(const tgsi::Instruction::DstRegister dst, int c,
1397 Value *val, Value *ptr);
1398
1399 void adjustTempIndex(int arrayId, int &idx, int &idx2d) const;
1400 Value *applySrcMod(Value *, int s, int c);
1401
1402 Symbol *makeSym(uint file, int fileIndex, int idx, int c, uint32_t addr);
1403 Symbol *srcToSym(tgsi::Instruction::SrcRegister, int c);
1404 Symbol *dstToSym(tgsi::Instruction::DstRegister, int c);
1405
1406 bool handleInstruction(const struct tgsi_full_instruction *);
1407 void exportOutputs();
1408 inline Subroutine *getSubroutine(unsigned ip);
1409 inline Subroutine *getSubroutine(Function *);
1410 inline bool isEndOfSubroutine(uint ip);
1411
1412 void loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask);
1413
1414 // R,S,L,C,Dx,Dy encode TGSI sources for respective values (0xSf for auto)
1415 void setTexRS(TexInstruction *, unsigned int& s, int R, int S);
1416 void handleTEX(Value *dst0[4], int R, int S, int L, int C, int Dx, int Dy);
1417 void handleTXF(Value *dst0[4], int R, int L_M);
1418 void handleTXQ(Value *dst0[4], enum TexQuery, int R);
1419 void handleLIT(Value *dst0[4]);
1420 void handleUserClipPlanes();
1421
1422 Symbol *getResourceBase(int r);
1423 void getResourceCoords(std::vector<Value *>&, int r, int s);
1424
1425 void handleLOAD(Value *dst0[4]);
1426 void handleSTORE();
1427 void handleATOM(Value *dst0[4], DataType, uint16_t subOp);
1428
1429 void handleINTERP(Value *dst0[4]);
1430
1431 uint8_t translateInterpMode(const struct nv50_ir_varying *var,
1432 operation& op);
1433 Value *interpolate(tgsi::Instruction::SrcRegister, int c, Value *ptr);
1434
1435 void insertConvergenceOps(BasicBlock *conv, BasicBlock *fork);
1436
1437 Value *buildDot(int dim);
1438
1439 class BindArgumentsPass : public Pass {
1440 public:
1441 BindArgumentsPass(Converter &conv) : conv(conv) { }
1442
1443 private:
1444 Converter &conv;
1445 Subroutine *sub;
1446
1447 inline const Location *getValueLocation(Subroutine *, Value *);
1448
1449 template<typename T> inline void
1450 updateCallArgs(Instruction *i, void (Instruction::*setArg)(int, Value *),
1451 T (Function::*proto));
1452
1453 template<typename T> inline void
1454 updatePrototype(BitSet *set, void (Function::*updateSet)(),
1455 T (Function::*proto));
1456
1457 protected:
1458 bool visit(Function *);
1459 bool visit(BasicBlock *bb) { return false; }
1460 };
1461
1462 private:
1463 const tgsi::Source *code;
1464 const struct nv50_ir_prog_info *info;
1465
1466 struct {
1467 std::map<unsigned, Subroutine> map;
1468 Subroutine *cur;
1469 } sub;
1470
1471 uint ip; // instruction pointer
1472
1473 tgsi::Instruction tgsi;
1474
1475 DataType dstTy;
1476 DataType srcTy;
1477
1478 DataArray tData; // TGSI_FILE_TEMPORARY
1479 DataArray lData; // TGSI_FILE_TEMPORARY, for indirect arrays
1480 DataArray aData; // TGSI_FILE_ADDRESS
1481 DataArray pData; // TGSI_FILE_PREDICATE
1482 DataArray oData; // TGSI_FILE_OUTPUT (if outputs in registers)
1483
1484 Value *zero;
1485 Value *fragCoord[4];
1486 Value *clipVtx[4];
1487
1488 Value *vtxBase[5]; // base address of vertex in primitive (for TP/GP)
1489 uint8_t vtxBaseValid;
1490
1491 Value *outBase; // base address of vertex out patch (for TCP)
1492
1493 Stack condBBs; // fork BB, then else clause BB
1494 Stack joinBBs; // fork BB, for inserting join ops on ENDIF
1495 Stack loopBBs; // loop headers
1496 Stack breakBBs; // end of / after loop
1497
1498 Value *viewport;
1499 };
1500
1501 Symbol *
1502 Converter::srcToSym(tgsi::Instruction::SrcRegister src, int c)
1503 {
1504 const int swz = src.getSwizzle(c);
1505
1506 /* TODO: Use Array ID when it's available for the index */
1507 return makeSym(src.getFile(),
1508 src.is2D() ? src.getIndex(1) : 0,
1509 src.getIndex(0), swz,
1510 src.getIndex(0) * 16 + swz * 4);
1511 }
1512
1513 Symbol *
1514 Converter::dstToSym(tgsi::Instruction::DstRegister dst, int c)
1515 {
1516 /* TODO: Use Array ID when it's available for the index */
1517 return makeSym(dst.getFile(),
1518 dst.is2D() ? dst.getIndex(1) : 0,
1519 dst.getIndex(0), c,
1520 dst.getIndex(0) * 16 + c * 4);
1521 }
1522
1523 Symbol *
1524 Converter::makeSym(uint tgsiFile, int fileIdx, int idx, int c, uint32_t address)
1525 {
1526 Symbol *sym = new_Symbol(prog, tgsi::translateFile(tgsiFile));
1527
1528 sym->reg.fileIndex = fileIdx;
1529
1530 if (tgsiFile == TGSI_FILE_MEMORY &&
1531 code->memoryFiles[fileIdx].mem_type == TGSI_MEMORY_TYPE_SHARED)
1532 sym->setFile(FILE_MEMORY_SHARED);
1533
1534 if (idx >= 0) {
1535 if (sym->reg.file == FILE_SHADER_INPUT)
1536 sym->setOffset(info->in[idx].slot[c] * 4);
1537 else
1538 if (sym->reg.file == FILE_SHADER_OUTPUT)
1539 sym->setOffset(info->out[idx].slot[c] * 4);
1540 else
1541 if (sym->reg.file == FILE_SYSTEM_VALUE)
1542 sym->setSV(tgsi::translateSysVal(info->sv[idx].sn), c);
1543 else
1544 sym->setOffset(address);
1545 } else {
1546 sym->setOffset(address);
1547 }
1548 return sym;
1549 }
1550
1551 uint8_t
1552 Converter::translateInterpMode(const struct nv50_ir_varying *var, operation& op)
1553 {
1554 uint8_t mode = NV50_IR_INTERP_PERSPECTIVE;
1555
1556 if (var->flat)
1557 mode = NV50_IR_INTERP_FLAT;
1558 else
1559 if (var->linear)
1560 mode = NV50_IR_INTERP_LINEAR;
1561 else
1562 if (var->sc)
1563 mode = NV50_IR_INTERP_SC;
1564
1565 op = (mode == NV50_IR_INTERP_PERSPECTIVE || mode == NV50_IR_INTERP_SC)
1566 ? OP_PINTERP : OP_LINTERP;
1567
1568 if (var->centroid)
1569 mode |= NV50_IR_INTERP_CENTROID;
1570
1571 return mode;
1572 }
1573
1574 Value *
1575 Converter::interpolate(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1576 {
1577 operation op;
1578
1579 // XXX: no way to know interpolation mode if we don't know what's accessed
1580 const uint8_t mode = translateInterpMode(&info->in[ptr ? 0 :
1581 src.getIndex(0)], op);
1582
1583 Instruction *insn = new_Instruction(func, op, TYPE_F32);
1584
1585 insn->setDef(0, getScratch());
1586 insn->setSrc(0, srcToSym(src, c));
1587 if (op == OP_PINTERP)
1588 insn->setSrc(1, fragCoord[3]);
1589 if (ptr)
1590 insn->setIndirect(0, 0, ptr);
1591
1592 insn->setInterpolate(mode);
1593
1594 bb->insertTail(insn);
1595 return insn->getDef(0);
1596 }
1597
1598 Value *
1599 Converter::applySrcMod(Value *val, int s, int c)
1600 {
1601 Modifier m = tgsi.getSrc(s).getMod(c);
1602 DataType ty = tgsi.inferSrcType();
1603
1604 if (m & Modifier(NV50_IR_MOD_ABS))
1605 val = mkOp1v(OP_ABS, ty, getScratch(), val);
1606
1607 if (m & Modifier(NV50_IR_MOD_NEG))
1608 val = mkOp1v(OP_NEG, ty, getScratch(), val);
1609
1610 return val;
1611 }
1612
1613 Value *
1614 Converter::getVertexBase(int s)
1615 {
1616 assert(s < 5);
1617 if (!(vtxBaseValid & (1 << s))) {
1618 const int index = tgsi.getSrc(s).getIndex(1);
1619 Value *rel = NULL;
1620 if (tgsi.getSrc(s).isIndirect(1))
1621 rel = fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL);
1622 vtxBaseValid |= 1 << s;
1623 vtxBase[s] = mkOp2v(OP_PFETCH, TYPE_U32, getSSA(4, FILE_ADDRESS),
1624 mkImm(index), rel);
1625 }
1626 return vtxBase[s];
1627 }
1628
1629 Value *
1630 Converter::getOutputBase(int s)
1631 {
1632 assert(s < 5);
1633 if (!(vtxBaseValid & (1 << s))) {
1634 Value *offset = loadImm(NULL, tgsi.getSrc(s).getIndex(1));
1635 if (tgsi.getSrc(s).isIndirect(1))
1636 offset = mkOp2v(OP_ADD, TYPE_U32, getSSA(),
1637 fetchSrc(tgsi.getSrc(s).getIndirect(1), 0, NULL),
1638 offset);
1639 vtxBaseValid |= 1 << s;
1640 vtxBase[s] = mkOp2v(OP_ADD, TYPE_U32, getSSA(), outBase, offset);
1641 }
1642 return vtxBase[s];
1643 }
1644
1645 Value *
1646 Converter::fetchSrc(int s, int c)
1647 {
1648 Value *res;
1649 Value *ptr = NULL, *dimRel = NULL;
1650
1651 tgsi::Instruction::SrcRegister src = tgsi.getSrc(s);
1652
1653 if (src.isIndirect(0))
1654 ptr = fetchSrc(src.getIndirect(0), 0, NULL);
1655
1656 if (src.is2D()) {
1657 switch (src.getFile()) {
1658 case TGSI_FILE_OUTPUT:
1659 dimRel = getOutputBase(s);
1660 break;
1661 case TGSI_FILE_INPUT:
1662 dimRel = getVertexBase(s);
1663 break;
1664 case TGSI_FILE_CONSTANT:
1665 // on NVC0, this is valid and c{I+J}[k] == cI[(J << 16) + k]
1666 if (src.isIndirect(1))
1667 dimRel = fetchSrc(src.getIndirect(1), 0, 0);
1668 break;
1669 default:
1670 break;
1671 }
1672 }
1673
1674 res = fetchSrc(src, c, ptr);
1675
1676 if (dimRel)
1677 res->getInsn()->setIndirect(0, 1, dimRel);
1678
1679 return applySrcMod(res, s, c);
1680 }
1681
1682 Converter::DataArray *
1683 Converter::getArrayForFile(unsigned file, int idx)
1684 {
1685 switch (file) {
1686 case TGSI_FILE_TEMPORARY:
1687 return idx == 0 ? &tData : &lData;
1688 case TGSI_FILE_PREDICATE:
1689 return &pData;
1690 case TGSI_FILE_ADDRESS:
1691 return &aData;
1692 case TGSI_FILE_OUTPUT:
1693 assert(prog->getType() == Program::TYPE_FRAGMENT);
1694 return &oData;
1695 default:
1696 assert(!"invalid/unhandled TGSI source file");
1697 return NULL;
1698 }
1699 }
1700
1701 Value *
1702 Converter::shiftAddress(Value *index)
1703 {
1704 if (!index)
1705 return NULL;
1706 return mkOp2v(OP_SHL, TYPE_U32, getSSA(4, FILE_ADDRESS), index, mkImm(4));
1707 }
1708
1709 void
1710 Converter::adjustTempIndex(int arrayId, int &idx, int &idx2d) const
1711 {
1712 std::map<int, int>::const_iterator it =
1713 code->indirectTempOffsets.find(arrayId);
1714 if (it == code->indirectTempOffsets.end())
1715 return;
1716
1717 idx2d = 1;
1718 idx += it->second;
1719 }
1720
1721 Value *
1722 Converter::fetchSrc(tgsi::Instruction::SrcRegister src, int c, Value *ptr)
1723 {
1724 int idx2d = src.is2D() ? src.getIndex(1) : 0;
1725 int idx = src.getIndex(0);
1726 const int swz = src.getSwizzle(c);
1727 Instruction *ld;
1728
1729 switch (src.getFile()) {
1730 case TGSI_FILE_IMMEDIATE:
1731 assert(!ptr);
1732 return loadImm(NULL, info->immd.data[idx * 4 + swz]);
1733 case TGSI_FILE_CONSTANT:
1734 return mkLoadv(TYPE_U32, srcToSym(src, c), shiftAddress(ptr));
1735 case TGSI_FILE_INPUT:
1736 if (prog->getType() == Program::TYPE_FRAGMENT) {
1737 // don't load masked inputs, won't be assigned a slot
1738 if (!ptr && !(info->in[idx].mask & (1 << swz)))
1739 return loadImm(NULL, swz == TGSI_SWIZZLE_W ? 1.0f : 0.0f);
1740 return interpolate(src, c, shiftAddress(ptr));
1741 } else
1742 if (prog->getType() == Program::TYPE_GEOMETRY) {
1743 if (!ptr && info->in[idx].sn == TGSI_SEMANTIC_PRIMID)
1744 return mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_PRIMITIVE_ID, 0));
1745 // XXX: This is going to be a problem with scalar arrays, i.e. when
1746 // we cannot assume that the address is given in units of vec4.
1747 //
1748 // nv50 and nvc0 need different things here, so let the lowering
1749 // passes decide what to do with the address
1750 if (ptr)
1751 return mkLoadv(TYPE_U32, srcToSym(src, c), ptr);
1752 }
1753 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1754 ld->perPatch = info->in[idx].patch;
1755 return ld->getDef(0);
1756 case TGSI_FILE_OUTPUT:
1757 assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL);
1758 ld = mkLoad(TYPE_U32, getSSA(), srcToSym(src, c), shiftAddress(ptr));
1759 ld->perPatch = info->out[idx].patch;
1760 return ld->getDef(0);
1761 case TGSI_FILE_SYSTEM_VALUE:
1762 assert(!ptr);
1763 ld = mkOp1(OP_RDSV, TYPE_U32, getSSA(), srcToSym(src, c));
1764 ld->perPatch = info->sv[idx].patch;
1765 return ld->getDef(0);
1766 case TGSI_FILE_TEMPORARY: {
1767 int arrayid = src.getArrayId();
1768 if (!arrayid)
1769 arrayid = code->tempArrayId[idx];
1770 adjustTempIndex(arrayid, idx, idx2d);
1771 }
1772 /* fallthrough */
1773 default:
1774 return getArrayForFile(src.getFile(), idx2d)->load(
1775 sub.cur->values, idx, swz, shiftAddress(ptr));
1776 }
1777 }
1778
1779 Value *
1780 Converter::acquireDst(int d, int c)
1781 {
1782 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1783 const unsigned f = dst.getFile();
1784 int idx = dst.getIndex(0);
1785 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1786
1787 if (dst.isMasked(c) || f == TGSI_FILE_BUFFER || f == TGSI_FILE_MEMORY)
1788 return NULL;
1789
1790 if (dst.isIndirect(0) ||
1791 f == TGSI_FILE_SYSTEM_VALUE ||
1792 (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT))
1793 return getScratch();
1794
1795 if (f == TGSI_FILE_TEMPORARY) {
1796 int arrayid = dst.getArrayId();
1797 if (!arrayid)
1798 arrayid = code->tempArrayId[idx];
1799 adjustTempIndex(arrayid, idx, idx2d);
1800 }
1801
1802 return getArrayForFile(f, idx2d)-> acquire(sub.cur->values, idx, c);
1803 }
1804
1805 void
1806 Converter::storeDst(int d, int c, Value *val)
1807 {
1808 const tgsi::Instruction::DstRegister dst = tgsi.getDst(d);
1809
1810 if (tgsi.getSaturate()) {
1811 mkOp1(OP_SAT, dstTy, val, val);
1812 }
1813
1814 Value *ptr = NULL;
1815 if (dst.isIndirect(0))
1816 ptr = shiftAddress(fetchSrc(dst.getIndirect(0), 0, NULL));
1817
1818 if (info->io.genUserClip > 0 &&
1819 dst.getFile() == TGSI_FILE_OUTPUT &&
1820 !dst.isIndirect(0) && dst.getIndex(0) == code->clipVertexOutput) {
1821 mkMov(clipVtx[c], val);
1822 val = clipVtx[c];
1823 }
1824
1825 storeDst(dst, c, val, ptr);
1826 }
1827
1828 void
1829 Converter::storeDst(const tgsi::Instruction::DstRegister dst, int c,
1830 Value *val, Value *ptr)
1831 {
1832 const unsigned f = dst.getFile();
1833 int idx = dst.getIndex(0);
1834 int idx2d = dst.is2D() ? dst.getIndex(1) : 0;
1835
1836 if (f == TGSI_FILE_SYSTEM_VALUE) {
1837 assert(!ptr);
1838 mkOp2(OP_WRSV, TYPE_U32, NULL, dstToSym(dst, c), val);
1839 } else
1840 if (f == TGSI_FILE_OUTPUT && prog->getType() != Program::TYPE_FRAGMENT) {
1841
1842 if (ptr || (info->out[idx].mask & (1 << c))) {
1843 /* Save the viewport index into a scratch register so that it can be
1844 exported at EMIT time */
1845 if (info->out[idx].sn == TGSI_SEMANTIC_VIEWPORT_INDEX &&
1846 viewport != NULL)
1847 mkOp1(OP_MOV, TYPE_U32, viewport, val);
1848 else
1849 mkStore(OP_EXPORT, TYPE_U32, dstToSym(dst, c), ptr, val)->perPatch =
1850 info->out[idx].patch;
1851 }
1852 } else
1853 if (f == TGSI_FILE_TEMPORARY ||
1854 f == TGSI_FILE_PREDICATE ||
1855 f == TGSI_FILE_ADDRESS ||
1856 f == TGSI_FILE_OUTPUT) {
1857 if (f == TGSI_FILE_TEMPORARY) {
1858 int arrayid = dst.getArrayId();
1859 if (!arrayid)
1860 arrayid = code->tempArrayId[idx];
1861 adjustTempIndex(arrayid, idx, idx2d);
1862 }
1863
1864 getArrayForFile(f, idx2d)->store(sub.cur->values, idx, c, ptr, val);
1865 } else {
1866 assert(!"invalid dst file");
1867 }
1868 }
1869
1870 #define FOR_EACH_DST_ENABLED_CHANNEL(d, chan, inst) \
1871 for (chan = 0; chan < 4; ++chan) \
1872 if (!inst.getDst(d).isMasked(chan))
1873
1874 Value *
1875 Converter::buildDot(int dim)
1876 {
1877 assert(dim > 0);
1878
1879 Value *src0 = fetchSrc(0, 0), *src1 = fetchSrc(1, 0);
1880 Value *dotp = getScratch();
1881
1882 mkOp2(OP_MUL, TYPE_F32, dotp, src0, src1);
1883
1884 for (int c = 1; c < dim; ++c) {
1885 src0 = fetchSrc(0, c);
1886 src1 = fetchSrc(1, c);
1887 mkOp3(OP_MAD, TYPE_F32, dotp, src0, src1, dotp);
1888 }
1889 return dotp;
1890 }
1891
1892 void
1893 Converter::insertConvergenceOps(BasicBlock *conv, BasicBlock *fork)
1894 {
1895 FlowInstruction *join = new_FlowInstruction(func, OP_JOIN, NULL);
1896 join->fixed = 1;
1897 conv->insertHead(join);
1898
1899 assert(!fork->joinAt);
1900 fork->joinAt = new_FlowInstruction(func, OP_JOINAT, conv);
1901 fork->insertBefore(fork->getExit(), fork->joinAt);
1902 }
1903
1904 void
1905 Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S)
1906 {
1907 unsigned rIdx = 0, sIdx = 0;
1908
1909 if (R >= 0)
1910 rIdx = tgsi.getSrc(R).getIndex(0);
1911 if (S >= 0)
1912 sIdx = tgsi.getSrc(S).getIndex(0);
1913
1914 tex->setTexture(tgsi.getTexture(code, R), rIdx, sIdx);
1915
1916 if (tgsi.getSrc(R).isIndirect(0)) {
1917 tex->tex.rIndirectSrc = s;
1918 tex->setSrc(s++, fetchSrc(tgsi.getSrc(R).getIndirect(0), 0, NULL));
1919 }
1920 if (S >= 0 && tgsi.getSrc(S).isIndirect(0)) {
1921 tex->tex.sIndirectSrc = s;
1922 tex->setSrc(s++, fetchSrc(tgsi.getSrc(S).getIndirect(0), 0, NULL));
1923 }
1924 }
1925
1926 void
1927 Converter::handleTXQ(Value *dst0[4], enum TexQuery query, int R)
1928 {
1929 TexInstruction *tex = new_TexInstruction(func, OP_TXQ);
1930 tex->tex.query = query;
1931 unsigned int c, d;
1932
1933 for (d = 0, c = 0; c < 4; ++c) {
1934 if (!dst0[c])
1935 continue;
1936 tex->tex.mask |= 1 << c;
1937 tex->setDef(d++, dst0[c]);
1938 }
1939 if (query == TXQ_DIMS)
1940 tex->setSrc((c = 0), fetchSrc(0, 0)); // mip level
1941 else
1942 tex->setSrc((c = 0), zero);
1943
1944 setTexRS(tex, ++c, R, -1);
1945
1946 bb->insertTail(tex);
1947 }
1948
1949 void
1950 Converter::loadProjTexCoords(Value *dst[4], Value *src[4], unsigned int mask)
1951 {
1952 Value *proj = fetchSrc(0, 3);
1953 Instruction *insn = proj->getUniqueInsn();
1954 int c;
1955
1956 if (insn->op == OP_PINTERP) {
1957 bb->insertTail(insn = cloneForward(func, insn));
1958 insn->op = OP_LINTERP;
1959 insn->setInterpolate(NV50_IR_INTERP_LINEAR | insn->getSampleMode());
1960 insn->setSrc(1, NULL);
1961 proj = insn->getDef(0);
1962 }
1963 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), proj);
1964
1965 for (c = 0; c < 4; ++c) {
1966 if (!(mask & (1 << c)))
1967 continue;
1968 if ((insn = src[c]->getUniqueInsn())->op != OP_PINTERP)
1969 continue;
1970 mask &= ~(1 << c);
1971
1972 bb->insertTail(insn = cloneForward(func, insn));
1973 insn->setInterpolate(NV50_IR_INTERP_PERSPECTIVE | insn->getSampleMode());
1974 insn->setSrc(1, proj);
1975 dst[c] = insn->getDef(0);
1976 }
1977 if (!mask)
1978 return;
1979
1980 proj = mkOp1v(OP_RCP, TYPE_F32, getSSA(), fetchSrc(0, 3));
1981
1982 for (c = 0; c < 4; ++c)
1983 if (mask & (1 << c))
1984 dst[c] = mkOp2v(OP_MUL, TYPE_F32, getSSA(), src[c], proj);
1985 }
1986
1987 // order of nv50 ir sources: x y z layer lod/bias shadow
1988 // order of TGSI TEX sources: x y z layer shadow lod/bias
1989 // lowering will finally set the hw specific order (like array first on nvc0)
1990 void
1991 Converter::handleTEX(Value *dst[4], int R, int S, int L, int C, int Dx, int Dy)
1992 {
1993 Value *arg[4], *src[8];
1994 Value *lod = NULL, *shd = NULL;
1995 unsigned int s, c, d;
1996 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
1997
1998 TexInstruction::Target tgt = tgsi.getTexture(code, R);
1999
2000 for (s = 0; s < tgt.getArgCount(); ++s)
2001 arg[s] = src[s] = fetchSrc(0, s);
2002
2003 if (texi->op == OP_TXL || texi->op == OP_TXB)
2004 lod = fetchSrc(L >> 4, L & 3);
2005
2006 if (C == 0x0f)
2007 C = 0x00 | MAX2(tgt.getArgCount(), 2); // guess DC src
2008
2009 if (tgsi.getOpcode() == TGSI_OPCODE_TG4 &&
2010 tgt == TEX_TARGET_CUBE_ARRAY_SHADOW)
2011 shd = fetchSrc(1, 0);
2012 else if (tgt.isShadow())
2013 shd = fetchSrc(C >> 4, C & 3);
2014
2015 if (texi->op == OP_TXD) {
2016 for (c = 0; c < tgt.getDim() + tgt.isCube(); ++c) {
2017 texi->dPdx[c].set(fetchSrc(Dx >> 4, (Dx & 3) + c));
2018 texi->dPdy[c].set(fetchSrc(Dy >> 4, (Dy & 3) + c));
2019 }
2020 }
2021
2022 // cube textures don't care about projection value, it's divided out
2023 if (tgsi.getOpcode() == TGSI_OPCODE_TXP && !tgt.isCube() && !tgt.isArray()) {
2024 unsigned int n = tgt.getDim();
2025 if (shd) {
2026 arg[n] = shd;
2027 ++n;
2028 assert(tgt.getDim() == tgt.getArgCount());
2029 }
2030 loadProjTexCoords(src, arg, (1 << n) - 1);
2031 if (shd)
2032 shd = src[n - 1];
2033 }
2034
2035 for (c = 0, d = 0; c < 4; ++c) {
2036 if (dst[c]) {
2037 texi->setDef(d++, dst[c]);
2038 texi->tex.mask |= 1 << c;
2039 } else {
2040 // NOTE: maybe hook up def too, for CSE
2041 }
2042 }
2043 for (s = 0; s < tgt.getArgCount(); ++s)
2044 texi->setSrc(s, src[s]);
2045 if (lod)
2046 texi->setSrc(s++, lod);
2047 if (shd)
2048 texi->setSrc(s++, shd);
2049
2050 setTexRS(texi, s, R, S);
2051
2052 if (tgsi.getOpcode() == TGSI_OPCODE_SAMPLE_C_LZ)
2053 texi->tex.levelZero = true;
2054 if (tgsi.getOpcode() == TGSI_OPCODE_TG4 && !tgt.isShadow())
2055 texi->tex.gatherComp = tgsi.getSrc(1).getValueU32(0, info);
2056
2057 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2058 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2059 for (c = 0; c < 3; ++c) {
2060 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2061 texi->offset[s][c].setInsn(texi);
2062 }
2063 }
2064
2065 bb->insertTail(texi);
2066 }
2067
2068 // 1st source: xyz = coordinates, w = lod/sample
2069 // 2nd source: offset
2070 void
2071 Converter::handleTXF(Value *dst[4], int R, int L_M)
2072 {
2073 TexInstruction *texi = new_TexInstruction(func, tgsi.getOP());
2074 int ms;
2075 unsigned int c, d, s;
2076
2077 texi->tex.target = tgsi.getTexture(code, R);
2078
2079 ms = texi->tex.target.isMS() ? 1 : 0;
2080 texi->tex.levelZero = ms; /* MS textures don't have mip-maps */
2081
2082 for (c = 0, d = 0; c < 4; ++c) {
2083 if (dst[c]) {
2084 texi->setDef(d++, dst[c]);
2085 texi->tex.mask |= 1 << c;
2086 }
2087 }
2088 for (c = 0; c < (texi->tex.target.getArgCount() - ms); ++c)
2089 texi->setSrc(c, fetchSrc(0, c));
2090 texi->setSrc(c++, fetchSrc(L_M >> 4, L_M & 3)); // lod or ms
2091
2092 setTexRS(texi, c, R, -1);
2093
2094 texi->tex.useOffsets = tgsi.getNumTexOffsets();
2095 for (s = 0; s < tgsi.getNumTexOffsets(); ++s) {
2096 for (c = 0; c < 3; ++c) {
2097 texi->offset[s][c].set(fetchSrc(tgsi.getTexOffset(s), c, NULL));
2098 texi->offset[s][c].setInsn(texi);
2099 }
2100 }
2101
2102 bb->insertTail(texi);
2103 }
2104
2105 void
2106 Converter::handleLIT(Value *dst0[4])
2107 {
2108 Value *val0 = NULL;
2109 unsigned int mask = tgsi.getDst(0).getMask();
2110
2111 if (mask & (1 << 0))
2112 loadImm(dst0[0], 1.0f);
2113
2114 if (mask & (1 << 3))
2115 loadImm(dst0[3], 1.0f);
2116
2117 if (mask & (3 << 1)) {
2118 val0 = getScratch();
2119 mkOp2(OP_MAX, TYPE_F32, val0, fetchSrc(0, 0), zero);
2120 if (mask & (1 << 1))
2121 mkMov(dst0[1], val0);
2122 }
2123
2124 if (mask & (1 << 2)) {
2125 Value *src1 = fetchSrc(0, 1), *src3 = fetchSrc(0, 3);
2126 Value *val1 = getScratch(), *val3 = getScratch();
2127
2128 Value *pos128 = loadImm(NULL, +127.999999f);
2129 Value *neg128 = loadImm(NULL, -127.999999f);
2130
2131 mkOp2(OP_MAX, TYPE_F32, val1, src1, zero);
2132 mkOp2(OP_MAX, TYPE_F32, val3, src3, neg128);
2133 mkOp2(OP_MIN, TYPE_F32, val3, val3, pos128);
2134 mkOp2(OP_POW, TYPE_F32, val3, val1, val3);
2135
2136 mkCmp(OP_SLCT, CC_GT, TYPE_F32, dst0[2], TYPE_F32, val3, zero, val0);
2137 }
2138 }
2139
2140 static inline bool
2141 isResourceSpecial(const int r)
2142 {
2143 return (r == TGSI_RESOURCE_GLOBAL ||
2144 r == TGSI_RESOURCE_LOCAL ||
2145 r == TGSI_RESOURCE_PRIVATE ||
2146 r == TGSI_RESOURCE_INPUT);
2147 }
2148
2149 static inline bool
2150 isResourceRaw(const tgsi::Source *code, const int r)
2151 {
2152 return isResourceSpecial(r) || code->resources[r].raw;
2153 }
2154
2155 static inline nv50_ir::TexTarget
2156 getResourceTarget(const tgsi::Source *code, int r)
2157 {
2158 if (isResourceSpecial(r))
2159 return nv50_ir::TEX_TARGET_BUFFER;
2160 return tgsi::translateTexture(code->resources.at(r).target);
2161 }
2162
2163 Symbol *
2164 Converter::getResourceBase(const int r)
2165 {
2166 Symbol *sym = NULL;
2167
2168 switch (r) {
2169 case TGSI_RESOURCE_GLOBAL:
2170 sym = new_Symbol(prog, nv50_ir::FILE_MEMORY_GLOBAL,
2171 info->io.auxCBSlot);
2172 break;
2173 case TGSI_RESOURCE_LOCAL:
2174 assert(prog->getType() == Program::TYPE_COMPUTE);
2175 sym = mkSymbol(nv50_ir::FILE_MEMORY_SHARED, 0, TYPE_U32,
2176 info->prop.cp.sharedOffset);
2177 break;
2178 case TGSI_RESOURCE_PRIVATE:
2179 sym = mkSymbol(nv50_ir::FILE_MEMORY_LOCAL, 0, TYPE_U32,
2180 info->bin.tlsSpace);
2181 break;
2182 case TGSI_RESOURCE_INPUT:
2183 assert(prog->getType() == Program::TYPE_COMPUTE);
2184 sym = mkSymbol(nv50_ir::FILE_SHADER_INPUT, 0, TYPE_U32,
2185 info->prop.cp.inputOffset);
2186 break;
2187 default:
2188 sym = new_Symbol(prog,
2189 nv50_ir::FILE_MEMORY_GLOBAL, code->resources.at(r).slot);
2190 break;
2191 }
2192 return sym;
2193 }
2194
2195 void
2196 Converter::getResourceCoords(std::vector<Value *> &coords, int r, int s)
2197 {
2198 const int arg =
2199 TexInstruction::Target(getResourceTarget(code, r)).getArgCount();
2200
2201 for (int c = 0; c < arg; ++c)
2202 coords.push_back(fetchSrc(s, c));
2203
2204 // NOTE: TGSI_RESOURCE_GLOBAL needs FILE_GPR; this is an nv50 quirk
2205 if (r == TGSI_RESOURCE_LOCAL ||
2206 r == TGSI_RESOURCE_PRIVATE ||
2207 r == TGSI_RESOURCE_INPUT)
2208 coords[0] = mkOp1v(OP_MOV, TYPE_U32, getScratch(4, FILE_ADDRESS),
2209 coords[0]);
2210 }
2211
2212 static inline int
2213 partitionLoadStore(uint8_t comp[2], uint8_t size[2], uint8_t mask)
2214 {
2215 int n = 0;
2216
2217 while (mask) {
2218 if (mask & 1) {
2219 size[n]++;
2220 } else {
2221 if (size[n])
2222 comp[n = 1] = size[0] + 1;
2223 else
2224 comp[n]++;
2225 }
2226 mask >>= 1;
2227 }
2228 if (size[0] == 3) {
2229 n = 1;
2230 size[0] = (comp[0] == 1) ? 1 : 2;
2231 size[1] = 3 - size[0];
2232 comp[1] = comp[0] + size[0];
2233 }
2234 return n + 1;
2235 }
2236
2237 // For raw loads, granularity is 4 byte.
2238 // Usage of the texture read mask on OP_SULDP is not allowed.
2239 void
2240 Converter::handleLOAD(Value *dst0[4])
2241 {
2242 const int r = tgsi.getSrc(0).getIndex(0);
2243 int c;
2244 std::vector<Value *> off, src, ldv, def;
2245
2246 if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER ||
2247 tgsi.getSrc(0).getFile() == TGSI_FILE_MEMORY) {
2248 for (c = 0; c < 4; ++c) {
2249 if (!dst0[c])
2250 continue;
2251
2252 Value *off = fetchSrc(1, c);
2253 Symbol *sym;
2254 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE) {
2255 off = NULL;
2256 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2257 tgsi.getSrc(1).getValueU32(0, info) + 4 * c);
2258 } else {
2259 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 4 * c);
2260 }
2261
2262 Instruction *ld = mkLoad(TYPE_U32, dst0[c], sym, off);
2263 ld->cache = tgsi.getCacheMode();
2264 if (tgsi.getSrc(0).isIndirect(0))
2265 ld->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
2266 }
2267 return;
2268 }
2269
2270 getResourceCoords(off, r, 1);
2271
2272 if (isResourceRaw(code, r)) {
2273 uint8_t mask = 0;
2274 uint8_t comp[2] = { 0, 0 };
2275 uint8_t size[2] = { 0, 0 };
2276
2277 Symbol *base = getResourceBase(r);
2278
2279 // determine the base and size of the at most 2 load ops
2280 for (c = 0; c < 4; ++c)
2281 if (!tgsi.getDst(0).isMasked(c))
2282 mask |= 1 << (tgsi.getSrc(0).getSwizzle(c) - TGSI_SWIZZLE_X);
2283
2284 int n = partitionLoadStore(comp, size, mask);
2285
2286 src = off;
2287
2288 def.resize(4); // index by component, the ones we need will be non-NULL
2289 for (c = 0; c < 4; ++c) {
2290 if (dst0[c] && tgsi.getSrc(0).getSwizzle(c) == (TGSI_SWIZZLE_X + c))
2291 def[c] = dst0[c];
2292 else
2293 if (mask & (1 << c))
2294 def[c] = getScratch();
2295 }
2296
2297 const bool useLd = isResourceSpecial(r) ||
2298 (info->io.nv50styleSurfaces &&
2299 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2300
2301 for (int i = 0; i < n; ++i) {
2302 ldv.assign(def.begin() + comp[i], def.begin() + comp[i] + size[i]);
2303
2304 if (comp[i]) // adjust x component of source address if necessary
2305 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2306 off[0], mkImm(comp[i] * 4));
2307 else
2308 src[0] = off[0];
2309
2310 if (useLd) {
2311 Instruction *ld =
2312 mkLoad(typeOfSize(size[i] * 4), ldv[0], base, src[0]);
2313 for (size_t c = 1; c < ldv.size(); ++c)
2314 ld->setDef(c, ldv[c]);
2315 } else {
2316 mkTex(OP_SULDB, getResourceTarget(code, r), code->resources[r].slot,
2317 0, ldv, src)->dType = typeOfSize(size[i] * 4);
2318 }
2319 }
2320 } else {
2321 def.resize(4);
2322 for (c = 0; c < 4; ++c) {
2323 if (!dst0[c] || tgsi.getSrc(0).getSwizzle(c) != (TGSI_SWIZZLE_X + c))
2324 def[c] = getScratch();
2325 else
2326 def[c] = dst0[c];
2327 }
2328
2329 mkTex(OP_SULDP, getResourceTarget(code, r), code->resources[r].slot, 0,
2330 def, off);
2331 }
2332 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2333 if (dst0[c] != def[c])
2334 mkMov(dst0[c], def[tgsi.getSrc(0).getSwizzle(c)]);
2335 }
2336
2337 // For formatted stores, the write mask on OP_SUSTP can be used.
2338 // Raw stores have to be split.
2339 void
2340 Converter::handleSTORE()
2341 {
2342 const int r = tgsi.getDst(0).getIndex(0);
2343 int c;
2344 std::vector<Value *> off, src, dummy;
2345
2346 if (tgsi.getDst(0).getFile() == TGSI_FILE_BUFFER ||
2347 tgsi.getDst(0).getFile() == TGSI_FILE_MEMORY) {
2348 for (c = 0; c < 4; ++c) {
2349 if (!(tgsi.getDst(0).getMask() & (1 << c)))
2350 continue;
2351
2352 Symbol *sym;
2353 Value *off;
2354 if (tgsi.getSrc(0).getFile() == TGSI_FILE_IMMEDIATE) {
2355 off = NULL;
2356 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c,
2357 tgsi.getSrc(0).getValueU32(0, info) + 4 * c);
2358 } else {
2359 off = fetchSrc(0, 0);
2360 sym = makeSym(tgsi.getDst(0).getFile(), r, -1, c, 4 * c);
2361 }
2362
2363 Instruction *st = mkStore(OP_STORE, TYPE_U32, sym, off, fetchSrc(1, c));
2364 st->cache = tgsi.getCacheMode();
2365 if (tgsi.getDst(0).isIndirect(0))
2366 st->setIndirect(0, 1, fetchSrc(tgsi.getDst(0).getIndirect(0), 0, 0));
2367 }
2368 return;
2369 }
2370
2371 getResourceCoords(off, r, 0);
2372 src = off;
2373 const int s = src.size();
2374
2375 if (isResourceRaw(code, r)) {
2376 uint8_t comp[2] = { 0, 0 };
2377 uint8_t size[2] = { 0, 0 };
2378
2379 int n = partitionLoadStore(comp, size, tgsi.getDst(0).getMask());
2380
2381 Symbol *base = getResourceBase(r);
2382
2383 const bool useSt = isResourceSpecial(r) ||
2384 (info->io.nv50styleSurfaces &&
2385 code->resources[r].target == TGSI_TEXTURE_BUFFER);
2386
2387 for (int i = 0; i < n; ++i) {
2388 if (comp[i]) // adjust x component of source address if necessary
2389 src[0] = mkOp2v(OP_ADD, TYPE_U32, getSSA(4, off[0]->reg.file),
2390 off[0], mkImm(comp[i] * 4));
2391 else
2392 src[0] = off[0];
2393
2394 const DataType stTy = typeOfSize(size[i] * 4);
2395
2396 if (useSt) {
2397 Instruction *st =
2398 mkStore(OP_STORE, stTy, base, NULL, fetchSrc(1, comp[i]));
2399 for (c = 1; c < size[i]; ++c)
2400 st->setSrc(1 + c, fetchSrc(1, comp[i] + c));
2401 st->setIndirect(0, 0, src[0]);
2402 } else {
2403 // attach values to be stored
2404 src.resize(s + size[i]);
2405 for (c = 0; c < size[i]; ++c)
2406 src[s + c] = fetchSrc(1, comp[i] + c);
2407 mkTex(OP_SUSTB, getResourceTarget(code, r), code->resources[r].slot,
2408 0, dummy, src)->setType(stTy);
2409 }
2410 }
2411 } else {
2412 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2413 src.push_back(fetchSrc(1, c));
2414
2415 mkTex(OP_SUSTP, getResourceTarget(code, r), code->resources[r].slot, 0,
2416 dummy, src)->tex.mask = tgsi.getDst(0).getMask();
2417 }
2418 }
2419
2420 // XXX: These only work on resources with the single-component u32/s32 formats.
2421 // Therefore the result is replicated. This might not be intended by TGSI, but
2422 // operating on more than 1 component would produce undefined results because
2423 // they do not exist.
2424 void
2425 Converter::handleATOM(Value *dst0[4], DataType ty, uint16_t subOp)
2426 {
2427 const int r = tgsi.getSrc(0).getIndex(0);
2428 std::vector<Value *> srcv;
2429 std::vector<Value *> defv;
2430 LValue *dst = getScratch();
2431
2432 if (tgsi.getSrc(0).getFile() == TGSI_FILE_BUFFER ||
2433 tgsi.getSrc(0).getFile() == TGSI_FILE_MEMORY) {
2434 for (int c = 0; c < 4; ++c) {
2435 if (!dst0[c])
2436 continue;
2437
2438 Instruction *insn;
2439 Value *off = fetchSrc(1, c), *off2 = NULL;
2440 Value *sym;
2441 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE)
2442 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c,
2443 tgsi.getSrc(1).getValueU32(c, info));
2444 else
2445 sym = makeSym(tgsi.getSrc(0).getFile(), r, -1, c, 0);
2446 if (tgsi.getSrc(0).isIndirect(0))
2447 off2 = fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0);
2448 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2449 insn = mkOp3(OP_ATOM, ty, dst, sym, fetchSrc(2, c), fetchSrc(3, c));
2450 else
2451 insn = mkOp2(OP_ATOM, ty, dst, sym, fetchSrc(2, c));
2452 if (tgsi.getSrc(1).getFile() != TGSI_FILE_IMMEDIATE)
2453 insn->setIndirect(0, 0, off);
2454 if (off2)
2455 insn->setIndirect(0, 1, off2);
2456 insn->subOp = subOp;
2457 }
2458 for (int c = 0; c < 4; ++c)
2459 if (dst0[c])
2460 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2461 return;
2462 }
2463
2464
2465 getResourceCoords(srcv, r, 1);
2466
2467 if (isResourceSpecial(r)) {
2468 assert(r != TGSI_RESOURCE_INPUT);
2469 Instruction *insn;
2470 insn = mkOp2(OP_ATOM, ty, dst, getResourceBase(r), fetchSrc(2, 0));
2471 insn->subOp = subOp;
2472 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2473 insn->setSrc(2, fetchSrc(3, 0));
2474 insn->setIndirect(0, 0, srcv.at(0));
2475 } else {
2476 operation op = isResourceRaw(code, r) ? OP_SUREDB : OP_SUREDP;
2477 TexTarget targ = getResourceTarget(code, r);
2478 int idx = code->resources[r].slot;
2479 defv.push_back(dst);
2480 srcv.push_back(fetchSrc(2, 0));
2481 if (subOp == NV50_IR_SUBOP_ATOM_CAS)
2482 srcv.push_back(fetchSrc(3, 0));
2483 TexInstruction *tex = mkTex(op, targ, idx, 0, defv, srcv);
2484 tex->subOp = subOp;
2485 tex->tex.mask = 1;
2486 tex->setType(ty);
2487 }
2488
2489 for (int c = 0; c < 4; ++c)
2490 if (dst0[c])
2491 dst0[c] = dst; // not equal to rDst so handleInstruction will do mkMov
2492 }
2493
2494 void
2495 Converter::handleINTERP(Value *dst[4])
2496 {
2497 // Check whether the input is linear. All other attributes ignored.
2498 Instruction *insn;
2499 Value *offset = NULL, *ptr = NULL, *w = NULL;
2500 bool linear;
2501 operation op;
2502 int c, mode;
2503
2504 tgsi::Instruction::SrcRegister src = tgsi.getSrc(0);
2505 assert(src.getFile() == TGSI_FILE_INPUT);
2506
2507 if (src.isIndirect(0))
2508 ptr = fetchSrc(src.getIndirect(0), 0, NULL);
2509
2510 // XXX: no way to know interp mode if we don't know the index
2511 linear = info->in[ptr ? 0 : src.getIndex(0)].linear;
2512 if (linear) {
2513 op = OP_LINTERP;
2514 mode = NV50_IR_INTERP_LINEAR;
2515 } else {
2516 op = OP_PINTERP;
2517 mode = NV50_IR_INTERP_PERSPECTIVE;
2518 }
2519
2520 switch (tgsi.getOpcode()) {
2521 case TGSI_OPCODE_INTERP_CENTROID:
2522 mode |= NV50_IR_INTERP_CENTROID;
2523 break;
2524 case TGSI_OPCODE_INTERP_SAMPLE:
2525 insn = mkOp1(OP_PIXLD, TYPE_U32, (offset = getScratch()), fetchSrc(1, 0));
2526 insn->subOp = NV50_IR_SUBOP_PIXLD_OFFSET;
2527 mode |= NV50_IR_INTERP_OFFSET;
2528 break;
2529 case TGSI_OPCODE_INTERP_OFFSET: {
2530 // The input in src1.xy is float, but we need a single 32-bit value
2531 // where the upper and lower 16 bits are encoded in S0.12 format. We need
2532 // to clamp the input coordinates to (-0.5, 0.4375), multiply by 4096,
2533 // and then convert to s32.
2534 Value *offs[2];
2535 for (c = 0; c < 2; c++) {
2536 offs[c] = fetchSrc(1, c);
2537 mkOp2(OP_MIN, TYPE_F32, offs[c], offs[c], loadImm(NULL, 0.4375f));
2538 mkOp2(OP_MAX, TYPE_F32, offs[c], offs[c], loadImm(NULL, -0.5f));
2539 mkOp2(OP_MUL, TYPE_F32, offs[c], offs[c], loadImm(NULL, 4096.0f));
2540 mkCvt(OP_CVT, TYPE_S32, offs[c], TYPE_F32, offs[c]);
2541 }
2542 offset = mkOp3v(OP_INSBF, TYPE_U32, getScratch(),
2543 offs[1], mkImm(0x1010), offs[0]);
2544 mode |= NV50_IR_INTERP_OFFSET;
2545 break;
2546 }
2547 }
2548
2549 if (op == OP_PINTERP) {
2550 if (offset) {
2551 w = mkOp2v(OP_RDSV, TYPE_F32, getSSA(), mkSysVal(SV_POSITION, 3), offset);
2552 mkOp1(OP_RCP, TYPE_F32, w, w);
2553 } else {
2554 w = fragCoord[3];
2555 }
2556 }
2557
2558
2559 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2560 insn = mkOp1(op, TYPE_F32, dst[c], srcToSym(src, c));
2561 if (op == OP_PINTERP)
2562 insn->setSrc(1, w);
2563 if (ptr)
2564 insn->setIndirect(0, 0, ptr);
2565 if (offset)
2566 insn->setSrc(op == OP_PINTERP ? 2 : 1, offset);
2567
2568 insn->setInterpolate(mode);
2569 }
2570 }
2571
2572 Converter::Subroutine *
2573 Converter::getSubroutine(unsigned ip)
2574 {
2575 std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2576
2577 if (it == sub.map.end())
2578 it = sub.map.insert(std::make_pair(
2579 ip, Subroutine(new Function(prog, "SUB", ip)))).first;
2580
2581 return &it->second;
2582 }
2583
2584 Converter::Subroutine *
2585 Converter::getSubroutine(Function *f)
2586 {
2587 unsigned ip = f->getLabel();
2588 std::map<unsigned, Subroutine>::iterator it = sub.map.find(ip);
2589
2590 if (it == sub.map.end())
2591 it = sub.map.insert(std::make_pair(ip, Subroutine(f))).first;
2592
2593 return &it->second;
2594 }
2595
2596 bool
2597 Converter::isEndOfSubroutine(uint ip)
2598 {
2599 assert(ip < code->scan.num_instructions);
2600 tgsi::Instruction insn(&code->insns[ip]);
2601 return (insn.getOpcode() == TGSI_OPCODE_END ||
2602 insn.getOpcode() == TGSI_OPCODE_ENDSUB ||
2603 // does END occur at end of main or the very end ?
2604 insn.getOpcode() == TGSI_OPCODE_BGNSUB);
2605 }
2606
2607 bool
2608 Converter::handleInstruction(const struct tgsi_full_instruction *insn)
2609 {
2610 Instruction *geni;
2611
2612 Value *dst0[4], *rDst0[4];
2613 Value *src0, *src1, *src2, *src3;
2614 Value *val0, *val1;
2615 int c;
2616
2617 tgsi = tgsi::Instruction(insn);
2618
2619 bool useScratchDst = tgsi.checkDstSrcAliasing();
2620
2621 operation op = tgsi.getOP();
2622 dstTy = tgsi.inferDstType();
2623 srcTy = tgsi.inferSrcType();
2624
2625 unsigned int mask = tgsi.dstCount() ? tgsi.getDst(0).getMask() : 0;
2626
2627 if (tgsi.dstCount()) {
2628 for (c = 0; c < 4; ++c) {
2629 rDst0[c] = acquireDst(0, c);
2630 dst0[c] = (useScratchDst && rDst0[c]) ? getScratch() : rDst0[c];
2631 }
2632 }
2633
2634 switch (tgsi.getOpcode()) {
2635 case TGSI_OPCODE_ADD:
2636 case TGSI_OPCODE_UADD:
2637 case TGSI_OPCODE_AND:
2638 case TGSI_OPCODE_DIV:
2639 case TGSI_OPCODE_IDIV:
2640 case TGSI_OPCODE_UDIV:
2641 case TGSI_OPCODE_MAX:
2642 case TGSI_OPCODE_MIN:
2643 case TGSI_OPCODE_IMAX:
2644 case TGSI_OPCODE_IMIN:
2645 case TGSI_OPCODE_UMAX:
2646 case TGSI_OPCODE_UMIN:
2647 case TGSI_OPCODE_MOD:
2648 case TGSI_OPCODE_UMOD:
2649 case TGSI_OPCODE_MUL:
2650 case TGSI_OPCODE_UMUL:
2651 case TGSI_OPCODE_IMUL_HI:
2652 case TGSI_OPCODE_UMUL_HI:
2653 case TGSI_OPCODE_OR:
2654 case TGSI_OPCODE_SHL:
2655 case TGSI_OPCODE_ISHR:
2656 case TGSI_OPCODE_USHR:
2657 case TGSI_OPCODE_SUB:
2658 case TGSI_OPCODE_XOR:
2659 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2660 src0 = fetchSrc(0, c);
2661 src1 = fetchSrc(1, c);
2662 geni = mkOp2(op, dstTy, dst0[c], src0, src1);
2663 geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
2664 }
2665 break;
2666 case TGSI_OPCODE_MAD:
2667 case TGSI_OPCODE_UMAD:
2668 case TGSI_OPCODE_SAD:
2669 case TGSI_OPCODE_FMA:
2670 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2671 src0 = fetchSrc(0, c);
2672 src1 = fetchSrc(1, c);
2673 src2 = fetchSrc(2, c);
2674 mkOp3(op, dstTy, dst0[c], src0, src1, src2);
2675 }
2676 break;
2677 case TGSI_OPCODE_MOV:
2678 case TGSI_OPCODE_ABS:
2679 case TGSI_OPCODE_CEIL:
2680 case TGSI_OPCODE_FLR:
2681 case TGSI_OPCODE_TRUNC:
2682 case TGSI_OPCODE_RCP:
2683 case TGSI_OPCODE_SQRT:
2684 case TGSI_OPCODE_IABS:
2685 case TGSI_OPCODE_INEG:
2686 case TGSI_OPCODE_NOT:
2687 case TGSI_OPCODE_DDX:
2688 case TGSI_OPCODE_DDY:
2689 case TGSI_OPCODE_DDX_FINE:
2690 case TGSI_OPCODE_DDY_FINE:
2691 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2692 mkOp1(op, dstTy, dst0[c], fetchSrc(0, c));
2693 break;
2694 case TGSI_OPCODE_RSQ:
2695 src0 = fetchSrc(0, 0);
2696 val0 = getScratch();
2697 mkOp1(OP_ABS, TYPE_F32, val0, src0);
2698 mkOp1(OP_RSQ, TYPE_F32, val0, val0);
2699 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2700 mkMov(dst0[c], val0);
2701 break;
2702 case TGSI_OPCODE_ARL:
2703 case TGSI_OPCODE_ARR:
2704 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2705 const RoundMode rnd =
2706 tgsi.getOpcode() == TGSI_OPCODE_ARR ? ROUND_N : ROUND_M;
2707 src0 = fetchSrc(0, c);
2708 mkCvt(OP_CVT, TYPE_S32, dst0[c], TYPE_F32, src0)->rnd = rnd;
2709 }
2710 break;
2711 case TGSI_OPCODE_UARL:
2712 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2713 mkOp1(OP_MOV, TYPE_U32, dst0[c], fetchSrc(0, c));
2714 break;
2715 case TGSI_OPCODE_POW:
2716 val0 = mkOp2v(op, TYPE_F32, getScratch(), fetchSrc(0, 0), fetchSrc(1, 0));
2717 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2718 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2719 break;
2720 case TGSI_OPCODE_EX2:
2721 case TGSI_OPCODE_LG2:
2722 val0 = mkOp1(op, TYPE_F32, getScratch(), fetchSrc(0, 0))->getDef(0);
2723 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2724 mkOp1(OP_MOV, TYPE_F32, dst0[c], val0);
2725 break;
2726 case TGSI_OPCODE_COS:
2727 case TGSI_OPCODE_SIN:
2728 val0 = getScratch();
2729 if (mask & 7) {
2730 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 0));
2731 mkOp1(op, TYPE_F32, val0, val0);
2732 for (c = 0; c < 3; ++c)
2733 if (dst0[c])
2734 mkMov(dst0[c], val0);
2735 }
2736 if (dst0[3]) {
2737 mkOp1(OP_PRESIN, TYPE_F32, val0, fetchSrc(0, 3));
2738 mkOp1(op, TYPE_F32, dst0[3], val0);
2739 }
2740 break;
2741 case TGSI_OPCODE_SCS:
2742 if (mask & 3) {
2743 val0 = mkOp1v(OP_PRESIN, TYPE_F32, getSSA(), fetchSrc(0, 0));
2744 if (dst0[0])
2745 mkOp1(OP_COS, TYPE_F32, dst0[0], val0);
2746 if (dst0[1])
2747 mkOp1(OP_SIN, TYPE_F32, dst0[1], val0);
2748 }
2749 if (dst0[2])
2750 loadImm(dst0[2], 0.0f);
2751 if (dst0[3])
2752 loadImm(dst0[3], 1.0f);
2753 break;
2754 case TGSI_OPCODE_EXP:
2755 src0 = fetchSrc(0, 0);
2756 val0 = mkOp1v(OP_FLOOR, TYPE_F32, getSSA(), src0);
2757 if (dst0[1])
2758 mkOp2(OP_SUB, TYPE_F32, dst0[1], src0, val0);
2759 if (dst0[0])
2760 mkOp1(OP_EX2, TYPE_F32, dst0[0], val0);
2761 if (dst0[2])
2762 mkOp1(OP_EX2, TYPE_F32, dst0[2], src0);
2763 if (dst0[3])
2764 loadImm(dst0[3], 1.0f);
2765 break;
2766 case TGSI_OPCODE_LOG:
2767 src0 = mkOp1v(OP_ABS, TYPE_F32, getSSA(), fetchSrc(0, 0));
2768 val0 = mkOp1v(OP_LG2, TYPE_F32, dst0[2] ? dst0[2] : getSSA(), src0);
2769 if (dst0[0] || dst0[1])
2770 val1 = mkOp1v(OP_FLOOR, TYPE_F32, dst0[0] ? dst0[0] : getSSA(), val0);
2771 if (dst0[1]) {
2772 mkOp1(OP_EX2, TYPE_F32, dst0[1], val1);
2773 mkOp1(OP_RCP, TYPE_F32, dst0[1], dst0[1]);
2774 mkOp2(OP_MUL, TYPE_F32, dst0[1], dst0[1], src0);
2775 }
2776 if (dst0[3])
2777 loadImm(dst0[3], 1.0f);
2778 break;
2779 case TGSI_OPCODE_DP2:
2780 val0 = buildDot(2);
2781 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2782 mkMov(dst0[c], val0);
2783 break;
2784 case TGSI_OPCODE_DP3:
2785 val0 = buildDot(3);
2786 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2787 mkMov(dst0[c], val0);
2788 break;
2789 case TGSI_OPCODE_DP4:
2790 val0 = buildDot(4);
2791 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2792 mkMov(dst0[c], val0);
2793 break;
2794 case TGSI_OPCODE_DPH:
2795 val0 = buildDot(3);
2796 src1 = fetchSrc(1, 3);
2797 mkOp2(OP_ADD, TYPE_F32, val0, val0, src1);
2798 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2799 mkMov(dst0[c], val0);
2800 break;
2801 case TGSI_OPCODE_DST:
2802 if (dst0[0])
2803 loadImm(dst0[0], 1.0f);
2804 if (dst0[1]) {
2805 src0 = fetchSrc(0, 1);
2806 src1 = fetchSrc(1, 1);
2807 mkOp2(OP_MUL, TYPE_F32, dst0[1], src0, src1);
2808 }
2809 if (dst0[2])
2810 mkMov(dst0[2], fetchSrc(0, 2));
2811 if (dst0[3])
2812 mkMov(dst0[3], fetchSrc(1, 3));
2813 break;
2814 case TGSI_OPCODE_LRP:
2815 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2816 src0 = fetchSrc(0, c);
2817 src1 = fetchSrc(1, c);
2818 src2 = fetchSrc(2, c);
2819 mkOp3(OP_MAD, TYPE_F32, dst0[c],
2820 mkOp2v(OP_SUB, TYPE_F32, getSSA(), src1, src2), src0, src2);
2821 }
2822 break;
2823 case TGSI_OPCODE_LIT:
2824 handleLIT(dst0);
2825 break;
2826 case TGSI_OPCODE_XPD:
2827 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2828 if (c < 3) {
2829 val0 = getSSA();
2830 src0 = fetchSrc(1, (c + 1) % 3);
2831 src1 = fetchSrc(0, (c + 2) % 3);
2832 mkOp2(OP_MUL, TYPE_F32, val0, src0, src1);
2833 mkOp1(OP_NEG, TYPE_F32, val0, val0);
2834
2835 src0 = fetchSrc(0, (c + 1) % 3);
2836 src1 = fetchSrc(1, (c + 2) % 3);
2837 mkOp3(OP_MAD, TYPE_F32, dst0[c], src0, src1, val0);
2838 } else {
2839 loadImm(dst0[c], 1.0f);
2840 }
2841 }
2842 break;
2843 case TGSI_OPCODE_ISSG:
2844 case TGSI_OPCODE_SSG:
2845 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2846 src0 = fetchSrc(0, c);
2847 val0 = getScratch();
2848 val1 = getScratch();
2849 mkCmp(OP_SET, CC_GT, srcTy, val0, srcTy, src0, zero);
2850 mkCmp(OP_SET, CC_LT, srcTy, val1, srcTy, src0, zero);
2851 if (srcTy == TYPE_F32)
2852 mkOp2(OP_SUB, TYPE_F32, dst0[c], val0, val1);
2853 else
2854 mkOp2(OP_SUB, TYPE_S32, dst0[c], val1, val0);
2855 }
2856 break;
2857 case TGSI_OPCODE_UCMP:
2858 srcTy = TYPE_U32;
2859 /* fallthrough */
2860 case TGSI_OPCODE_CMP:
2861 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2862 src0 = fetchSrc(0, c);
2863 src1 = fetchSrc(1, c);
2864 src2 = fetchSrc(2, c);
2865 if (src1 == src2)
2866 mkMov(dst0[c], src1);
2867 else
2868 mkCmp(OP_SLCT, (srcTy == TYPE_F32) ? CC_LT : CC_NE,
2869 srcTy, dst0[c], srcTy, src1, src2, src0);
2870 }
2871 break;
2872 case TGSI_OPCODE_FRC:
2873 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2874 src0 = fetchSrc(0, c);
2875 val0 = getScratch();
2876 mkOp1(OP_FLOOR, TYPE_F32, val0, src0);
2877 mkOp2(OP_SUB, TYPE_F32, dst0[c], src0, val0);
2878 }
2879 break;
2880 case TGSI_OPCODE_ROUND:
2881 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2882 mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F32, fetchSrc(0, c))
2883 ->rnd = ROUND_NI;
2884 break;
2885 case TGSI_OPCODE_CLAMP:
2886 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2887 src0 = fetchSrc(0, c);
2888 src1 = fetchSrc(1, c);
2889 src2 = fetchSrc(2, c);
2890 val0 = getScratch();
2891 mkOp2(OP_MIN, TYPE_F32, val0, src0, src1);
2892 mkOp2(OP_MAX, TYPE_F32, dst0[c], val0, src2);
2893 }
2894 break;
2895 case TGSI_OPCODE_SLT:
2896 case TGSI_OPCODE_SGE:
2897 case TGSI_OPCODE_SEQ:
2898 case TGSI_OPCODE_SGT:
2899 case TGSI_OPCODE_SLE:
2900 case TGSI_OPCODE_SNE:
2901 case TGSI_OPCODE_FSEQ:
2902 case TGSI_OPCODE_FSGE:
2903 case TGSI_OPCODE_FSLT:
2904 case TGSI_OPCODE_FSNE:
2905 case TGSI_OPCODE_ISGE:
2906 case TGSI_OPCODE_ISLT:
2907 case TGSI_OPCODE_USEQ:
2908 case TGSI_OPCODE_USGE:
2909 case TGSI_OPCODE_USLT:
2910 case TGSI_OPCODE_USNE:
2911 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
2912 src0 = fetchSrc(0, c);
2913 src1 = fetchSrc(1, c);
2914 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
2915 }
2916 break;
2917 case TGSI_OPCODE_KILL_IF:
2918 val0 = new_LValue(func, FILE_PREDICATE);
2919 mask = 0;
2920 for (c = 0; c < 4; ++c) {
2921 const int s = tgsi.getSrc(0).getSwizzle(c);
2922 if (mask & (1 << s))
2923 continue;
2924 mask |= 1 << s;
2925 mkCmp(OP_SET, CC_LT, TYPE_F32, val0, TYPE_F32, fetchSrc(0, c), zero);
2926 mkOp(OP_DISCARD, TYPE_NONE, NULL)->setPredicate(CC_P, val0);
2927 }
2928 break;
2929 case TGSI_OPCODE_KILL:
2930 mkOp(OP_DISCARD, TYPE_NONE, NULL);
2931 break;
2932 case TGSI_OPCODE_TEX:
2933 case TGSI_OPCODE_TXB:
2934 case TGSI_OPCODE_TXL:
2935 case TGSI_OPCODE_TXP:
2936 case TGSI_OPCODE_LODQ:
2937 // R S L C Dx Dy
2938 handleTEX(dst0, 1, 1, 0x03, 0x0f, 0x00, 0x00);
2939 break;
2940 case TGSI_OPCODE_TXD:
2941 handleTEX(dst0, 3, 3, 0x03, 0x0f, 0x10, 0x20);
2942 break;
2943 case TGSI_OPCODE_TG4:
2944 handleTEX(dst0, 2, 2, 0x03, 0x0f, 0x00, 0x00);
2945 break;
2946 case TGSI_OPCODE_TEX2:
2947 handleTEX(dst0, 2, 2, 0x03, 0x10, 0x00, 0x00);
2948 break;
2949 case TGSI_OPCODE_TXB2:
2950 case TGSI_OPCODE_TXL2:
2951 handleTEX(dst0, 2, 2, 0x10, 0x0f, 0x00, 0x00);
2952 break;
2953 case TGSI_OPCODE_SAMPLE:
2954 case TGSI_OPCODE_SAMPLE_B:
2955 case TGSI_OPCODE_SAMPLE_D:
2956 case TGSI_OPCODE_SAMPLE_L:
2957 case TGSI_OPCODE_SAMPLE_C:
2958 case TGSI_OPCODE_SAMPLE_C_LZ:
2959 handleTEX(dst0, 1, 2, 0x30, 0x30, 0x30, 0x40);
2960 break;
2961 case TGSI_OPCODE_TXF:
2962 handleTXF(dst0, 1, 0x03);
2963 break;
2964 case TGSI_OPCODE_SAMPLE_I:
2965 handleTXF(dst0, 1, 0x03);
2966 break;
2967 case TGSI_OPCODE_SAMPLE_I_MS:
2968 handleTXF(dst0, 1, 0x20);
2969 break;
2970 case TGSI_OPCODE_TXQ:
2971 case TGSI_OPCODE_SVIEWINFO:
2972 handleTXQ(dst0, TXQ_DIMS, 1);
2973 break;
2974 case TGSI_OPCODE_TXQS:
2975 // The TXQ_TYPE query returns samples in its 3rd arg, but we need it to
2976 // be in .x
2977 dst0[1] = dst0[2] = dst0[3] = NULL;
2978 std::swap(dst0[0], dst0[2]);
2979 handleTXQ(dst0, TXQ_TYPE, 0);
2980 std::swap(dst0[0], dst0[2]);
2981 break;
2982 case TGSI_OPCODE_F2I:
2983 case TGSI_OPCODE_F2U:
2984 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2985 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c))->rnd = ROUND_Z;
2986 break;
2987 case TGSI_OPCODE_I2F:
2988 case TGSI_OPCODE_U2F:
2989 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2990 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, fetchSrc(0, c));
2991 break;
2992 case TGSI_OPCODE_PK2H:
2993 val0 = getScratch();
2994 val1 = getScratch();
2995 mkCvt(OP_CVT, TYPE_F16, val0, TYPE_F32, fetchSrc(0, 0));
2996 mkCvt(OP_CVT, TYPE_F16, val1, TYPE_F32, fetchSrc(0, 1));
2997 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi)
2998 mkOp3(OP_INSBF, TYPE_U32, dst0[c], val1, mkImm(0x1010), val0);
2999 break;
3000 case TGSI_OPCODE_UP2H:
3001 src0 = fetchSrc(0, 0);
3002 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3003 geni = mkCvt(OP_CVT, TYPE_F32, dst0[c], TYPE_F16, src0);
3004 geni->subOp = c & 1;
3005 }
3006 break;
3007 case TGSI_OPCODE_EMIT:
3008 /* export the saved viewport index */
3009 if (viewport != NULL) {
3010 Symbol *vpSym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_U32,
3011 info->out[info->io.viewportId].slot[0] * 4);
3012 mkStore(OP_EXPORT, TYPE_U32, vpSym, NULL, viewport);
3013 }
3014 /* fallthrough */
3015 case TGSI_OPCODE_ENDPRIM:
3016 {
3017 // get vertex stream (must be immediate)
3018 unsigned int stream = tgsi.getSrc(0).getValueU32(0, info);
3019 if (stream && op == OP_RESTART)
3020 break;
3021 src0 = mkImm(stream);
3022 mkOp1(op, TYPE_U32, NULL, src0)->fixed = 1;
3023 break;
3024 }
3025 case TGSI_OPCODE_IF:
3026 case TGSI_OPCODE_UIF:
3027 {
3028 BasicBlock *ifBB = new BasicBlock(func);
3029
3030 bb->cfg.attach(&ifBB->cfg, Graph::Edge::TREE);
3031 condBBs.push(bb);
3032 joinBBs.push(bb);
3033
3034 mkFlow(OP_BRA, NULL, CC_NOT_P, fetchSrc(0, 0))->setType(srcTy);
3035
3036 setPosition(ifBB, true);
3037 }
3038 break;
3039 case TGSI_OPCODE_ELSE:
3040 {
3041 BasicBlock *elseBB = new BasicBlock(func);
3042 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3043
3044 forkBB->cfg.attach(&elseBB->cfg, Graph::Edge::TREE);
3045 condBBs.push(bb);
3046
3047 forkBB->getExit()->asFlow()->target.bb = elseBB;
3048 if (!bb->isTerminated())
3049 mkFlow(OP_BRA, NULL, CC_ALWAYS, NULL);
3050
3051 setPosition(elseBB, true);
3052 }
3053 break;
3054 case TGSI_OPCODE_ENDIF:
3055 {
3056 BasicBlock *convBB = new BasicBlock(func);
3057 BasicBlock *prevBB = reinterpret_cast<BasicBlock *>(condBBs.pop().u.p);
3058 BasicBlock *forkBB = reinterpret_cast<BasicBlock *>(joinBBs.pop().u.p);
3059
3060 if (!bb->isTerminated()) {
3061 // we only want join if none of the clauses ended with CONT/BREAK/RET
3062 if (prevBB->getExit()->op == OP_BRA && joinBBs.getSize() < 6)
3063 insertConvergenceOps(convBB, forkBB);
3064 mkFlow(OP_BRA, convBB, CC_ALWAYS, NULL);
3065 bb->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3066 }
3067
3068 if (prevBB->getExit()->op == OP_BRA) {
3069 prevBB->cfg.attach(&convBB->cfg, Graph::Edge::FORWARD);
3070 prevBB->getExit()->asFlow()->target.bb = convBB;
3071 }
3072 setPosition(convBB, true);
3073 }
3074 break;
3075 case TGSI_OPCODE_BGNLOOP:
3076 {
3077 BasicBlock *lbgnBB = new BasicBlock(func);
3078 BasicBlock *lbrkBB = new BasicBlock(func);
3079
3080 loopBBs.push(lbgnBB);
3081 breakBBs.push(lbrkBB);
3082 if (loopBBs.getSize() > func->loopNestingBound)
3083 func->loopNestingBound++;
3084
3085 mkFlow(OP_PREBREAK, lbrkBB, CC_ALWAYS, NULL);
3086
3087 bb->cfg.attach(&lbgnBB->cfg, Graph::Edge::TREE);
3088 setPosition(lbgnBB, true);
3089 mkFlow(OP_PRECONT, lbgnBB, CC_ALWAYS, NULL);
3090 }
3091 break;
3092 case TGSI_OPCODE_ENDLOOP:
3093 {
3094 BasicBlock *loopBB = reinterpret_cast<BasicBlock *>(loopBBs.pop().u.p);
3095
3096 if (!bb->isTerminated()) {
3097 mkFlow(OP_CONT, loopBB, CC_ALWAYS, NULL);
3098 bb->cfg.attach(&loopBB->cfg, Graph::Edge::BACK);
3099 }
3100 setPosition(reinterpret_cast<BasicBlock *>(breakBBs.pop().u.p), true);
3101
3102 // If the loop never breaks (e.g. only has RET's inside), then there
3103 // will be no way to get to the break bb. However BGNLOOP will have
3104 // already made a PREBREAK to it, so it must be in the CFG.
3105 if (getBB()->cfg.incidentCount() == 0)
3106 loopBB->cfg.attach(&getBB()->cfg, Graph::Edge::TREE);
3107 }
3108 break;
3109 case TGSI_OPCODE_BRK:
3110 {
3111 if (bb->isTerminated())
3112 break;
3113 BasicBlock *brkBB = reinterpret_cast<BasicBlock *>(breakBBs.peek().u.p);
3114 mkFlow(OP_BREAK, brkBB, CC_ALWAYS, NULL);
3115 bb->cfg.attach(&brkBB->cfg, Graph::Edge::CROSS);
3116 }
3117 break;
3118 case TGSI_OPCODE_CONT:
3119 {
3120 if (bb->isTerminated())
3121 break;
3122 BasicBlock *contBB = reinterpret_cast<BasicBlock *>(loopBBs.peek().u.p);
3123 mkFlow(OP_CONT, contBB, CC_ALWAYS, NULL);
3124 contBB->explicitCont = true;
3125 bb->cfg.attach(&contBB->cfg, Graph::Edge::BACK);
3126 }
3127 break;
3128 case TGSI_OPCODE_BGNSUB:
3129 {
3130 Subroutine *s = getSubroutine(ip);
3131 BasicBlock *entry = new BasicBlock(s->f);
3132 BasicBlock *leave = new BasicBlock(s->f);
3133
3134 // multiple entrypoints possible, keep the graph connected
3135 if (prog->getType() == Program::TYPE_COMPUTE)
3136 prog->main->call.attach(&s->f->call, Graph::Edge::TREE);
3137
3138 sub.cur = s;
3139 s->f->setEntry(entry);
3140 s->f->setExit(leave);
3141 setPosition(entry, true);
3142 return true;
3143 }
3144 case TGSI_OPCODE_ENDSUB:
3145 {
3146 sub.cur = getSubroutine(prog->main);
3147 setPosition(BasicBlock::get(sub.cur->f->cfg.getRoot()), true);
3148 return true;
3149 }
3150 case TGSI_OPCODE_CAL:
3151 {
3152 Subroutine *s = getSubroutine(tgsi.getLabel());
3153 mkFlow(OP_CALL, s->f, CC_ALWAYS, NULL);
3154 func->call.attach(&s->f->call, Graph::Edge::TREE);
3155 return true;
3156 }
3157 case TGSI_OPCODE_RET:
3158 {
3159 if (bb->isTerminated())
3160 return true;
3161 BasicBlock *leave = BasicBlock::get(func->cfgExit);
3162
3163 if (!isEndOfSubroutine(ip + 1)) {
3164 // insert a PRERET at the entry if this is an early return
3165 // (only needed for sharing code in the epilogue)
3166 BasicBlock *pos = getBB();
3167 setPosition(BasicBlock::get(func->cfg.getRoot()), false);
3168 mkFlow(OP_PRERET, leave, CC_ALWAYS, NULL)->fixed = 1;
3169 setPosition(pos, true);
3170 }
3171 mkFlow(OP_RET, NULL, CC_ALWAYS, NULL)->fixed = 1;
3172 bb->cfg.attach(&leave->cfg, Graph::Edge::CROSS);
3173 }
3174 break;
3175 case TGSI_OPCODE_END:
3176 {
3177 // attach and generate epilogue code
3178 BasicBlock *epilogue = BasicBlock::get(func->cfgExit);
3179 bb->cfg.attach(&epilogue->cfg, Graph::Edge::TREE);
3180 setPosition(epilogue, true);
3181 if (prog->getType() == Program::TYPE_FRAGMENT)
3182 exportOutputs();
3183 if (info->io.genUserClip > 0)
3184 handleUserClipPlanes();
3185 mkOp(OP_EXIT, TYPE_NONE, NULL)->terminator = 1;
3186 }
3187 break;
3188 case TGSI_OPCODE_SWITCH:
3189 case TGSI_OPCODE_CASE:
3190 ERROR("switch/case opcode encountered, should have been lowered\n");
3191 abort();
3192 break;
3193 case TGSI_OPCODE_LOAD:
3194 handleLOAD(dst0);
3195 break;
3196 case TGSI_OPCODE_STORE:
3197 handleSTORE();
3198 break;
3199 case TGSI_OPCODE_BARRIER:
3200 geni = mkOp2(OP_BAR, TYPE_U32, NULL, mkImm(0), mkImm(0));
3201 geni->fixed = 1;
3202 geni->subOp = NV50_IR_SUBOP_BAR_SYNC;
3203 break;
3204 case TGSI_OPCODE_MFENCE:
3205 case TGSI_OPCODE_LFENCE:
3206 case TGSI_OPCODE_SFENCE:
3207 geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3208 geni->fixed = 1;
3209 geni->subOp = tgsi::opcodeToSubOp(tgsi.getOpcode());
3210 break;
3211 case TGSI_OPCODE_MEMBAR:
3212 geni = mkOp(OP_MEMBAR, TYPE_NONE, NULL);
3213 geni->fixed = 1;
3214 if (tgsi.getSrc(0).getValueU32(0, info) & TGSI_MEMBAR_THREAD_GROUP)
3215 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, CTA);
3216 else
3217 geni->subOp = NV50_IR_SUBOP_MEMBAR(M, GL);
3218 break;
3219 case TGSI_OPCODE_ATOMUADD:
3220 case TGSI_OPCODE_ATOMXCHG:
3221 case TGSI_OPCODE_ATOMCAS:
3222 case TGSI_OPCODE_ATOMAND:
3223 case TGSI_OPCODE_ATOMOR:
3224 case TGSI_OPCODE_ATOMXOR:
3225 case TGSI_OPCODE_ATOMUMIN:
3226 case TGSI_OPCODE_ATOMIMIN:
3227 case TGSI_OPCODE_ATOMUMAX:
3228 case TGSI_OPCODE_ATOMIMAX:
3229 handleATOM(dst0, dstTy, tgsi::opcodeToSubOp(tgsi.getOpcode()));
3230 break;
3231 case TGSI_OPCODE_RESQ:
3232 geni = mkOp1(OP_SUQ, TYPE_U32, dst0[0],
3233 makeSym(TGSI_FILE_BUFFER, tgsi.getSrc(0).getIndex(0), -1, 0, 0));
3234 if (tgsi.getSrc(0).isIndirect(0))
3235 geni->setIndirect(0, 1, fetchSrc(tgsi.getSrc(0).getIndirect(0), 0, 0));
3236 break;
3237 case TGSI_OPCODE_IBFE:
3238 case TGSI_OPCODE_UBFE:
3239 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3240 src0 = fetchSrc(0, c);
3241 if (tgsi.getSrc(1).getFile() == TGSI_FILE_IMMEDIATE &&
3242 tgsi.getSrc(2).getFile() == TGSI_FILE_IMMEDIATE) {
3243 src1 = loadImm(NULL, tgsi.getSrc(2).getValueU32(c, info) << 8 |
3244 tgsi.getSrc(1).getValueU32(c, info));
3245 } else {
3246 src1 = fetchSrc(1, c);
3247 src2 = fetchSrc(2, c);
3248 mkOp3(OP_INSBF, TYPE_U32, src1, src2, mkImm(0x808), src1);
3249 }
3250 mkOp2(OP_EXTBF, dstTy, dst0[c], src0, src1);
3251 }
3252 break;
3253 case TGSI_OPCODE_BFI:
3254 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3255 src0 = fetchSrc(0, c);
3256 src1 = fetchSrc(1, c);
3257 src2 = fetchSrc(2, c);
3258 src3 = fetchSrc(3, c);
3259 mkOp3(OP_INSBF, TYPE_U32, src2, src3, mkImm(0x808), src2);
3260 mkOp3(OP_INSBF, TYPE_U32, dst0[c], src1, src2, src0);
3261 }
3262 break;
3263 case TGSI_OPCODE_LSB:
3264 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3265 src0 = fetchSrc(0, c);
3266 geni = mkOp2(OP_EXTBF, TYPE_U32, src0, src0, mkImm(0x2000));
3267 geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3268 geni = mkOp1(OP_BFIND, TYPE_U32, dst0[c], src0);
3269 geni->subOp = NV50_IR_SUBOP_BFIND_SAMT;
3270 }
3271 break;
3272 case TGSI_OPCODE_IMSB:
3273 case TGSI_OPCODE_UMSB:
3274 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3275 src0 = fetchSrc(0, c);
3276 mkOp1(OP_BFIND, srcTy, dst0[c], src0);
3277 }
3278 break;
3279 case TGSI_OPCODE_BREV:
3280 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3281 src0 = fetchSrc(0, c);
3282 geni = mkOp2(OP_EXTBF, TYPE_U32, dst0[c], src0, mkImm(0x2000));
3283 geni->subOp = NV50_IR_SUBOP_EXTBF_REV;
3284 }
3285 break;
3286 case TGSI_OPCODE_POPC:
3287 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3288 src0 = fetchSrc(0, c);
3289 mkOp2(OP_POPCNT, TYPE_U32, dst0[c], src0, src0);
3290 }
3291 break;
3292 case TGSI_OPCODE_INTERP_CENTROID:
3293 case TGSI_OPCODE_INTERP_SAMPLE:
3294 case TGSI_OPCODE_INTERP_OFFSET:
3295 handleINTERP(dst0);
3296 break;
3297 case TGSI_OPCODE_D2I:
3298 case TGSI_OPCODE_D2U:
3299 case TGSI_OPCODE_D2F: {
3300 int pos = 0;
3301 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3302 Value *dreg = getSSA(8);
3303 src0 = fetchSrc(0, pos);
3304 src1 = fetchSrc(0, pos + 1);
3305 mkOp2(OP_MERGE, TYPE_U64, dreg, src0, src1);
3306 mkCvt(OP_CVT, dstTy, dst0[c], srcTy, dreg);
3307 pos += 2;
3308 }
3309 break;
3310 }
3311 case TGSI_OPCODE_I2D:
3312 case TGSI_OPCODE_U2D:
3313 case TGSI_OPCODE_F2D:
3314 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3315 Value *dreg = getSSA(8);
3316 mkCvt(OP_CVT, dstTy, dreg, srcTy, fetchSrc(0, c / 2));
3317 mkSplit(&dst0[c], 4, dreg);
3318 c++;
3319 }
3320 break;
3321 case TGSI_OPCODE_DABS:
3322 case TGSI_OPCODE_DNEG:
3323 case TGSI_OPCODE_DRCP:
3324 case TGSI_OPCODE_DSQRT:
3325 case TGSI_OPCODE_DRSQ:
3326 case TGSI_OPCODE_DTRUNC:
3327 case TGSI_OPCODE_DCEIL:
3328 case TGSI_OPCODE_DFLR:
3329 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3330 src0 = getSSA(8);
3331 Value *dst = getSSA(8), *tmp[2];
3332 tmp[0] = fetchSrc(0, c);
3333 tmp[1] = fetchSrc(0, c + 1);
3334 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3335 mkOp1(op, dstTy, dst, src0);
3336 mkSplit(&dst0[c], 4, dst);
3337 c++;
3338 }
3339 break;
3340 case TGSI_OPCODE_DFRAC:
3341 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3342 src0 = getSSA(8);
3343 Value *dst = getSSA(8), *tmp[2];
3344 tmp[0] = fetchSrc(0, c);
3345 tmp[1] = fetchSrc(0, c + 1);
3346 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3347 mkOp1(OP_FLOOR, TYPE_F64, dst, src0);
3348 mkOp2(OP_SUB, TYPE_F64, dst, src0, dst);
3349 mkSplit(&dst0[c], 4, dst);
3350 c++;
3351 }
3352 break;
3353 case TGSI_OPCODE_DSLT:
3354 case TGSI_OPCODE_DSGE:
3355 case TGSI_OPCODE_DSEQ:
3356 case TGSI_OPCODE_DSNE: {
3357 int pos = 0;
3358 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3359 Value *tmp[2];
3360
3361 src0 = getSSA(8);
3362 src1 = getSSA(8);
3363 tmp[0] = fetchSrc(0, pos);
3364 tmp[1] = fetchSrc(0, pos + 1);
3365 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3366 tmp[0] = fetchSrc(1, pos);
3367 tmp[1] = fetchSrc(1, pos + 1);
3368 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3369 mkCmp(op, tgsi.getSetCond(), dstTy, dst0[c], srcTy, src0, src1);
3370 pos += 2;
3371 }
3372 break;
3373 }
3374 case TGSI_OPCODE_DADD:
3375 case TGSI_OPCODE_DMUL:
3376 case TGSI_OPCODE_DMAX:
3377 case TGSI_OPCODE_DMIN:
3378 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3379 src0 = getSSA(8);
3380 src1 = getSSA(8);
3381 Value *dst = getSSA(8), *tmp[2];
3382 tmp[0] = fetchSrc(0, c);
3383 tmp[1] = fetchSrc(0, c + 1);
3384 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3385 tmp[0] = fetchSrc(1, c);
3386 tmp[1] = fetchSrc(1, c + 1);
3387 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3388 mkOp2(op, dstTy, dst, src0, src1);
3389 mkSplit(&dst0[c], 4, dst);
3390 c++;
3391 }
3392 break;
3393 case TGSI_OPCODE_DMAD:
3394 case TGSI_OPCODE_DFMA:
3395 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3396 src0 = getSSA(8);
3397 src1 = getSSA(8);
3398 src2 = getSSA(8);
3399 Value *dst = getSSA(8), *tmp[2];
3400 tmp[0] = fetchSrc(0, c);
3401 tmp[1] = fetchSrc(0, c + 1);
3402 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3403 tmp[0] = fetchSrc(1, c);
3404 tmp[1] = fetchSrc(1, c + 1);
3405 mkOp2(OP_MERGE, TYPE_U64, src1, tmp[0], tmp[1]);
3406 tmp[0] = fetchSrc(2, c);
3407 tmp[1] = fetchSrc(2, c + 1);
3408 mkOp2(OP_MERGE, TYPE_U64, src2, tmp[0], tmp[1]);
3409 mkOp3(op, dstTy, dst, src0, src1, src2);
3410 mkSplit(&dst0[c], 4, dst);
3411 c++;
3412 }
3413 break;
3414 case TGSI_OPCODE_DROUND:
3415 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3416 src0 = getSSA(8);
3417 Value *dst = getSSA(8), *tmp[2];
3418 tmp[0] = fetchSrc(0, c);
3419 tmp[1] = fetchSrc(0, c + 1);
3420 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3421 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F64, src0)
3422 ->rnd = ROUND_NI;
3423 mkSplit(&dst0[c], 4, dst);
3424 c++;
3425 }
3426 break;
3427 case TGSI_OPCODE_DSSG:
3428 FOR_EACH_DST_ENABLED_CHANNEL(0, c, tgsi) {
3429 src0 = getSSA(8);
3430 Value *dst = getSSA(8), *dstF32 = getSSA(), *tmp[2];
3431 tmp[0] = fetchSrc(0, c);
3432 tmp[1] = fetchSrc(0, c + 1);
3433 mkOp2(OP_MERGE, TYPE_U64, src0, tmp[0], tmp[1]);
3434
3435 val0 = getScratch();
3436 val1 = getScratch();
3437 // The zero is wrong here since it's only 32-bit, but it works out in
3438 // the end since it gets replaced with $r63.
3439 mkCmp(OP_SET, CC_GT, TYPE_F32, val0, TYPE_F64, src0, zero);
3440 mkCmp(OP_SET, CC_LT, TYPE_F32, val1, TYPE_F64, src0, zero);
3441 mkOp2(OP_SUB, TYPE_F32, dstF32, val0, val1);
3442 mkCvt(OP_CVT, TYPE_F64, dst, TYPE_F32, dstF32);
3443 mkSplit(&dst0[c], 4, dst);
3444 c++;
3445 }
3446 break;
3447 default:
3448 ERROR("unhandled TGSI opcode: %u\n", tgsi.getOpcode());
3449 assert(0);
3450 break;
3451 }
3452
3453 if (tgsi.dstCount()) {
3454 for (c = 0; c < 4; ++c) {
3455 if (!dst0[c])
3456 continue;
3457 if (dst0[c] != rDst0[c])
3458 mkMov(rDst0[c], dst0[c]);
3459 storeDst(0, c, rDst0[c]);
3460 }
3461 }
3462 vtxBaseValid = 0;
3463
3464 return true;
3465 }
3466
3467 void
3468 Converter::handleUserClipPlanes()
3469 {
3470 Value *res[8];
3471 int n, i, c;
3472
3473 for (c = 0; c < 4; ++c) {
3474 for (i = 0; i < info->io.genUserClip; ++i) {
3475 Symbol *sym = mkSymbol(FILE_MEMORY_CONST, info->io.auxCBSlot,
3476 TYPE_F32, info->io.ucpBase + i * 16 + c * 4);
3477 Value *ucp = mkLoadv(TYPE_F32, sym, NULL);
3478 if (c == 0)
3479 res[i] = mkOp2v(OP_MUL, TYPE_F32, getScratch(), clipVtx[c], ucp);
3480 else
3481 mkOp3(OP_MAD, TYPE_F32, res[i], clipVtx[c], ucp, res[i]);
3482 }
3483 }
3484
3485 const int first = info->numOutputs - (info->io.genUserClip + 3) / 4;
3486
3487 for (i = 0; i < info->io.genUserClip; ++i) {
3488 n = i / 4 + first;
3489 c = i % 4;
3490 Symbol *sym =
3491 mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32, info->out[n].slot[c] * 4);
3492 mkStore(OP_EXPORT, TYPE_F32, sym, NULL, res[i]);
3493 }
3494 }
3495
3496 void
3497 Converter::exportOutputs()
3498 {
3499 for (unsigned int i = 0; i < info->numOutputs; ++i) {
3500 for (unsigned int c = 0; c < 4; ++c) {
3501 if (!oData.exists(sub.cur->values, i, c))
3502 continue;
3503 Symbol *sym = mkSymbol(FILE_SHADER_OUTPUT, 0, TYPE_F32,
3504 info->out[i].slot[c] * 4);
3505 Value *val = oData.load(sub.cur->values, i, c, NULL);
3506 if (val)
3507 mkStore(OP_EXPORT, TYPE_F32, sym, NULL, val);
3508 }
3509 }
3510 }
3511
3512 Converter::Converter(Program *ir, const tgsi::Source *code) : BuildUtil(ir),
3513 code(code),
3514 tgsi(NULL),
3515 tData(this), lData(this), aData(this), pData(this), oData(this)
3516 {
3517 info = code->info;
3518
3519 const unsigned tSize = code->fileSize(TGSI_FILE_TEMPORARY);
3520 const unsigned pSize = code->fileSize(TGSI_FILE_PREDICATE);
3521 const unsigned aSize = code->fileSize(TGSI_FILE_ADDRESS);
3522 const unsigned oSize = code->fileSize(TGSI_FILE_OUTPUT);
3523
3524 tData.setup(TGSI_FILE_TEMPORARY, 0, 0, tSize, 4, 4, FILE_GPR, 0);
3525 lData.setup(TGSI_FILE_TEMPORARY, 1, 0, tSize, 4, 4, FILE_MEMORY_LOCAL, 0);
3526 pData.setup(TGSI_FILE_PREDICATE, 0, 0, pSize, 4, 4, FILE_PREDICATE, 0);
3527 aData.setup(TGSI_FILE_ADDRESS, 0, 0, aSize, 4, 4, FILE_GPR, 0);
3528 oData.setup(TGSI_FILE_OUTPUT, 0, 0, oSize, 4, 4, FILE_GPR, 0);
3529
3530 zero = mkImm((uint32_t)0);
3531
3532 vtxBaseValid = 0;
3533 }
3534
3535 Converter::~Converter()
3536 {
3537 }
3538
3539 inline const Converter::Location *
3540 Converter::BindArgumentsPass::getValueLocation(Subroutine *s, Value *v)
3541 {
3542 ValueMap::l_iterator it = s->values.l.find(v);
3543 return it == s->values.l.end() ? NULL : &it->second;
3544 }
3545
3546 template<typename T> inline void
3547 Converter::BindArgumentsPass::updateCallArgs(
3548 Instruction *i, void (Instruction::*setArg)(int, Value *),
3549 T (Function::*proto))
3550 {
3551 Function *g = i->asFlow()->target.fn;
3552 Subroutine *subg = conv.getSubroutine(g);
3553
3554 for (unsigned a = 0; a < (g->*proto).size(); ++a) {
3555 Value *v = (g->*proto)[a].get();
3556 const Converter::Location &l = *getValueLocation(subg, v);
3557 Converter::DataArray *array = conv.getArrayForFile(l.array, l.arrayIdx);
3558
3559 (i->*setArg)(a, array->acquire(sub->values, l.i, l.c));
3560 }
3561 }
3562
3563 template<typename T> inline void
3564 Converter::BindArgumentsPass::updatePrototype(
3565 BitSet *set, void (Function::*updateSet)(), T (Function::*proto))
3566 {
3567 (func->*updateSet)();
3568
3569 for (unsigned i = 0; i < set->getSize(); ++i) {
3570 Value *v = func->getLValue(i);
3571 const Converter::Location *l = getValueLocation(sub, v);
3572
3573 // only include values with a matching TGSI register
3574 if (set->test(i) && l && !conv.code->locals.count(*l))
3575 (func->*proto).push_back(v);
3576 }
3577 }
3578
3579 bool
3580 Converter::BindArgumentsPass::visit(Function *f)
3581 {
3582 sub = conv.getSubroutine(f);
3583
3584 for (ArrayList::Iterator bi = f->allBBlocks.iterator();
3585 !bi.end(); bi.next()) {
3586 for (Instruction *i = BasicBlock::get(bi)->getFirst();
3587 i; i = i->next) {
3588 if (i->op == OP_CALL && !i->asFlow()->builtin) {
3589 updateCallArgs(i, &Instruction::setSrc, &Function::ins);
3590 updateCallArgs(i, &Instruction::setDef, &Function::outs);
3591 }
3592 }
3593 }
3594
3595 if (func == prog->main && prog->getType() != Program::TYPE_COMPUTE)
3596 return true;
3597 updatePrototype(&BasicBlock::get(f->cfg.getRoot())->liveSet,
3598 &Function::buildLiveSets, &Function::ins);
3599 updatePrototype(&BasicBlock::get(f->cfgExit)->defSet,
3600 &Function::buildDefSets, &Function::outs);
3601
3602 return true;
3603 }
3604
3605 bool
3606 Converter::run()
3607 {
3608 BasicBlock *entry = new BasicBlock(prog->main);
3609 BasicBlock *leave = new BasicBlock(prog->main);
3610
3611 prog->main->setEntry(entry);
3612 prog->main->setExit(leave);
3613
3614 setPosition(entry, true);
3615 sub.cur = getSubroutine(prog->main);
3616
3617 if (info->io.genUserClip > 0) {
3618 for (int c = 0; c < 4; ++c)
3619 clipVtx[c] = getScratch();
3620 }
3621
3622 switch (prog->getType()) {
3623 case Program::TYPE_TESSELLATION_CONTROL:
3624 outBase = mkOp2v(
3625 OP_SUB, TYPE_U32, getSSA(),
3626 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_LANEID, 0)),
3627 mkOp1v(OP_RDSV, TYPE_U32, getSSA(), mkSysVal(SV_INVOCATION_ID, 0)));
3628 break;
3629 case Program::TYPE_FRAGMENT: {
3630 Symbol *sv = mkSysVal(SV_POSITION, 3);
3631 fragCoord[3] = mkOp1v(OP_RDSV, TYPE_F32, getSSA(), sv);
3632 mkOp1(OP_RCP, TYPE_F32, fragCoord[3], fragCoord[3]);
3633 break;
3634 }
3635 default:
3636 break;
3637 }
3638
3639 if (info->io.viewportId >= 0)
3640 viewport = getScratch();
3641 else
3642 viewport = NULL;
3643
3644 for (ip = 0; ip < code->scan.num_instructions; ++ip) {
3645 if (!handleInstruction(&code->insns[ip]))
3646 return false;
3647 }
3648
3649 if (!BindArgumentsPass(*this).run(prog))
3650 return false;
3651
3652 return true;
3653 }
3654
3655 } // unnamed namespace
3656
3657 namespace nv50_ir {
3658
3659 bool
3660 Program::makeFromTGSI(struct nv50_ir_prog_info *info)
3661 {
3662 tgsi::Source src(info);
3663 if (!src.scanSource())
3664 return false;
3665 tlsSize = info->bin.tlsSpace;
3666
3667 Converter builder(this, &src);
3668 return builder.run();
3669 }
3670
3671 } // namespace nv50_ir