src/gallium/drivers/nouveau/codegen/nv50_ir_build_util.cpp

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include "codegen/nv50_ir.h"
  24 #include "codegen/nv50_ir_build_util.h"
  25
  26 namespace nv50_ir {
  27
  28 BuildUtil::BuildUtil()
  29 {
  30    init(NULL);
  31 }
  32
  33 BuildUtil::BuildUtil(Program *prog)
  34 {
  35    init(prog);
  36 }
  37
  38 void
  39 BuildUtil::init(Program *prog)
  40 {
  41    this->prog = prog;
  42
  43    func = NULL;
  44    bb = NULL;
  45    pos = NULL;
  46
  47    memset(imms, 0, sizeof(imms));
  48    immCount = 0;
  49 }
  50
  51 void
  52 BuildUtil::addImmediate(ImmediateValue *imm)
  53 {
  54    if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4)
  55       return;
  56
  57    unsigned int pos = u32Hash(imm->reg.data.u32);
  58
  59    while (imms[pos])
  60       pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
  61    imms[pos] = imm;
  62    immCount++;
  63 }
  64
  65 Instruction *
  66 BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src)
  67 {
  68    Instruction *insn = new_Instruction(func, op, ty);
  69
  70    insn->setDef(0, dst);
  71    insn->setSrc(0, src);
  72
  73    insert(insn);
  74    return insn;
  75 }
  76
  77 Instruction *
  78 BuildUtil::mkOp2(operation op, DataType ty, Value *dst,
  79                  Value *src0, Value *src1)
  80 {
  81    Instruction *insn = new_Instruction(func, op, ty);
  82
  83    insn->setDef(0, dst);
  84    insn->setSrc(0, src0);
  85    insn->setSrc(1, src1);
  86
  87    insert(insn);
  88    return insn;
  89 }
  90
  91 Instruction *
  92 BuildUtil::mkOp3(operation op, DataType ty, Value *dst,
  93                  Value *src0, Value *src1, Value *src2)
  94 {
  95    Instruction *insn = new_Instruction(func, op, ty);
  96
  97    insn->setDef(0, dst);
  98    insn->setSrc(0, src0);
  99    insn->setSrc(1, src1);
 100    insn->setSrc(2, src2);
 101
 102    insert(insn);
 103    return insn;
 104 }
 105
 106 Instruction *
 107 BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr)
 108 {
 109    Instruction *insn = new_Instruction(func, OP_LOAD, ty);
 110
 111    insn->setDef(0, dst);
 112    insn->setSrc(0, mem);
 113    if (ptr)
 114       insn->setIndirect(0, 0, ptr);
 115
 116    insert(insn);
 117    return insn;
 118 }
 119
 120 Instruction *
 121 BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr,
 122                    Value *stVal)
 123 {
 124    Instruction *insn = new_Instruction(func, op, ty);
 125
 126    insn->setSrc(0, mem);
 127    insn->setSrc(1, stVal);
 128    if (ptr)
 129       insn->setIndirect(0, 0, ptr);
 130
 131    insert(insn);
 132    return insn;
 133 }
 134
 135 Instruction *
 136 BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset,
 137                    Value *attrRel, Value *primRel)
 138 {
 139    Symbol *sym = mkSymbol(file, 0, ty, offset);
 140
 141    Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym);
 142
 143    insn->setIndirect(0, 0, attrRel);
 144    insn->setIndirect(0, 1, primRel);
 145
 146    // already inserted
 147    return insn;
 148 }
 149
 150 Instruction *
 151 BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel)
 152 {
 153    operation op = OP_LINTERP;
 154    DataType ty = TYPE_F32;
 155
 156    if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT)
 157       ty = TYPE_U32;
 158    else
 159    if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE)
 160       op = OP_PINTERP;
 161
 162    Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset);
 163
 164    Instruction *insn = mkOp1(op, ty, dst, sym);
 165    insn->setIndirect(0, 0, rel);
 166    insn->setInterpolate(mode);
 167    return insn;
 168 }
 169
 170 Instruction *
 171 BuildUtil::mkMov(Value *dst, Value *src, DataType ty)
 172 {
 173    Instruction *insn = new_Instruction(func, OP_MOV, ty);
 174
 175    insn->setDef(0, dst);
 176    insn->setSrc(0, src);
 177
 178    insert(insn);
 179    return insn;
 180 }
 181
 182 Instruction *
 183 BuildUtil::mkMovToReg(int id, Value *src)
 184 {
 185    Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size));
 186
 187    insn->setDef(0, new_LValue(func, FILE_GPR));
 188    insn->getDef(0)->reg.data.id = id;
 189    insn->setSrc(0, src);
 190
 191    insert(insn);
 192    return insn;
 193 }
 194
 195 Instruction *
 196 BuildUtil::mkMovFromReg(Value *dst, int id)
 197 {
 198    Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size));
 199
 200    insn->setDef(0, dst);
 201    insn->setSrc(0, new_LValue(func, FILE_GPR));
 202    insn->getSrc(0)->reg.data.id = id;
 203
 204    insert(insn);
 205    return insn;
 206 }
 207
 208 Instruction *
 209 BuildUtil::mkCvt(operation op,
 210                  DataType dstTy, Value *dst, DataType srcTy, Value *src)
 211 {
 212    Instruction *insn = new_Instruction(func, op, dstTy);
 213
 214    insn->setType(dstTy, srcTy);
 215    insn->setDef(0, dst);
 216    insn->setSrc(0, src);
 217
 218    insert(insn);
 219    return insn;
 220 }
 221
 222 CmpInstruction *
 223 BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst,
 224                  DataType srcTy, Value *src0, Value *src1, Value *src2)
 225 {
 226    CmpInstruction *insn = new_CmpInstruction(func, op);
 227
 228    insn->setType((dst->reg.file == FILE_PREDICATE ||
 229                   dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy);
 230    insn->setCondition(cc);
 231    insn->setDef(0, dst);
 232    insn->setSrc(0, src0);
 233    insn->setSrc(1, src1);
 234    if (src2)
 235       insn->setSrc(2, src2);
 236
 237    if (dst->reg.file == FILE_FLAGS)
 238       insn->flagsDef = 0;
 239
 240    insert(insn);
 241    return insn;
 242 }
 243
 244 TexInstruction *
 245 BuildUtil::mkTex(operation op, TexTarget targ,
 246                  uint16_t tic, uint16_t tsc,
 247                  const std::vector<Value *> &def,
 248                  const std::vector<Value *> &src)
 249 {
 250    TexInstruction *tex = new_TexInstruction(func, op);
 251
 252    for (size_t d = 0; d < def.size() && def[d]; ++d)
 253       tex->setDef(d, def[d]);
 254    for (size_t s = 0; s < src.size() && src[s]; ++s)
 255       tex->setSrc(s, src[s]);
 256
 257    tex->setTexture(targ, tic, tsc);
 258
 259    insert(tex);
 260    return tex;
 261 }
 262
 263 Instruction *
 264 BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1)
 265 {
 266    Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1);
 267    quadop->subOp = q;
 268    quadop->lanes = l;
 269    return quadop;
 270 }
 271
 272 Instruction *
 273 BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc)
 274 {
 275    LValue *def0 = getSSA();
 276    LValue *def1 = getSSA();
 277
 278    mkMov(def0, trSrc)->setPredicate(CC_P, pred);
 279    mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred);
 280
 281    return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1);
 282 }
 283
 284 Instruction *
 285 BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val)
 286 {
 287    Instruction *insn = NULL;
 288
 289    const DataType fTy = typeOfSize(halfSize * 2);
 290
 291    if (val->reg.file == FILE_IMMEDIATE)
 292       val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0);
 293
 294    if (isMemoryFile(val->reg.file)) {
 295       h[0] = cloneShallow(getFunction(), val);
 296       h[1] = cloneShallow(getFunction(), val);
 297       h[0]->reg.size = halfSize;
 298       h[1]->reg.size = halfSize;
 299       h[1]->reg.data.offset += halfSize;
 300    } else {
 301       h[0] = getSSA(halfSize, val->reg.file);
 302       h[1] = getSSA(halfSize, val->reg.file);
 303       insn = mkOp1(OP_SPLIT, fTy, h[0], val);
 304       insn->setDef(1, h[1]);
 305    }
 306    return insn;
 307 }
 308
 309 FlowInstruction *
 310 BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred)
 311 {
 312    FlowInstruction *insn = new_FlowInstruction(func, op, targ);
 313
 314    if (pred)
 315       insn->setPredicate(cc, pred);
 316
 317    insert(insn);
 318    return insn;
 319 }
 320
 321 void
 322 BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit)
 323 {
 324    static const uint16_t baseSize2[16] =
 325    {
 326       0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220,
 327       0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040,
 328    };
 329
 330    int base = 0;
 331
 332    for (; rMask; rMask >>= 4, base += 4) {
 333       const uint32_t mask = rMask & 0xf;
 334       if (!mask)
 335          continue;
 336       int base1 = (baseSize2[mask] >>  0) & 0xf;
 337       int size1 = (baseSize2[mask] >>  4) & 0xf;
 338       int base2 = (baseSize2[mask] >>  8) & 0xf;
 339       int size2 = (baseSize2[mask] >> 12) & 0xf;
 340       Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL);
 341       if (1) { // size1 can't be 0
 342          LValue *reg = new_LValue(func, f);
 343          reg->reg.size = size1 << unit;
 344          reg->reg.data.id = base + base1;
 345          insn->setDef(0, reg);
 346       }
 347       if (size2) {
 348          LValue *reg = new_LValue(func, f);
 349          reg->reg.size = size2 << unit;
 350          reg->reg.data.id = base + base2;
 351          insn->setDef(1, reg);
 352       }
 353    }
 354 }
 355
 356 ImmediateValue *
 357 BuildUtil::mkImm(uint32_t u)
 358 {
 359    unsigned int pos = u32Hash(u);
 360
 361    while (imms[pos] && imms[pos]->reg.data.u32 != u)
 362       pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE;
 363
 364    ImmediateValue *imm = imms[pos];
 365    if (!imm) {
 366       imm = new_ImmediateValue(prog, u);
 367       addImmediate(imm);
 368    }
 369    return imm;
 370 }
 371
 372 ImmediateValue *
 373 BuildUtil::mkImm(uint64_t u)
 374 {
 375    ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0);
 376
 377    imm->reg.size = 8;
 378    imm->reg.type = TYPE_U64;
 379    imm->reg.data.u64 = u;
 380
 381    return imm;
 382 }
 383
 384 ImmediateValue *
 385 BuildUtil::mkImm(float f)
 386 {
 387    union {
 388       float f32;
 389       uint32_t u32;
 390    } u;
 391    u.f32 = f;
 392    return mkImm(u.u32);
 393 }
 394
 395 ImmediateValue *
 396 BuildUtil::mkImm(double d)
 397 {
 398    return new_ImmediateValue(prog, d);
 399 }
 400
 401 Value *
 402 BuildUtil::loadImm(Value *dst, float f)
 403 {
 404    return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f));
 405 }
 406
 407 Value *
 408 BuildUtil::loadImm(Value *dst, double d)
 409 {
 410    return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d));
 411 }
 412
 413 Value *
 414 BuildUtil::loadImm(Value *dst, uint32_t u)
 415 {
 416    return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u));
 417 }
 418
 419 Value *
 420 BuildUtil::loadImm(Value *dst, uint64_t u)
 421 {
 422    return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u));
 423 }
 424
 425 Symbol *
 426 BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty,
 427                     uint32_t baseAddr)
 428 {
 429    Symbol *sym = new_Symbol(prog, file, fileIndex);
 430
 431    sym->setOffset(baseAddr);
 432    sym->reg.type = ty;
 433    sym->reg.size = typeSizeof(ty);
 434
 435    return sym;
 436 }
 437
 438 Symbol *
 439 BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex)
 440 {
 441    Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0);
 442
 443    assert(svIndex < 4 || svName == SV_CLIP_DISTANCE);
 444
 445    switch (svName) {
 446    case SV_POSITION:
 447    case SV_FACE:
 448    case SV_YDIR:
 449    case SV_POINT_SIZE:
 450    case SV_POINT_COORD:
 451    case SV_CLIP_DISTANCE:
 452    case SV_TESS_OUTER:
 453    case SV_TESS_INNER:
 454    case SV_TESS_COORD:
 455       sym->reg.type = TYPE_F32;
 456       break;
 457    default:
 458       sym->reg.type = TYPE_U32;
 459       break;
 460    }
 461    sym->reg.size = typeSizeof(sym->reg.type);
 462
 463    sym->reg.data.sv.sv = svName;
 464    sym->reg.data.sv.index = svIndex;
 465
 466    return sym;
 467 }
 468
 469 void
 470 BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx,
 471                             uint32_t base, int len, int vecDim, int eltSize,
 472                             DataFile file, int8_t fileIdx)
 473 {
 474    this->array = array;
 475    this->arrayIdx = arrayIdx;
 476    this->baseAddr = base;
 477    this->arrayLen = len;
 478    this->vecDim = vecDim;
 479    this->eltSize = eltSize;
 480    this->file = file;
 481    this->regOnly = !isMemoryFile(file);
 482
 483    if (!regOnly) {
 484       baseSym = new_Symbol(up->getProgram(), file, fileIdx);
 485       baseSym->setOffset(baseAddr);
 486       baseSym->reg.size = eltSize;
 487    } else {
 488       baseSym = NULL;
 489    }
 490 }
 491
 492 Value *
 493 BuildUtil::DataArray::acquire(ValueMap &m, int i, int c)
 494 {
 495    if (regOnly) {
 496       Value *v = lookup(m, i, c);
 497       if (!v)
 498          v = insert(m, i, c, new_LValue(up->getFunction(), file));
 499
 500       return v;
 501    } else {
 502       return up->getScratch(eltSize);
 503    }
 504 }
 505
 506 Value *
 507 BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr)
 508 {
 509    if (regOnly) {
 510       Value *v = lookup(m, i, c);
 511       if (!v)
 512          v = insert(m, i, c, new_LValue(up->getFunction(), file));
 513
 514       return v;
 515    } else {
 516       Value *sym = lookup(m, i, c);
 517       if (!sym)
 518          sym = insert(m, i, c, mkSymbol(i, c));
 519
 520       return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr);
 521    }
 522 }
 523
 524 void
 525 BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value)
 526 {
 527    if (regOnly) {
 528       assert(!ptr);
 529       if (!lookup(m, i, c))
 530          insert(m, i, c, value);
 531
 532       assert(lookup(m, i, c) == value);
 533    } else {
 534       Value *sym = lookup(m, i, c);
 535       if (!sym)
 536          sym = insert(m, i, c, mkSymbol(i, c));
 537
 538       const DataType stTy = typeOfSize(value->reg.size);
 539
 540       up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value);
 541    }
 542 }
 543
 544 Symbol *
 545 BuildUtil::DataArray::mkSymbol(int i, int c)
 546 {
 547    const unsigned int idx = i * vecDim + c;
 548    Symbol *sym = new_Symbol(up->getProgram(), file, 0);
 549
 550    assert(baseSym || (idx < arrayLen && c < vecDim));
 551
 552    sym->reg.size = eltSize;
 553    sym->reg.type = typeOfSize(eltSize);
 554    sym->setAddress(baseSym, baseAddr + idx * eltSize);
 555    return sym;
 556 }
 557
 558
 559 Instruction *
 560 BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i,
 561                               Value *zero,
 562                               Value *carry)
 563 {
 564    DataType hTy;
 565    int srcNr;
 566
 567    switch (i->dType) {
 568    case TYPE_U64: hTy = TYPE_U32; break;
 569    case TYPE_S64: hTy = TYPE_S32; break;
 570    case TYPE_F64:
 571       if (i->op == OP_MOV) {
 572          hTy = TYPE_U32;
 573          break;
 574       }
 575       /* fallthrough */
 576    default:
 577       return NULL;
 578    }
 579
 580    switch (i->op) {
 581    case OP_MOV: srcNr = 1; break;
 582    case OP_ADD:
 583    case OP_SUB:
 584       if (!carry)
 585          return NULL;
 586       srcNr = 2;
 587       break;
 588    default:
 589       // TODO when needed
 590       return NULL;
 591    }
 592
 593    i->setType(hTy);
 594    i->setDef(0, cloneShallow(fn, i->getDef(0)));
 595    i->getDef(0)->reg.size = 4;
 596    Instruction *lo = i;
 597    Instruction *hi = cloneForward(fn, i);
 598    lo->bb->insertAfter(lo, hi);
 599
 600    hi->getDef(0)->reg.data.id++;
 601
 602    for (int s = 0; s < srcNr; ++s) {
 603       if (lo->getSrc(s)->reg.size < 8) {
 604          hi->setSrc(s, zero);
 605       } else {
 606          if (lo->getSrc(s)->refCount() > 1)
 607             lo->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
 608          lo->getSrc(s)->reg.size /= 2;
 609          hi->setSrc(s, cloneShallow(fn, lo->getSrc(s)));
 610
 611          switch (hi->src(s).getFile()) {
 612          case FILE_IMMEDIATE:
 613             hi->getSrc(s)->reg.data.u64 >>= 32;
 614             break;
 615          case FILE_MEMORY_CONST:
 616          case FILE_MEMORY_SHARED:
 617          case FILE_SHADER_INPUT:
 618             hi->getSrc(s)->reg.data.offset += 4;
 619             break;
 620          default:
 621             assert(hi->src(s).getFile() == FILE_GPR);
 622             hi->getSrc(s)->reg.data.id++;
 623             break;
 624          }
 625       }
 626    }
 627    if (srcNr == 2) {
 628       lo->setDef(1, carry);
 629       hi->setFlagsSrc(hi->srcCount(), carry);
 630    }
 631    return hi;
 632 }
 633
 634 } // namespace nv50_ir