src/gallium/drivers/nv50/nv50_pc_emit.c

   1 /*
   2  * Copyright 2010 Christoph Bumiller
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  18  * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19  * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 #include "nv50_context.h"
  24 #include "nv50_pc.h"
  25
  26 // Definitions
  27
  28 #define FLAGS_CC_SHIFT    7
  29 #define FLAGS_ID_SHIFT    12
  30 #define FLAGS_WR_ID_SHIFT 4
  31 #define FLAGS_CC_MASK     (0x1f << FLAGS_CC_SHIFT)
  32 #define FLAGS_ID_MASK     (0x03 << FLAGS_ID_SHIFT)
  33 #define FLAGS_WR_EN       (1 << 6)
  34 #define FLAGS_WR_ID_MASK  (0x3 << FLAGS_WR_ID_SHIFT)
  35
  36 const ubyte nv50_inst_min_size_tab[NV_OP_COUNT] =
  37 {
  38    0, 0, 0, 8, 8, 4, 4, 4, 8, 4, 4, 8, 8, 8, 8, 8, /* 15 */
  39    8, 8, 8, 4, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, /* 31 */
  40    8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, /* 47 */
  41    4, 8, 8, 8, 8, 8, 0, 0, 8
  42 };
  43
  44 /* XXX: silence, you ! */
  45 unsigned
  46 nv50_inst_min_size(struct nv_instruction *i);
  47
  48 unsigned
  49 nv50_inst_min_size(struct nv_instruction *i)
  50 {
  51    int n;
  52
  53    if (nv50_inst_min_size_tab[i->opcode] > 4)
  54       return 8;
  55
  56    if (i->def[0] && i->def[0]->reg.file != NV_FILE_GPR)
  57       return 8;
  58    if (i->def[0]->join->reg.id > 63)
  59       return 8;
  60
  61    for (n = 0; n < 3; ++n) {
  62       if (!i->src[n])
  63          break;
  64       if (i->src[n]->value->reg.file != NV_FILE_GPR &&
  65           i->src[n]->value->reg.file != NV_FILE_MEM_V)
  66          return 8;
  67       if (i->src[n]->value->reg.id > 63)
  68          return 8;
  69    }
  70
  71    if (i->flags_def || i->flags_src || i->src[4])
  72       return 8;
  73
  74    if (i->is_join)
  75       return 8;
  76
  77    if (i->src[2]) {
  78       if (i->saturate || i->src[2]->mod)
  79          return 8;
  80       if (i->src[0]->mod ^ i->src[1]->mod)
  81          return 8;
  82       if ((i->src[0]->mod | i->src[1]->mod) & NV_MOD_ABS)
  83          return 8;
  84       if (i->def[0]->join->reg.id < 0 ||
  85           i->def[0]->join->reg.id != i->src[2]->value->join->reg.id)
  86          return 8;
  87    }
  88
  89    return nv50_inst_min_size_tab[i->opcode];
  90 }
  91
  92 static INLINE ubyte
  93 STYPE(struct nv_instruction *nvi, int s)
  94 {
  95    return nvi->src[s]->typecast;
  96 }
  97
  98 static INLINE ubyte
  99 DTYPE(struct nv_instruction *nvi, int d)
 100 {
 101    return nvi->def[d]->reg.type;
 102 }
 103
 104 static INLINE struct nv_reg *
 105 SREG(struct nv_ref *ref)
 106 {
 107    return &ref->value->join->reg;
 108 }
 109
 110 static INLINE struct nv_reg *
 111 DREG(struct nv_value *val)
 112 {
 113    return &val->join->reg;
 114 }
 115
 116 static INLINE ubyte
 117 SFILE(struct nv_instruction *nvi, int s)
 118 {
 119    return nvi->src[s]->value->reg.file;
 120 }
 121
 122 static INLINE ubyte
 123 DFILE(struct nv_instruction *nvi, int d)
 124 {
 125    return nvi->def[0]->reg.file;
 126 }
 127
 128 static INLINE void
 129 SID(struct nv_pc *pc, struct nv_ref *ref, int pos)
 130 {
 131    pc->emit[pos / 32] |= SREG(ref)->id << (pos % 32);
 132 }
 133
 134 static INLINE void
 135 DID(struct nv_pc *pc, struct nv_value *val, int pos)
 136 {
 137    pc->emit[pos / 32] |= DREG(val)->id << (pos % 32);
 138 }
 139
 140 static INLINE uint32_t
 141 get_immd_u32(struct nv_ref *ref)
 142 {
 143    assert(ref->value->reg.file == NV_FILE_IMM);
 144    return ref->value->reg.imm.u32;
 145 }
 146
 147 static INLINE void
 148 set_immd_u32(struct nv_pc *pc, uint32_t u32)
 149 {
 150    pc->emit[1] |= 3;
 151    pc->emit[0] |= (u32 & 0x3f) << 16;
 152    pc->emit[1] |= (u32 >> 6) << 2;
 153 }
 154
 155 static INLINE void
 156 set_immd(struct nv_pc *pc, struct nv_ref *ref)
 157 {
 158    assert(ref->value->reg.file == NV_FILE_IMM);
 159    set_immd_u32(pc, get_immd_u32(ref));
 160 }
 161
 162 static void
 163 new_fixup(struct nv_pc *pc, unsigned type, uint32_t data, uint32_t m, int s)
 164 {
 165    const unsigned size = sizeof(struct nv_fixup);
 166    const unsigned n = pc->num_fixups;
 167    return;
 168
 169    if (!(n % 8))
 170       pc->fixups = REALLOC(pc->fixups, n * size, (n + 8) * size);
 171
 172    pc->fixups[n].offset = pc->bin_pos + (s / 32);
 173    pc->fixups[n].type = type;
 174    pc->fixups[n].data = data;
 175    pc->fixups[n].mask = m << (s % 32);
 176    pc->fixups[n].shift = s % 32;
 177
 178    ++pc->num_fixups;
 179
 180    assert(((data << (s % 32)) & pc->fixups[n].mask) == (data << (s % 32)));
 181 }
 182
 183 static void
 184 nv_pc_alloc_immd(struct nv_pc *pc, struct nv_ref *ref)
 185 {
 186    uint32_t i, val = get_immd_u32(ref);
 187
 188    for (i = 0; i < pc->immd_count; ++i)
 189       if (pc->immd_buf[i] == val)
 190          break;
 191
 192    if (i == pc->immd_count) {
 193       if (!(pc->immd_count % 8))
 194          pc->immd_buf = REALLOC(pc->immd_buf,
 195                                 pc->immd_count * 4, (pc->immd_count + 8) * 4);
 196       pc->immd_buf[pc->immd_count++] = val;
 197    }
 198
 199    SREG(ref)->id = i;
 200 }
 201
 202 static INLINE void
 203 set_pred(struct nv_pc *pc, struct nv_instruction *i)
 204 {
 205    assert(!(pc->emit[1] & 0x00003f80));
 206
 207    pc->emit[1] |= i->cc << 7;
 208    if (i->flags_src)
 209       pc->emit[1] |= SREG(i->flags_src)->id << 12;
 210 }
 211
 212 static INLINE void
 213 set_pred_wr(struct nv_pc *pc, struct nv_instruction *i)
 214 {
 215    assert(!(pc->emit[1] & 0x00000070));
 216
 217    if (i->flags_def)
 218       pc->emit[1] |= (DREG(i->flags_def)->id << 4) | 0x40;
 219 }
 220
 221 static INLINE void
 222 set_a16_bits(struct nv_pc *pc, uint id)
 223 {
 224    ++id; /* $a0 is always 0 */
 225    pc->emit[0] |= (id & 3) << 26;
 226    pc->emit[1] |= id & 4;
 227 }
 228
 229 static INLINE void
 230 set_addr(struct nv_pc *pc, struct nv_instruction *i)
 231 {
 232    if (i->src[4])
 233       set_a16_bits(pc, SREG(i->src[4])->id);
 234 }
 235
 236 static void
 237 set_dst(struct nv_pc *pc, struct nv_value *value)
 238 {
 239    struct nv_reg *reg = &value->join->reg;
 240
 241    if (reg->id < 0) {
 242       debug_printf("WARNING: unused dst, hope we can bucket it !\n");
 243       pc->emit[0] |= 127 << 2;
 244       pc->emit[1] |= 0x8;
 245       return;
 246    }
 247
 248    if (reg->file == NV_FILE_OUT)
 249       pc->emit[1] |= 0x8;
 250    else
 251    if (reg->file == NV_FILE_ADDR)
 252            assert(0);
 253
 254    pc->emit[0] |= reg->id << 2;
 255 }
 256
 257 static void
 258 set_src_0(struct nv_pc *pc, struct nv_ref *ref)
 259 {
 260    struct nv_reg *reg = SREG(ref);
 261
 262    if (reg->file == NV_FILE_MEM_S)
 263       pc->emit[1] |= 0x00200000;
 264    else
 265    if (reg->file == NV_FILE_MEM_P)
 266       pc->emit[0] |= 0x01800000;
 267    else
 268    if (reg->file != NV_FILE_GPR)
 269       NOUVEAU_ERR("invalid src0 register file: %d\n", reg->file);
 270
 271    assert(reg->id < 128);
 272    pc->emit[0] |= reg->id << 9;
 273 }
 274
 275 static void
 276 set_src_1(struct nv_pc *pc, struct nv_ref *ref)
 277 {
 278    struct nv_reg *reg = SREG(ref);
 279
 280    if (reg->file >= NV_FILE_MEM_C(0) &&
 281        reg->file <= NV_FILE_MEM_C(15)) {
 282       assert(!(pc->emit[1] & 0x01800000));
 283
 284       pc->emit[0] |= 0x00800000;
 285       pc->emit[1] |= (reg->file - NV_FILE_MEM_C(0)) << 22;
 286    } else
 287    if (reg->file != NV_FILE_GPR)
 288       NOUVEAU_ERR("invalid src1 register file: %d\n", reg->file);
 289
 290    assert(reg->id < 128);
 291    pc->emit[0] |= reg->id << 16;
 292 }
 293
 294 static void
 295 set_src_2(struct nv_pc *pc, struct nv_ref *ref)
 296 {
 297    struct nv_reg *reg = SREG(ref);
 298
 299    if (reg->file >= NV_FILE_MEM_C(0) &&
 300        reg->file <= NV_FILE_MEM_C(15)) {
 301       assert(!(pc->emit[1] & 0x01800000));
 302
 303       pc->emit[0] |= 0x01000000;
 304       pc->emit[1] |= (reg->file - NV_FILE_MEM_C(0)) << 22;
 305    } else
 306    if (reg->file != NV_FILE_GPR)
 307       NOUVEAU_ERR("invalid src2 register file: %d\n", reg->file);
 308
 309    assert(reg->id < 128);
 310    pc->emit[1] |= reg->id << 14;
 311 }
 312
 313 /* the default form:
 314  * - long instruction
 315  * - 1 to 3 sources in slots 0, 1, 2
 316  * - address & flags
 317  */
 318 static void
 319 emit_form_MAD(struct nv_pc *pc, struct nv_instruction *i)
 320 {
 321    pc->emit[0] |= 1;
 322
 323    set_pred(pc, i);
 324    set_pred_wr(pc, i);
 325
 326    if (i->def[0])
 327       set_dst(pc, i->def[0]);
 328    else {
 329       pc->emit[0] |= 0x01fc;
 330       pc->emit[1] |= 0x0008;
 331    }
 332
 333    if (i->src[0])
 334       set_src_0(pc, i->src[0]);
 335
 336    if (i->src[1])
 337       set_src_1(pc, i->src[1]);
 338
 339    if (i->src[2])
 340       set_src_2(pc, i->src[2]);
 341
 342    set_addr(pc, i);
 343 }
 344
 345 /* like default form, but 2nd source in slot 2, no 3rd source */
 346 static void
 347 emit_form_ADD(struct nv_pc *pc, struct nv_instruction *i)
 348 {
 349    pc->emit[0] |= 1;
 350
 351    if (i->def[0])
 352       set_dst(pc, i->def[0]);
 353    else {
 354       pc->emit[0] |= 0x01fc;
 355       pc->emit[1] |= 0x0008;
 356    }
 357
 358    set_pred(pc, i);
 359    set_pred_wr(pc, i);
 360
 361    if (i->src[0])
 362       set_src_0(pc, i->src[0]);
 363
 364    if (i->src[1])
 365       set_src_2(pc, i->src[1]);
 366
 367    set_addr(pc, i);
 368 }
 369
 370 /* short mul */
 371 static void
 372 emit_form_MUL(struct nv_pc *pc, struct nv_instruction *i)
 373 {
 374    assert(!i->is_long && !(pc->emit[0] & 1));
 375
 376    assert(i->def[0]);
 377    set_dst(pc, i->def[0]);
 378
 379    if (i->src[0])
 380       set_src_0(pc, i->src[0]);
 381
 382    if (i->src[1])
 383       set_src_1(pc, i->src[1]);
 384 }
 385
 386 /* default immediate form
 387  * - 1 to 3 sources where last is immediate
 388  * - no address or predicate possible
 389  */
 390 static void
 391 emit_form_IMM(struct nv_pc *pc, struct nv_instruction *i, ubyte mod_mask)
 392 {
 393    pc->emit[0] |= 1;
 394
 395    assert(i->def[0]);
 396    assert(i->src[0]);
 397    set_dst(pc, i->def[0]);
 398
 399    assert(!i->src[4] && !i->flags_src && !i->flags_def);
 400
 401    if (i->src[2]) {
 402       set_immd(pc, i->src[2]);
 403       set_src_0(pc, i->src[1]);
 404       set_src_1(pc, i->src[0]);
 405    } else
 406    if (i->src[1]) {
 407       set_immd(pc, i->src[1]);
 408       set_src_0(pc, i->src[0]);
 409    } else
 410       set_immd(pc, i->src[0]);
 411
 412    assert(!mod_mask);
 413 }
 414
 415 static void
 416 set_ld_st_size(struct nv_pc *pc, ubyte type)
 417 {
 418    switch (type) {
 419    case NV_TYPE_F64:
 420       pc->emit[1] |= 0x8000;
 421       break;
 422    case NV_TYPE_F32:
 423    case NV_TYPE_S32:
 424    case NV_TYPE_U32:
 425       pc->emit[1] |= 0xc000;
 426       break;
 427    case NV_TYPE_S16:
 428       pc->emit[1] |= 0x6000;
 429       break;
 430    case NV_TYPE_U16:
 431       pc->emit[1] |= 0x4000;
 432       break;
 433    case NV_TYPE_S8:
 434       pc->emit[1] |= 0x2000;
 435       break;
 436    default:
 437       break;
 438    }
 439 }
 440
 441 static void
 442 emit_ld(struct nv_pc *pc, struct nv_instruction *i)
 443 {
 444    ubyte sf = SFILE(i, 0);
 445
 446    if (sf == NV_FILE_IMM) {
 447       sf = NV_FILE_MEM_C(0);
 448       nv_pc_alloc_immd(pc, i->src[0]);
 449
 450       new_fixup(pc, NV_FIXUP_PARAM_RELOC, SREG(i->src[0])->id, 0xffff, 9);
 451    }
 452
 453    if (sf == NV_FILE_MEM_S ||
 454        sf == NV_FILE_MEM_P) {
 455       pc->emit[0] = 0x10000001;
 456       pc->emit[1] = 0x04200000 | (0x3c << 12);
 457       if (sf == NV_FILE_MEM_P)
 458          pc->emit[0] |= 0x01800000;
 459    } else
 460    if (sf >= NV_FILE_MEM_C(0) &&
 461        sf <= NV_FILE_MEM_C(15)) {
 462       pc->emit[0] = 0x10000001;
 463       pc->emit[1] = 0x24000000;
 464       pc->emit[1] |= (sf - NV_FILE_MEM_C(0)) << 22;
 465    } else
 466    if (sf >= NV_FILE_MEM_G(0) &&
 467        sf <= NV_FILE_MEM_G(15)) {
 468       pc->emit[0] = 0xd0000001 | ((sf - NV_FILE_MEM_G(0)) << 16);
 469       pc->emit[1] = 0xa0000000;
 470
 471       assert(i->src[4] && SREG(i->src[4])->file == NV_FILE_GPR);
 472       SID(pc, i->src[4], 9);
 473    } else
 474    if (sf == NV_FILE_MEM_L) {
 475       pc->emit[0] = 0xd0000001;
 476       pc->emit[1] = 0x40000000;
 477    } else {
 478       NOUVEAU_ERR("invalid ld source file\n");
 479       abort();
 480    }
 481
 482    set_ld_st_size(pc, STYPE(i, 0));
 483
 484    set_dst(pc, i->def[0]);
 485    set_pred_wr(pc, i);
 486
 487    set_pred(pc, i);
 488
 489    if (sf < NV_FILE_MEM_G(0) ||
 490        sf > NV_FILE_MEM_G(15)) {
 491       SID(pc, i->src[0], 9);
 492       set_addr(pc, i);
 493    }
 494 }
 495
 496 static void
 497 emit_st(struct nv_pc *pc, struct nv_instruction *i)
 498 {
 499
 500 }
 501
 502 static int
 503 verify_mov(struct nv_instruction *i)
 504 {
 505    ubyte sf = SFILE(i, 0);
 506    ubyte df = DFILE(i, 0);
 507
 508    if (df == NV_FILE_GPR)
 509       return 0;
 510
 511    if (df != NV_FILE_OUT &&
 512        df != NV_FILE_FLAGS &&
 513        df != NV_FILE_ADDR)
 514       return 1;
 515
 516    if (sf == NV_FILE_FLAGS)
 517       return 2;
 518    if (sf == NV_FILE_ADDR)
 519       return 3;
 520    if (sf == NV_FILE_IMM && df != NV_FILE_OUT)
 521       return 4;
 522
 523    return 0;
 524 }
 525
 526 static void
 527 emit_mov(struct nv_pc *pc, struct nv_instruction *i)
 528 {
 529    assert(!verify_mov(i));
 530
 531    if (SFILE(i, 0) >= NV_FILE_MEM_S)
 532       emit_ld(pc, i);
 533    else
 534    if (SFILE(i, 0) == NV_FILE_FLAGS) {
 535       pc->emit[0] = 0x00000001 | (DREG(i->def[0])->id << 2);
 536       pc->emit[1] = 0x20000780 | (SREG(i->src[0])->id << 12);
 537    } else
 538    if (SFILE(i, 0) == NV_FILE_ADDR) {
 539       pc->emit[0] = 0x00000001 | (DREG(i->def[0])->id << 2);
 540       pc->emit[1] = 0x40000780;
 541       set_a16_bits(pc, SREG(i->src[0])->id);
 542    } else
 543    if (DFILE(i, 0) == NV_FILE_FLAGS) {
 544       pc->emit[0] = 0x000001fd;
 545       pc->emit[1] = 0xa0000788 | (1 << 6);
 546       pc->emit[0] |= SREG(i->src[0])->id << 9;
 547       pc->emit[1] |= DREG(i->def[0])->id << 4;
 548    } else
 549    if (SFILE(i, 0) == NV_FILE_IMM) {
 550       if (i->opcode == NV_OP_LDA) {
 551          emit_ld(pc, i);
 552       } else {
 553          pc->emit[0] = 0x10008001;
 554          pc->emit[1] = 0x00000003;
 555
 556          emit_form_IMM(pc, i, 0);
 557       }
 558    } else {
 559       pc->emit[0] = 0x10000000;
 560       pc->emit[0] |= DREG(i->def[0])->id << 2;
 561       pc->emit[0] |= SREG(i->src[0])->id << 9;
 562
 563       if (!i->is_long) {
 564          pc->emit[0] |= 0x8000;
 565       } else {
 566          pc->emit[0] |= 0x00000001;
 567          pc->emit[1] = 0x0403c000;
 568
 569          set_pred(pc, i);
 570       }
 571    }
 572
 573    if (DFILE(i, 0) == NV_FILE_OUT)
 574       pc->emit[1] |= 0x8;
 575 }
 576
 577 static void
 578 emit_interp(struct nv_pc *pc, struct nv_instruction *i)
 579 {
 580    pc->emit[0] = 0x80000000;
 581
 582    assert(DFILE(i, 0) == NV_FILE_GPR);
 583    assert(SFILE(i, 0) == NV_FILE_MEM_V);
 584
 585    DID(pc, i->def[0], 2);
 586    SID(pc, i->src[0], 16);
 587
 588    if (i->flat)
 589       pc->emit[0] |= 1 << 8;
 590    else
 591    if (i->opcode == NV_OP_PINTERP) {
 592       pc->emit[0] |= 1 << 25;
 593       pc->emit[0] |= SREG(i->src[1])->id << 9;
 594    }
 595
 596    if (i->centroid)
 597       pc->emit[0] |= 1 << 24;
 598
 599    assert(i->is_long || !i->flags_src);
 600
 601    if (i->is_long) {
 602       set_pred(pc, i);
 603
 604       pc->emit[1] |=
 605               (pc->emit[0] & (3 << 24)) >> (24 - 16) |
 606               (pc->emit[0] & (1 <<  8)) >> (18 -  8);
 607
 608       pc->emit[0] |= 1;
 609       pc->emit[0] &= ~0x03000100;
 610    }
 611 }
 612
 613 static void
 614 emit_minmax(struct nv_pc *pc, struct nv_instruction *i)
 615 {
 616    pc->emit[0] = 0x30000000;
 617    pc->emit[1] = (i->opcode == NV_OP_MIN) ? (2 << 28) : 0;
 618
 619    switch (DTYPE(i, 0)) {
 620    case NV_TYPE_F32:
 621       pc->emit[0] |= 0x80000000;
 622       pc->emit[1] |= 0x80000000;
 623       break;
 624    case NV_TYPE_S32:
 625       pc->emit[1] |= 0x8c000000;
 626       break;
 627    case NV_TYPE_U32:
 628       pc->emit[1] |= 0x84000000;
 629       break;
 630    }
 631
 632    emit_form_MAD(pc, i);
 633
 634    if (i->src[0]->mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
 635    if (i->src[1]->mod & NV_MOD_ABS) pc->emit[1] |= 0x00080000;
 636 }
 637
 638 static void
 639 emit_add_f32(struct nv_pc *pc, struct nv_instruction *i)
 640 {
 641    pc->emit[0] = 0xb0000000;
 642
 643    if (SFILE(i, 1) == NV_FILE_IMM) {
 644       emit_form_IMM(pc, i, 0);
 645
 646       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 0x8000;
 647       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 648    } else
 649    if (i->is_long) {
 650       emit_form_ADD(pc, i);
 651
 652       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[1] |= 1 << 26;
 653       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[1] |= 1 << 27;
 654    } else {
 655       emit_form_MUL(pc, i);
 656
 657       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 0x8000;
 658       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 659    }
 660 }
 661
 662 static void
 663 emit_add_b32(struct nv_pc *pc, struct nv_instruction *i)
 664 {
 665    pc->emit[0] = 0x20008000;
 666
 667    if (SFILE(i, 1) == NV_FILE_IMM) {
 668       emit_form_IMM(pc, i, 0);
 669    } else
 670    if (i->is_long) {
 671       pc->emit[0] = 0x20000000;
 672       pc->emit[1] = 0x04000000;
 673       emit_form_ADD(pc, i);
 674    } else {
 675       emit_form_MUL(pc, i);
 676    }
 677
 678    if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 28;
 679    if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 680 }
 681
 682 static void
 683 emit_add_a16(struct nv_pc *pc, struct nv_instruction *i)
 684 {
 685    pc->emit[0] = 0xd0000001 | (get_immd_u32(i->src[0]) << 9);
 686    pc->emit[1] = 0x20000000;
 687
 688    pc->emit[0] |= (DREG(i->def[0])->id + 1) << 2;
 689
 690    set_pred(pc, i);
 691
 692    if (i->src[1])
 693       set_a16_bits(pc, SREG(i->src[1])->id);
 694 }
 695
 696 static void
 697 emit_flow(struct nv_pc *pc, struct nv_instruction *i, ubyte flow_op)
 698 {
 699    pc->emit[0] = 0x00000003 | (flow_op << 28);
 700    pc->emit[1] = 0x00000000;
 701
 702    set_pred(pc, i);
 703
 704    if (i->target && (i->opcode != NV_OP_BREAK)) {
 705       new_fixup(pc, NV_FIXUP_CFLOW_RELOC, i->target->bin_pos, 0x7ff800, 11);
 706       pc->emit[0] |= (i->target->bin_pos / 4) << 11;
 707    }
 708 }
 709
 710 static INLINE void
 711 emit_add(struct nv_pc *pc, struct nv_instruction *i)
 712 {
 713    if (DFILE(i, 0) == NV_FILE_ADDR)
 714       emit_add_a16(pc, i);
 715    else {
 716       switch (DTYPE(i, 0)) {
 717       case NV_TYPE_F32:
 718          emit_add_f32(pc, i);
 719          break;
 720       case NV_TYPE_U32:
 721       case NV_TYPE_S32:
 722          emit_add_b32(pc, i);
 723          break;
 724       }
 725    }
 726 }
 727
 728 static void
 729 emit_bitop2(struct nv_pc *pc, struct nv_instruction *i)
 730 {
 731    pc->emit[0] = 0xd0000000;
 732
 733    if (SFILE(i, 0) == NV_FILE_IMM) {
 734       emit_form_IMM(pc, i, 0);
 735
 736       if (i->opcode == NV_OP_OR)
 737          pc->emit[0] |= 0x0100;
 738       else
 739       if (i->opcode == NV_OP_XOR)
 740          pc->emit[0] |= 0x8000;
 741    } else {
 742       emit_form_MAD(pc, i);
 743
 744       pc->emit[1] |= 0x04000000;
 745
 746       if (i->opcode == NV_OP_OR)
 747          pc->emit[1] |= 0x4000;
 748       else
 749       if (i->opcode == NV_OP_XOR)
 750          pc->emit[1] |= 0x8000;
 751    }
 752 }
 753
 754 static void
 755 emit_arl(struct nv_pc *pc, struct nv_instruction *i)
 756 {
 757    assert(SFILE(i, 0) == NV_FILE_GPR);
 758    assert(SFILE(i, 1) == NV_FILE_IMM);
 759
 760    assert(!i->flags_def);
 761
 762    pc->emit[0] = 0x00000001;
 763    pc->emit[1] = 0xc0000000;
 764
 765    set_dst(pc, i->def[0]);
 766    set_pred(pc, i);
 767    set_src_0(pc, i->src[0]);
 768    pc->emit[0] |= (get_immd_u32(i->src[1]) & 0x3f) << 16;
 769 }
 770
 771 static void
 772 emit_shift(struct nv_pc *pc, struct nv_instruction *i)
 773 {
 774    if (DFILE(i, 0) == NV_FILE_ADDR) {
 775       emit_arl(pc, i);
 776       return;
 777    }
 778
 779    pc->emit[0] = 0x30000001;
 780    pc->emit[1] = 0xc4000000;
 781
 782    if (i->opcode == NV_OP_SHR)
 783       pc->emit[1] |= 1 << 29;
 784
 785    if (SFILE(i, 1) == NV_FILE_IMM) {
 786       pc->emit[1] |= 1 << 20;
 787       pc->emit[0] |= (get_immd_u32(i->src[1]) & 0x7f) << 16;
 788
 789       set_pred(pc, i);
 790    } else
 791       emit_form_MAD(pc, i);
 792
 793    if (STYPE(i, 0) == NV_TYPE_S32)
 794       pc->emit[1] |= 1 << 27;
 795 }
 796
 797 static void
 798 emit_flop(struct nv_pc *pc, struct nv_instruction *i)
 799 {
 800    struct nv_ref *src0 = i->src[0];
 801
 802    pc->emit[0] = 0x90000000;
 803
 804    assert(SREG(src0)->type == NV_TYPE_F32);
 805    assert(SREG(src0)->file == NV_FILE_GPR);
 806
 807    if (!i->is_long) {
 808       emit_form_MUL(pc, i);
 809       assert(i->opcode == NV_OP_RCP && !src0->mod);
 810       return;
 811    }
 812
 813    pc->emit[1] = (i->opcode - NV_OP_RCP) << 29;
 814
 815    emit_form_MAD(pc, i);
 816
 817    if (src0->mod & NV_MOD_NEG) pc->emit[1] |= 0x04000000;
 818    if (src0->mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
 819 }
 820
 821 static void
 822 emit_mad_f32(struct nv_pc *pc, struct nv_instruction *i)
 823 {
 824    const boolean neg_mul = (i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG;
 825    const boolean neg_add = (i->src[2]->mod & NV_MOD_NEG);
 826
 827    pc->emit[0] = 0xe0000000;
 828
 829    if (!i->is_long) {
 830       emit_form_MUL(pc, i);
 831       assert(!neg_mul && !neg_add);
 832       return;
 833    }
 834
 835    emit_form_MAD(pc, i);
 836
 837    if (neg_mul) pc->emit[1] |= 0x04000000;
 838    if (neg_add) pc->emit[1] |= 0x08000000;
 839
 840    if (i->saturate)
 841       pc->emit[1] |= 0x20000000;
 842 }
 843
 844 static INLINE void
 845 emit_mad(struct nv_pc *pc, struct nv_instruction *i)
 846 {
 847    emit_mad_f32(pc, i);
 848 }
 849
 850 static void
 851 emit_mul_f32(struct nv_pc *pc, struct nv_instruction *i)
 852 {
 853    boolean neg = (i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG;
 854
 855    pc->emit[0] = 0xc0000000;
 856
 857    if (SFILE(i, 1) == NV_FILE_IMM) {
 858       emit_form_IMM(pc, i, 0);
 859
 860       if (neg)
 861          pc->emit[0] |= 0x8000;
 862    } else
 863    if (i->is_long) {
 864       emit_form_MAD(pc, i);
 865
 866       if (neg)
 867          pc->emit[1] |= 0x08 << 24;
 868    } else {
 869       emit_form_MUL(pc, i);
 870
 871       if (neg)
 872          pc->emit[0] |= 0x8000;
 873    }
 874 }
 875
 876 static void
 877 emit_set(struct nv_pc *pc, struct nv_instruction *nvi)
 878 {
 879    assert(nvi->is_long);
 880
 881    pc->emit[0] = 0x30000000;
 882    pc->emit[1] = 0x60000000;
 883
 884    pc->emit[1] |= nvi->set_cond << 14;
 885
 886    switch (STYPE(nvi, 0)) {
 887    case NV_TYPE_U32: pc->emit[1] |= 0x04000000; break;
 888    case NV_TYPE_S32: pc->emit[1] |= 0x0c000000; break;
 889    case NV_TYPE_F32: pc->emit[0] |= 0x80000000; break;
 890    default:
 891       assert(0);
 892       break;
 893    }
 894
 895    emit_form_MAD(pc, nvi);
 896 }
 897
 898 #define CVT_RN    (0x00 << 16)
 899 #define CVT_FLOOR (0x02 << 16)
 900 #define CVT_CEIL  (0x04 << 16)
 901 #define CVT_TRUNC (0x06 << 16)
 902 #define CVT_SAT   (0x08 << 16)
 903 #define CVT_ABS   (0x10 << 16)
 904
 905 #define CVT_X32_X32 0x04004000
 906 #define CVT_X32_S32 0x04014000
 907 #define CVT_F32_F32 ((0xc0 << 24) | CVT_X32_X32)
 908 #define CVT_S32_F32 ((0x88 << 24) | CVT_X32_X32)
 909 #define CVT_U32_F32 ((0x80 << 24) | CVT_X32_X32)
 910 #define CVT_F32_S32 ((0x40 << 24) | CVT_X32_S32)
 911 #define CVT_F32_U32 ((0x40 << 24) | CVT_X32_X32)
 912 #define CVT_S32_S32 ((0x08 << 24) | CVT_X32_S32)
 913 #define CVT_S32_U32 ((0x08 << 24) | CVT_X32_X32)
 914 #define CVT_U32_S32 ((0x00 << 24) | CVT_X32_S32)
 915 #define CVT_U32_U32 ((0x00 << 24) | CVT_X32_X32)
 916
 917 #define CVT_NEG 0x20000000
 918 #define CVT_RI  0x08000000
 919
 920 static void
 921 emit_cvt(struct nv_pc *pc, struct nv_instruction *nvi)
 922 {
 923    ubyte dst_type = nvi->def[0] ? DTYPE(nvi, 0) : STYPE(nvi, 0);
 924
 925    pc->emit[0] = 0xa0000000;
 926
 927    switch (dst_type) {
 928    case NV_TYPE_F32:
 929       switch (STYPE(nvi, 0)) {
 930       case NV_TYPE_F32: pc->emit[1] = CVT_F32_F32; break;
 931       case NV_TYPE_S32: pc->emit[1] = CVT_F32_S32; break;
 932       case NV_TYPE_U32: pc->emit[1] = CVT_F32_U32; break;
 933       }
 934       break;
 935    case NV_TYPE_S32:
 936       switch (STYPE(nvi, 0)) {
 937       case NV_TYPE_F32: pc->emit[1] = CVT_S32_F32; break;
 938       case NV_TYPE_S32: pc->emit[1] = CVT_S32_S32; break;
 939       case NV_TYPE_U32: pc->emit[1] = CVT_S32_U32; break;
 940       }
 941       break;
 942    case NV_TYPE_U32:
 943       switch (STYPE(nvi, 0)) {
 944       case NV_TYPE_F32: pc->emit[1] = CVT_U32_F32; break;
 945       case NV_TYPE_S32: pc->emit[1] = CVT_U32_S32; break;
 946       case NV_TYPE_U32: pc->emit[1] = CVT_U32_U32; break;
 947       }
 948       break;
 949    }
 950    if (pc->emit[1] == CVT_F32_F32 &&
 951        (nvi->opcode == NV_OP_CEIL || nvi->opcode == NV_OP_FLOOR ||
 952         nvi->opcode == NV_OP_TRUNC))
 953        pc->emit[1] |= CVT_RI;
 954
 955    switch (nvi->opcode) {
 956    case NV_OP_CEIL:  pc->emit[1] |= CVT_CEIL; break;
 957    case NV_OP_FLOOR: pc->emit[1] |= CVT_FLOOR; break;
 958    case NV_OP_TRUNC: pc->emit[1] |= CVT_TRUNC; break;
 959
 960    case NV_OP_ABS: pc->emit[1] |= CVT_ABS; break;
 961    case NV_OP_SAT: pc->emit[1] |= CVT_SAT; break;
 962    case NV_OP_NEG: pc->emit[1] |= CVT_NEG; break;
 963    default:
 964       assert(nvi->opcode == NV_OP_CVT);
 965       break;
 966    }
 967    assert(nvi->opcode != NV_OP_ABS || !(nvi->src[0]->mod & NV_MOD_NEG));
 968
 969    if (nvi->src[0]->mod & NV_MOD_NEG) pc->emit[1] ^= CVT_NEG;
 970    if (nvi->src[0]->mod & NV_MOD_ABS) pc->emit[1] |= CVT_ABS;
 971
 972    emit_form_MAD(pc, nvi);
 973 }
 974
 975 static void
 976 emit_tex(struct nv_pc *pc, struct nv_instruction *i)
 977 {
 978    pc->emit[0] = 0xf0000001;
 979    pc->emit[1] = 0x00000000;
 980
 981    DID(pc, i->def[0], 2);
 982
 983    set_pred(pc, i);
 984
 985    pc->emit[0] |= i->tex_t << 9;
 986    pc->emit[0] |= i->tex_s << 17;
 987
 988    pc->emit[0] |= i->tex_argc << 22;
 989
 990    pc->emit[0] |= (i->tex_mask & 0x3) << 25;
 991    pc->emit[1] |= (i->tex_mask & 0xc) << 12;
 992
 993    if (i->tex_live)
 994       pc->emit[1] |= 4;
 995
 996    if (i->tex_cube)
 997       pc->emit[0] |= 0x08000000;
 998
 999    if (i->opcode == NV_OP_TXB)
1000       pc->emit[1] |= 0x20000000;
1001    else
1002    if (i->opcode == NV_OP_TXL)
1003       pc->emit[1] |= 0x40000000;
1004    else
1005       pc->emit[0] -= 1 << 22;
1006 }
1007
1008 static void
1009 emit_cvt2fixed(struct nv_pc *pc, struct nv_instruction *i)
1010 {
1011    ubyte mod = i->src[0]->mod;
1012
1013    pc->emit[0] = 0xb0000000;
1014    pc->emit[1] = 0xc0000000;
1015
1016    if (i->opcode == NV_OP_PREEX2)
1017       pc->emit[1] |= 0x4000;
1018
1019    emit_form_MAD(pc, i);
1020
1021    if (mod & NV_MOD_NEG) pc->emit[1] |= 0x04000000;
1022    if (mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
1023 }
1024
1025 static void
1026 emit_ddx(struct nv_pc *pc, struct nv_instruction *i)
1027 {
1028    assert(i->is_long && SFILE(i, 0) == NV_FILE_GPR);
1029
1030    pc->emit[0] = (i->src[0]->mod & NV_MOD_NEG) ? 0xc0240001 : 0xc0140001;
1031    pc->emit[1] = (i->src[0]->mod & NV_MOD_NEG) ? 0x86400000 : 0x89800000;
1032
1033    DID(pc, i->def[0], 2);
1034    SID(pc, i->src[0], 9);
1035    SID(pc, i->src[0], 32 + 14);
1036
1037    set_pred(pc, i);
1038    set_pred_wr(pc, i);
1039 }
1040
1041 static void
1042 emit_ddy(struct nv_pc *pc, struct nv_instruction *i)
1043 {
1044    assert(i->is_long && SFILE(i, 0) == NV_FILE_GPR);
1045
1046    pc->emit[0] = (i->src[0]->mod & NV_MOD_NEG) ? 0xc0250001 : 0xc0150001;
1047    pc->emit[1] = (i->src[0]->mod & NV_MOD_NEG) ? 0x85800000 : 0x8a400000;
1048
1049    DID(pc, i->def[0], 2);
1050    SID(pc, i->src[0], 9);
1051    SID(pc, i->src[0], 32 + 14);
1052
1053    set_pred(pc, i);
1054    set_pred_wr(pc, i);
1055 }
1056
1057 void
1058 nv50_emit_instruction(struct nv_pc *pc, struct nv_instruction *i)
1059 {
1060    // nv_print_instruction(i);
1061
1062    switch (i->opcode) {
1063    case NV_OP_MOV:
1064       if (DFILE(i, 0) == NV_FILE_ADDR)
1065          emit_add_a16(pc, i);
1066       else
1067          emit_mov(pc, i);
1068       break;
1069    case NV_OP_LDA:
1070       emit_mov(pc, i);
1071       break;
1072    case NV_OP_STA:
1073       emit_st(pc, i);
1074       break;
1075    case NV_OP_LINTERP:
1076    case NV_OP_PINTERP:
1077       emit_interp(pc, i);
1078       break;
1079    case NV_OP_ADD:
1080       emit_add(pc, i);
1081       break;
1082    case NV_OP_AND:
1083    case NV_OP_OR:
1084    case NV_OP_XOR:
1085       emit_bitop2(pc, i);
1086       break;
1087    case NV_OP_CVT:
1088    case NV_OP_ABS:
1089    case NV_OP_NEG:
1090    case NV_OP_SAT:
1091    case NV_OP_CEIL:
1092    case NV_OP_FLOOR:
1093    case NV_OP_TRUNC:
1094       emit_cvt(pc, i);
1095       break;
1096    case NV_OP_DFDX:
1097       emit_ddx(pc, i);
1098       break;
1099    case NV_OP_DFDY:
1100       emit_ddy(pc, i);
1101       break;
1102    case NV_OP_RCP:
1103    case NV_OP_RSQ:
1104    case NV_OP_LG2:
1105    case NV_OP_SIN:
1106    case NV_OP_COS:
1107    case NV_OP_EX2:
1108       emit_flop(pc, i);
1109       break;
1110    case NV_OP_PRESIN:
1111    case NV_OP_PREEX2:
1112       emit_cvt2fixed(pc, i);
1113       break;
1114    case NV_OP_MAD:
1115       emit_mad(pc, i);
1116       break;
1117    case NV_OP_MAX:
1118    case NV_OP_MIN:
1119       emit_minmax(pc, i);
1120       break;
1121    case NV_OP_MUL:
1122       emit_mul_f32(pc, i);
1123       break;
1124    case NV_OP_SET:
1125       emit_set(pc, i);
1126       break;
1127    case NV_OP_SHL:
1128    case NV_OP_SHR:
1129       emit_shift(pc, i);
1130       break;
1131    case NV_OP_TEX:
1132    case NV_OP_TXB:
1133    case NV_OP_TXL:
1134       emit_tex(pc, i);
1135       break;
1136    case NV_OP_KIL:
1137       emit_flow(pc, i, 0x0);
1138       break;
1139    case NV_OP_BRA:
1140       emit_flow(pc, i, 0x1);
1141       break;
1142    case NV_OP_CALL:
1143       emit_flow(pc, i, 0x2);
1144       break;
1145    case NV_OP_RET:
1146       emit_flow(pc, i, 0x3);
1147       break;
1148    case NV_OP_BREAKADDR:
1149       emit_flow(pc, i, 0x4);
1150       break;
1151    case NV_OP_BREAK:
1152       emit_flow(pc, i, 0x5);
1153       break;
1154    case NV_OP_JOINAT:
1155       emit_flow(pc, i, 0xa);
1156       break;
1157    case NV_OP_NOP:
1158    case NV_OP_JOIN:
1159       pc->emit[0] = 0xf0000001;
1160       pc->emit[1] = 0xe0000000;
1161       break;
1162    case NV_OP_PHI:
1163    case NV_OP_UNDEF:
1164    case NV_OP_SUB:
1165       NOUVEAU_ERR("operation \"%s\" should have been eliminated\n",
1166                   nv_opcode_name(i->opcode));
1167       break;
1168    default:
1169       NOUVEAU_ERR("unhandled NV_OP: %d\n", i->opcode);
1170       abort();
1171       break;
1172    }
1173
1174    if (i->is_join) {
1175       assert(i->is_long && !(pc->emit[1] & 1));
1176       pc->emit[1] |= 2;
1177    }
1178
1179    assert((pc->emit[0] & 1) == i->is_long);
1180 }