src/gallium/drivers/nv50/nv50_pc_emit.c

   1 /*************************************************************************/
   2 /* Copyright (C) 2009                                                    */
   3 /*                                                                       */
   4 /* This program is free software: you can redistribute it and/or modify  */
   5 /* it under the terms of the GNU General Public License as published by  */
   6 /* the Free Software Foundation, either version 3 of the License, or     */
   7 /* (at your option) any later version.                                   */
   8 /*                                                                       */
   9 /* This program is distributed in the hope that it will be useful,       */
  10 /* but WITHOUT ANY WARRANTY; without even the implied warranty of        */
  11 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         */
  12 /* GNU General Public License for more details.                          */
  13 /*                                                                       */
  14 /* You should have received a copy of the GNU General Public License     */
  15 /* along with this program.  If not, see <http://www.gnu.org/licenses/>. */
  16 /*************************************************************************/
  17
  18 #include "nv50_context.h"
  19 #include "nv50_pc.h"
  20
  21 // Definitions
  22
  23 #define FLAGS_CC_SHIFT    7
  24 #define FLAGS_ID_SHIFT    12
  25 #define FLAGS_WR_ID_SHIFT 4
  26 #define FLAGS_CC_MASK     (0x1f << FLAGS_CC_SHIFT)
  27 #define FLAGS_ID_MASK     (0x03 << FLAGS_ID_SHIFT)
  28 #define FLAGS_WR_EN       (1 << 6)
  29 #define FLAGS_WR_ID_MASK  (0x3 << FLAGS_WR_ID_SHIFT)
  30
  31 const ubyte nv50_inst_min_size_tab[NV_OP_COUNT] =
  32 {
  33    0, 0, 0, 8, 8, 4, 4, 4, 8, 4, 4, 8, 8, 8, 8, 8, /* 15 */
  34    8, 8, 8, 4, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, /* 31 */
  35    8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, /* 47 */
  36    4, 8, 8, 8, 8, 8, 0, 0
  37 };
  38
  39 /* XXX: silence, you ! */
  40 unsigned
  41 nv50_inst_min_size(struct nv_instruction *i);
  42
  43 unsigned
  44 nv50_inst_min_size(struct nv_instruction *i)
  45 {
  46    int n;
  47
  48    if (nv50_inst_min_size_tab[i->opcode] > 4)
  49       return 8;
  50
  51    if (i->def[0] && i->def[0]->reg.file != NV_FILE_GPR)
  52       return 8;
  53    if (i->def[0]->join->reg.id > 63)
  54       return 8;
  55
  56    for (n = 0; n < 3; ++n) {
  57       if (!i->src[n])
  58          break;
  59       if (i->src[n]->value->reg.file != NV_FILE_GPR &&
  60           i->src[n]->value->reg.file != NV_FILE_MEM_V)
  61          return 8;
  62       if (i->src[n]->value->reg.id > 63)
  63          return 8;
  64    }
  65
  66    if (i->flags_def || i->flags_src || i->src[4])
  67       return 8;
  68
  69    if (i->src[2]) {
  70       if (i->saturate || i->src[2]->mod)
  71          return 8;
  72       if (i->src[0]->mod ^ i->src[1]->mod)
  73          return 8;
  74       if ((i->src[0]->mod | i->src[1]->mod) & NV_MOD_ABS)
  75          return 8;
  76       if (i->def[0]->join->reg.id < 0 ||
  77           i->def[0]->join->reg.id != i->src[2]->value->join->reg.id)
  78          return 8;
  79    }
  80
  81    return nv50_inst_min_size_tab[i->opcode];
  82 }
  83
  84 static INLINE ubyte
  85 STYPE(struct nv_instruction *nvi, int s)
  86 {
  87    return nvi->src[s]->typecast;
  88 }
  89
  90 static INLINE ubyte
  91 DTYPE(struct nv_instruction *nvi, int d)
  92 {
  93    return nvi->def[d]->reg.type;
  94 }
  95
  96 static INLINE struct nv_reg *
  97 SREG(struct nv_ref *ref)
  98 {
  99    return &ref->value->join->reg;
 100 }
 101
 102 static INLINE struct nv_reg *
 103 DREG(struct nv_value *val)
 104 {
 105    return &val->join->reg;
 106 }
 107
 108 static INLINE ubyte
 109 SFILE(struct nv_instruction *nvi, int s)
 110 {
 111    return nvi->src[s]->value->reg.file;
 112 }
 113
 114 static INLINE ubyte
 115 DFILE(struct nv_instruction *nvi, int d)
 116 {
 117    return nvi->def[0]->reg.file;
 118 }
 119
 120 static INLINE void
 121 SID(struct nv_pc *pc, struct nv_ref *ref, int pos)
 122 {
 123    pc->emit[pos / 32] |= SREG(ref)->id << (pos % 32);
 124 }
 125
 126 static INLINE void
 127 DID(struct nv_pc *pc, struct nv_value *val, int pos)
 128 {
 129    pc->emit[pos / 32] |= DREG(val)->id << (pos % 32);
 130 }
 131
 132 static INLINE uint32_t
 133 get_immd_u32(struct nv_ref *ref)
 134 {
 135    assert(ref->value->reg.file == NV_FILE_IMM);
 136    return ref->value->reg.imm.u32;
 137 }
 138
 139 static INLINE void
 140 set_immd_u32(struct nv_pc *pc, uint32_t u32)
 141 {
 142    pc->emit[1] |= 3;
 143    pc->emit[0] |= (u32 & 0x3f) << 16;
 144    pc->emit[1] |= (u32 >> 6) << 2;
 145 }
 146
 147 static INLINE void
 148 set_immd(struct nv_pc *pc, struct nv_ref *ref)
 149 {
 150    assert(ref->value->reg.file == NV_FILE_IMM);
 151    set_immd_u32(pc, get_immd_u32(ref));
 152 }
 153
 154 static void
 155 new_fixup(struct nv_pc *pc, unsigned type, uint32_t data, uint32_t m, int s)
 156 {
 157    const unsigned size = sizeof(struct nv_fixup);
 158    const unsigned n = pc->num_fixups;
 159    return;
 160
 161    if (!(n % 8))
 162       pc->fixups = REALLOC(pc->fixups, n * size, (n + 8) * size);
 163
 164    pc->fixups[n].offset = pc->bin_pos + (s / 32);
 165    pc->fixups[n].type = type;
 166    pc->fixups[n].data = data;
 167    pc->fixups[n].mask = m << (s % 32);
 168    pc->fixups[n].shift = s % 32;
 169
 170    ++pc->num_fixups;
 171
 172    assert(((data << (s % 32)) & pc->fixups[n].mask) == (data << (s % 32)));
 173 }
 174
 175 static void
 176 nv_pc_alloc_immd(struct nv_pc *pc, struct nv_ref *ref)
 177 {
 178    uint32_t i, val = get_immd_u32(ref);
 179
 180    for (i = 0; i < pc->immd_count; ++i)
 181       if (pc->immd_buf[i] == val)
 182          break;
 183
 184    if (i == pc->immd_count) {
 185       if (!(pc->immd_count % 8))
 186          pc->immd_buf = REALLOC(pc->immd_buf,
 187                                 pc->immd_count * 4, (pc->immd_count + 8) * 4);
 188       pc->immd_buf[pc->immd_count++] = val;
 189    }
 190
 191    SREG(ref)->id = i;
 192 }
 193
 194 static INLINE void
 195 set_pred(struct nv_pc *pc, struct nv_instruction *i)
 196 {
 197    assert(!(pc->emit[1] & 0x00003f80));
 198
 199    pc->emit[1] |= i->cc << 7;
 200    if (i->flags_src)
 201       pc->emit[1] |= SREG(i->flags_src)->id << 12;
 202 }
 203
 204 static INLINE void
 205 set_pred_wr(struct nv_pc *pc, struct nv_instruction *i)
 206 {
 207    assert(!(pc->emit[1] & 0x00000070));
 208
 209    if (i->flags_def)
 210       pc->emit[1] |= (DREG(i->flags_def)->id << 4) | 0x40;
 211 }
 212
 213 static INLINE void
 214 set_a16_bits(struct nv_pc *pc, uint id)
 215 {
 216    ++id; /* $a0 is always 0 */
 217    pc->emit[0] |= (id & 3) << 26;
 218    pc->emit[1] |= id & 4;
 219 }
 220
 221 static INLINE void
 222 set_addr(struct nv_pc *pc, struct nv_instruction *i)
 223 {
 224    if (i->src[4])
 225       set_a16_bits(pc, SREG(i->src[4])->id);
 226 }
 227
 228 static void
 229 set_dst(struct nv_pc *pc, struct nv_value *value)
 230 {
 231    struct nv_reg *reg = &value->join->reg;
 232
 233    if (reg->id < 0) {
 234       debug_printf("WARNING: unused dst, hope we can bucket it !\n");
 235       pc->emit[0] |= 127 << 2;
 236       pc->emit[1] |= 0x8;
 237       return;
 238    }
 239
 240    if (reg->file == NV_FILE_OUT)
 241       pc->emit[1] |= 0x8;
 242    else
 243    if (reg->file == NV_FILE_ADDR)
 244            assert(0);
 245
 246    pc->emit[0] |= reg->id << 2;
 247 }
 248
 249 static void
 250 set_src_0(struct nv_pc *pc, struct nv_ref *ref)
 251 {
 252    struct nv_reg *reg = SREG(ref);
 253
 254    if (reg->file == NV_FILE_MEM_S)
 255       pc->emit[1] |= 0x00200000;
 256    else
 257    if (reg->file == NV_FILE_MEM_P)
 258       pc->emit[0] |= 0x01800000;
 259    else
 260    if (reg->file != NV_FILE_GPR)
 261       NOUVEAU_ERR("invalid src0 register file: %d\n", reg->file);
 262
 263    assert(reg->id < 128);
 264    pc->emit[0] |= reg->id << 9;
 265 }
 266
 267 static void
 268 set_src_1(struct nv_pc *pc, struct nv_ref *ref)
 269 {
 270    struct nv_reg *reg = SREG(ref);
 271
 272    if (reg->file >= NV_FILE_MEM_C(0) &&
 273        reg->file <= NV_FILE_MEM_C(15)) {
 274       assert(!(pc->emit[1] & 0x01800000));
 275
 276       pc->emit[0] |= 0x00800000;
 277       pc->emit[1] |= (reg->file - NV_FILE_MEM_C(0)) << 22;
 278    } else
 279    if (reg->file != NV_FILE_GPR)
 280       NOUVEAU_ERR("invalid src1 register file: %d\n", reg->file);
 281
 282    assert(reg->id < 128);
 283    pc->emit[0] |= reg->id << 16;
 284 }
 285
 286 static void
 287 set_src_2(struct nv_pc *pc, struct nv_ref *ref)
 288 {
 289    struct nv_reg *reg = SREG(ref);
 290
 291    if (reg->file >= NV_FILE_MEM_C(0) &&
 292        reg->file <= NV_FILE_MEM_C(15)) {
 293       assert(!(pc->emit[1] & 0x01800000));
 294
 295       pc->emit[0] |= 0x01000000;
 296       pc->emit[1] |= (reg->file - NV_FILE_MEM_C(0)) << 22;
 297    } else
 298    if (reg->file != NV_FILE_GPR)
 299       NOUVEAU_ERR("invalid src2 register file: %d\n", reg->file);
 300
 301    assert(reg->id < 128);
 302    pc->emit[1] |= reg->id << 14;
 303 }
 304
 305 /* the default form:
 306  * - long instruction
 307  * - 1 to 3 sources in slots 0, 1, 2
 308  * - address & flags
 309  */
 310 static void
 311 emit_form_MAD(struct nv_pc *pc, struct nv_instruction *i)
 312 {
 313    pc->emit[0] |= 1;
 314
 315    set_pred(pc, i);
 316    set_pred_wr(pc, i);
 317
 318    if (i->def[0])
 319       set_dst(pc, i->def[0]);
 320    else {
 321       pc->emit[0] |= 0x01fc;
 322       pc->emit[1] |= 0x0008;
 323    }
 324
 325    if (i->src[0])
 326       set_src_0(pc, i->src[0]);
 327
 328    if (i->src[1])
 329       set_src_1(pc, i->src[1]);
 330
 331    if (i->src[2])
 332       set_src_2(pc, i->src[2]);
 333
 334    set_addr(pc, i);
 335 }
 336
 337 /* like default form, but 2nd source in slot 2, no 3rd source */
 338 static void
 339 emit_form_ADD(struct nv_pc *pc, struct nv_instruction *i)
 340 {
 341    pc->emit[0] |= 1;
 342
 343    if (i->def[0])
 344       set_dst(pc, i->def[0]);
 345    else {
 346       pc->emit[0] |= 0x01fc;
 347       pc->emit[1] |= 0x0008;
 348    }
 349
 350    set_pred(pc, i);
 351    set_pred_wr(pc, i);
 352
 353    if (i->src[0])
 354       set_src_0(pc, i->src[0]);
 355
 356    if (i->src[1])
 357       set_src_2(pc, i->src[1]);
 358
 359    set_addr(pc, i);
 360 }
 361
 362 /* short mul */
 363 static void
 364 emit_form_MUL(struct nv_pc *pc, struct nv_instruction *i)
 365 {
 366    assert(!i->is_long && !(pc->emit[0] & 1));
 367
 368    assert(i->def[0]);
 369    set_dst(pc, i->def[0]);
 370
 371    if (i->src[0])
 372       set_src_0(pc, i->src[0]);
 373
 374    if (i->src[1])
 375       set_src_1(pc, i->src[1]);
 376 }
 377
 378 /* default immediate form
 379  * - 1 to 3 sources where last is immediate
 380  * - no address or predicate possible
 381  */
 382 static void
 383 emit_form_IMM(struct nv_pc *pc, struct nv_instruction *i, ubyte mod_mask)
 384 {
 385    pc->emit[0] |= 1;
 386
 387    assert(i->def[0]);
 388    assert(i->src[0]);
 389    set_dst(pc, i->def[0]);
 390
 391    assert(!i->src[4] && !i->flags_src && !i->flags_def);
 392
 393    if (i->src[2]) {
 394       set_immd(pc, i->src[2]);
 395       set_src_0(pc, i->src[1]);
 396       set_src_1(pc, i->src[0]);
 397    } else
 398    if (i->src[1]) {
 399       set_immd(pc, i->src[1]);
 400       set_src_0(pc, i->src[0]);
 401    } else
 402       set_immd(pc, i->src[0]);
 403
 404    assert(!mod_mask);
 405 }
 406
 407 static void
 408 set_ld_st_size(struct nv_pc *pc, ubyte type)
 409 {
 410    switch (type) {
 411    case NV_TYPE_F64:
 412       pc->emit[1] |= 0x8000;
 413       break;
 414    case NV_TYPE_F32:
 415    case NV_TYPE_S32:
 416    case NV_TYPE_U32:
 417       pc->emit[1] |= 0xc000;
 418       break;
 419    case NV_TYPE_S16:
 420       pc->emit[1] |= 0x6000;
 421       break;
 422    case NV_TYPE_U16:
 423       pc->emit[1] |= 0x4000;
 424       break;
 425    case NV_TYPE_S8:
 426       pc->emit[1] |= 0x2000;
 427       break;
 428    default:
 429       break;
 430    }
 431 }
 432
 433 static void
 434 emit_ld(struct nv_pc *pc, struct nv_instruction *i)
 435 {
 436    ubyte sf = SFILE(i, 0);
 437
 438    if (sf == NV_FILE_IMM) {
 439       sf = NV_FILE_MEM_C(0);
 440       nv_pc_alloc_immd(pc, i->src[0]);
 441
 442       new_fixup(pc, NV_FIXUP_PARAM_RELOC, SREG(i->src[0])->id, 0xffff, 9);
 443    }
 444
 445    if (sf == NV_FILE_MEM_S ||
 446        sf == NV_FILE_MEM_P) {
 447       pc->emit[0] = 0x10000001;
 448       pc->emit[1] = 0x04200000 | (0x3c << 12);
 449       if (sf == NV_FILE_MEM_P)
 450          pc->emit[0] |= 0x01800000;
 451    } else
 452    if (sf >= NV_FILE_MEM_C(0) &&
 453        sf <= NV_FILE_MEM_C(15)) {
 454       pc->emit[0] = 0x10000001;
 455       pc->emit[1] = 0x24000000;
 456       pc->emit[1] |= (sf - NV_FILE_MEM_C(0)) << 22;
 457    } else
 458    if (sf >= NV_FILE_MEM_G(0) &&
 459        sf <= NV_FILE_MEM_G(15)) {
 460       pc->emit[0] = 0xd0000001 | ((sf - NV_FILE_MEM_G(0)) << 16);
 461       pc->emit[1] = 0xa0000000;
 462
 463       assert(i->src[4] && SREG(i->src[4])->file == NV_FILE_GPR);
 464       SID(pc, i->src[4], 9);
 465    } else
 466    if (sf == NV_FILE_MEM_L) {
 467       pc->emit[0] = 0xd0000001;
 468       pc->emit[1] = 0x40000000;
 469    } else {
 470       NOUVEAU_ERR("invalid ld source file\n");
 471       abort();
 472    }
 473
 474    set_ld_st_size(pc, STYPE(i, 0));
 475
 476    set_dst(pc, i->def[0]);
 477    set_pred_wr(pc, i);
 478
 479    set_pred(pc, i);
 480
 481    if (sf < NV_FILE_MEM_G(0) ||
 482        sf > NV_FILE_MEM_G(15)) {
 483       SID(pc, i->src[0], 9);
 484       set_addr(pc, i);
 485    }
 486 }
 487
 488 static void
 489 emit_st(struct nv_pc *pc, struct nv_instruction *i)
 490 {
 491
 492 }
 493
 494 static int
 495 verify_mov(struct nv_instruction *i)
 496 {
 497    ubyte sf = SFILE(i, 0);
 498    ubyte df = DFILE(i, 0);
 499
 500    if (df == NV_FILE_GPR)
 501       return 0;
 502
 503    if (df != NV_FILE_OUT &&
 504        df != NV_FILE_FLAGS &&
 505        df != NV_FILE_ADDR)
 506       return 1;
 507
 508    if (sf == NV_FILE_FLAGS)
 509       return 2;
 510    if (sf == NV_FILE_ADDR)
 511       return 3;
 512    if (sf == NV_FILE_IMM && df != NV_FILE_OUT)
 513       return 4;
 514
 515    return 0;
 516 }
 517
 518 static void
 519 emit_mov(struct nv_pc *pc, struct nv_instruction *i)
 520 {
 521    assert(!verify_mov(i));
 522
 523    if (SFILE(i, 0) >= NV_FILE_MEM_S)
 524       emit_ld(pc, i);
 525    else
 526    if (SFILE(i, 0) == NV_FILE_FLAGS) {
 527       pc->emit[0] = 0x00000001 | (DREG(i->def[0])->id << 2);
 528       pc->emit[1] = 0x20000780 | (SREG(i->src[0])->id << 12);
 529    } else
 530    if (SFILE(i, 0) == NV_FILE_ADDR) {
 531       pc->emit[0] = 0x00000001 | (DREG(i->def[0])->id << 2);
 532       pc->emit[1] = 0x40000780;
 533       set_a16_bits(pc, SREG(i->src[0])->id);
 534    } else
 535    if (DFILE(i, 0) == NV_FILE_FLAGS) {
 536       pc->emit[0] = 0x000001fd;
 537       pc->emit[1] = 0xa0000788 | (1 << 6);
 538       pc->emit[0] |= SREG(i->src[0])->id << 9;
 539       pc->emit[1] |= DREG(i->def[0])->id << 4;
 540    } else
 541    if (SFILE(i, 0) == NV_FILE_IMM) {
 542       if (i->opcode == NV_OP_LDA) {
 543          emit_ld(pc, i);
 544       } else {
 545          pc->emit[0] = 0x10008001;
 546          pc->emit[1] = 0x00000003;
 547
 548          emit_form_IMM(pc, i, 0);
 549       }
 550    } else {
 551       pc->emit[0] = 0x10000000;
 552       pc->emit[0] |= DREG(i->def[0])->id << 2;
 553       pc->emit[0] |= SREG(i->src[0])->id << 9;
 554
 555       if (!i->is_long) {
 556          pc->emit[0] |= 0x8000;
 557       } else {
 558          pc->emit[0] |= 0x00000001;
 559          pc->emit[1] = 0x0403c000;
 560
 561          set_pred(pc, i);
 562       }
 563    }
 564
 565    if (DFILE(i, 0) == NV_FILE_OUT)
 566       pc->emit[1] |= 0x8;
 567 }
 568
 569 static void
 570 emit_interp(struct nv_pc *pc, struct nv_instruction *i)
 571 {
 572    pc->emit[0] = 0x80000000;
 573
 574    assert(DFILE(i, 0) == NV_FILE_GPR);
 575    assert(SFILE(i, 0) == NV_FILE_MEM_V);
 576
 577    DID(pc, i->def[0], 2);
 578    SID(pc, i->src[0], 16);
 579
 580    if (i->flat)
 581       pc->emit[0] |= 1 << 8;
 582    else
 583    if (i->opcode == NV_OP_PINTERP) {
 584       pc->emit[0] |= 1 << 25;
 585       pc->emit[0] |= SREG(i->src[1])->id << 9;
 586    }
 587
 588    if (i->centroid)
 589       pc->emit[0] |= 1 << 24;
 590
 591    if (i->is_long) {
 592       pc->emit[1] |= 0x0780 |
 593               (pc->emit[0] & (3 << 24)) >> (24 - 16) |
 594               (pc->emit[0] & (1 <<  8)) >> (18 -  8);
 595
 596       pc->emit[0] |= 1;
 597       pc->emit[0] &= ~0x03000100;
 598    }
 599 }
 600
 601 static void
 602 emit_minmax(struct nv_pc *pc, struct nv_instruction *i)
 603 {
 604    pc->emit[0] = 0x30000000;
 605    pc->emit[1] = (i->opcode == NV_OP_MIN) ? (2 << 28) : 0;
 606
 607    switch (DTYPE(i, 0)) {
 608    case NV_TYPE_F32:
 609       pc->emit[0] |= 0x80000000;
 610       pc->emit[1] |= 0x80000000;
 611       break;
 612    case NV_TYPE_S32:
 613       pc->emit[1] |= 0x8c000000;
 614       break;
 615    case NV_TYPE_U32:
 616       pc->emit[1] |= 0x84000000;
 617       break;
 618    }
 619
 620    emit_form_MAD(pc, i);
 621
 622    if (i->src[0]->mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
 623    if (i->src[1]->mod & NV_MOD_ABS) pc->emit[1] |= 0x00080000;
 624 }
 625
 626 static void
 627 emit_add_f32(struct nv_pc *pc, struct nv_instruction *i)
 628 {
 629    pc->emit[0] = 0xb0000000;
 630
 631    if (SFILE(i, 1) == NV_FILE_IMM) {
 632       emit_form_IMM(pc, i, 0);
 633
 634       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 0x8000;
 635       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 636    } else
 637    if (i->is_long) {
 638       emit_form_ADD(pc, i);
 639
 640       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[1] |= 1 << 26;
 641       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[1] |= 1 << 27;
 642    } else {
 643       emit_form_MUL(pc, i);
 644
 645       if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 0x8000;
 646       if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 647    }
 648 }
 649
 650 static void
 651 emit_add_b32(struct nv_pc *pc, struct nv_instruction *i)
 652 {
 653    pc->emit[0] = 0x20008000;
 654
 655    if (SFILE(i, 1) == NV_FILE_IMM) {
 656       emit_form_IMM(pc, i, 0);
 657    } else
 658    if (i->is_long) {
 659       pc->emit[0] = 0x20000000;
 660       pc->emit[1] = 0x04000000;
 661       emit_form_ADD(pc, i);
 662    } else {
 663       emit_form_MUL(pc, i);
 664    }
 665
 666    if (i->src[0]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 28;
 667    if (i->src[1]->mod & NV_MOD_NEG) pc->emit[0] |= 1 << 22;
 668 }
 669
 670 static void
 671 emit_add_a16(struct nv_pc *pc, struct nv_instruction *i)
 672 {
 673    pc->emit[0] = 0xd0000001 | (get_immd_u32(i->src[0]) << 9);
 674    pc->emit[1] = 0x20000000;
 675
 676    pc->emit[0] |= (DREG(i->def[0])->id + 1) << 2;
 677
 678    set_pred(pc, i);
 679
 680    if (i->src[1])
 681       set_a16_bits(pc, SREG(i->src[1])->id);
 682 }
 683
 684 static void
 685 emit_flow(struct nv_pc *pc, struct nv_instruction *i, ubyte flow_op)
 686 {
 687    pc->emit[0] = 0x00000003 | (flow_op << 28);
 688    pc->emit[1] = 0x00000000;
 689
 690    set_pred(pc, i);
 691
 692    if (i->target) {
 693       new_fixup(pc, NV_FIXUP_CFLOW_RELOC, i->target->bin_pos, 0x7ff800, 11);
 694       pc->emit[0] |= (i->target->bin_pos / 4) << 11;
 695    }
 696 }
 697
 698 static INLINE void
 699 emit_add(struct nv_pc *pc, struct nv_instruction *i)
 700 {
 701    if (DFILE(i, 0) == NV_FILE_ADDR)
 702       emit_add_a16(pc, i);
 703    else {
 704       switch (DTYPE(i, 0)) {
 705       case NV_TYPE_F32:
 706          emit_add_f32(pc, i);
 707          break;
 708       case NV_TYPE_U32:
 709       case NV_TYPE_S32:
 710          emit_add_b32(pc, i);
 711          break;
 712       }
 713    }
 714 }
 715
 716 static void
 717 emit_bitop2(struct nv_pc *pc, struct nv_instruction *i)
 718 {
 719    pc->emit[0] = 0xd0000000;
 720
 721    if (SFILE(i, 0) == NV_FILE_IMM) {
 722       emit_form_IMM(pc, i, 0);
 723
 724       if (i->opcode == NV_OP_OR)
 725          pc->emit[0] |= 0x0100;
 726       else
 727       if (i->opcode == NV_OP_XOR)
 728          pc->emit[0] |= 0x8000;
 729    } else {
 730       emit_form_MAD(pc, i);
 731
 732       pc->emit[1] |= 0x04000000;
 733
 734       if (i->opcode == NV_OP_OR)
 735          pc->emit[1] |= 0x4000;
 736       else
 737       if (i->opcode == NV_OP_XOR)
 738          pc->emit[1] |= 0x8000;
 739    }
 740 }
 741
 742 static void
 743 emit_shift(struct nv_pc *pc, struct nv_instruction *i)
 744 {
 745    pc->emit[0] = 0x30000001;
 746    pc->emit[1] = 0xc4000000;
 747
 748    if (i->opcode == NV_OP_SHR)
 749       pc->emit[1] |= 1 << 29;
 750
 751    if (SFILE(i, 1) == NV_FILE_IMM) {
 752       pc->emit[1] |= 1 << 20;
 753       pc->emit[0] |= (get_immd_u32(i->src[1]) & 0x7f) << 16;
 754
 755       set_pred(pc, i);
 756    } else
 757       emit_form_MAD(pc, i);
 758
 759    if (STYPE(i, 0) == NV_TYPE_S32)
 760       pc->emit[1] |= 1 << 27;
 761 }
 762
 763 static void
 764 emit_flop(struct nv_pc *pc, struct nv_instruction *i)
 765 {
 766    struct nv_ref *src0 = i->src[0];
 767
 768    pc->emit[0] = 0x90000000;
 769
 770    assert(SREG(src0)->type == NV_TYPE_F32);
 771    assert(SREG(src0)->file == NV_FILE_GPR);
 772
 773    if (!i->is_long) {
 774       emit_form_MUL(pc, i);
 775       assert(i->opcode == NV_OP_RCP && !src0->mod);
 776       return;
 777    }
 778
 779    pc->emit[1] = (i->opcode - NV_OP_RCP) << 29;
 780
 781    emit_form_MAD(pc, i);
 782
 783    if (src0->mod & NV_MOD_NEG) pc->emit[1] |= 0x04000000;
 784    if (src0->mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
 785 }
 786
 787 static void
 788 emit_mad_f32(struct nv_pc *pc, struct nv_instruction *i)
 789 {
 790    const boolean neg_mul = (i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG;
 791    const boolean neg_add = (i->src[2]->mod & NV_MOD_NEG);
 792
 793    pc->emit[0] = 0xe0000000;
 794
 795    if (!i->is_long) {
 796       emit_form_MUL(pc, i);
 797       assert(!neg_mul && !neg_add);
 798       return;
 799    }
 800
 801    emit_form_MAD(pc, i);
 802
 803    if (neg_mul) pc->emit[1] |= 0x04000000;
 804    if (neg_add) pc->emit[1] |= 0x08000000;
 805
 806    if (i->saturate)
 807       pc->emit[1] |= 0x20000000;
 808 }
 809
 810 static INLINE void
 811 emit_mad(struct nv_pc *pc, struct nv_instruction *i)
 812 {
 813    emit_mad_f32(pc, i);
 814 }
 815
 816 static void
 817 emit_mul_f32(struct nv_pc *pc, struct nv_instruction *i)
 818 {
 819    boolean neg = (i->src[0]->mod ^ i->src[1]->mod) & NV_MOD_NEG;
 820
 821    pc->emit[0] = 0xc0000000;
 822
 823    if (SFILE(i, 1) == NV_FILE_IMM) {
 824       emit_form_IMM(pc, i, 0);
 825
 826       if (neg)
 827          pc->emit[0] |= 0x8000;
 828    } else
 829    if (i->is_long) {
 830       emit_form_MAD(pc, i);
 831
 832       if (neg)
 833          pc->emit[1] |= 0x08 << 24;
 834    } else {
 835       emit_form_MUL(pc, i);
 836
 837       if (neg)
 838          pc->emit[0] |= 0x8000;
 839    }
 840 }
 841
 842 static void
 843 emit_set(struct nv_pc *pc, struct nv_instruction *nvi)
 844 {
 845    assert(nvi->is_long);
 846
 847    pc->emit[0] = 0x30000000;
 848    pc->emit[1] = 0x60000000;
 849
 850    pc->emit[1] |= nvi->set_cond << 14;
 851
 852    switch (STYPE(nvi, 0)) {
 853    case NV_TYPE_U32: pc->emit[1] |= 0x04000000; break;
 854    case NV_TYPE_S32: pc->emit[1] |= 0x0c000000; break;
 855    case NV_TYPE_F32: pc->emit[0] |= 0x80000000; break;
 856    default:
 857       assert(0);
 858       break;
 859    }
 860
 861    emit_form_MAD(pc, nvi);
 862 }
 863
 864 #define CVT_RN    (0x00 << 16)
 865 #define CVT_FLOOR (0x02 << 16)
 866 #define CVT_CEIL  (0x04 << 16)
 867 #define CVT_TRUNC (0x06 << 16)
 868 #define CVT_SAT   (0x08 << 16)
 869 #define CVT_ABS   (0x10 << 16)
 870
 871 #define CVT_X32_X32 0x04004000
 872 #define CVT_X32_S32 0x04014000
 873 #define CVT_F32_F32 ((0xc0 << 24) | CVT_X32_X32)
 874 #define CVT_S32_F32 ((0x88 << 24) | CVT_X32_X32)
 875 #define CVT_U32_F32 ((0x80 << 24) | CVT_X32_X32)
 876 #define CVT_F32_S32 ((0x40 << 24) | CVT_X32_S32)
 877 #define CVT_F32_U32 ((0x40 << 24) | CVT_X32_X32)
 878 #define CVT_S32_S32 ((0x08 << 24) | CVT_X32_S32)
 879 #define CVT_S32_U32 ((0x08 << 24) | CVT_X32_X32)
 880 #define CVT_U32_S32 ((0x00 << 24) | CVT_X32_S32)
 881 #define CVT_U32_U32 ((0x00 << 24) | CVT_X32_X32)
 882
 883 #define CVT_NEG 0x20000000
 884 #define CVT_RI  0x08000000
 885
 886 static void
 887 emit_cvt(struct nv_pc *pc, struct nv_instruction *nvi)
 888 {
 889    ubyte dst_type = nvi->def[0] ? DTYPE(nvi, 0) : STYPE(nvi, 0);
 890
 891    pc->emit[0] = 0xa0000000;
 892
 893    switch (dst_type) {
 894    case NV_TYPE_F32:
 895       switch (STYPE(nvi, 0)) {
 896       case NV_TYPE_F32: pc->emit[1] = CVT_F32_F32; break;
 897       case NV_TYPE_S32: pc->emit[1] = CVT_F32_S32; break;
 898       case NV_TYPE_U32: pc->emit[1] = CVT_F32_U32; break;
 899       }
 900       break;
 901    case NV_TYPE_S32:
 902       switch (STYPE(nvi, 0)) {
 903       case NV_TYPE_F32: pc->emit[1] = CVT_S32_F32; break;
 904       case NV_TYPE_S32: pc->emit[1] = CVT_S32_S32; break;
 905       case NV_TYPE_U32: pc->emit[1] = CVT_S32_U32; break;
 906       }
 907       break;
 908    case NV_TYPE_U32:
 909       switch (STYPE(nvi, 0)) {
 910       case NV_TYPE_F32: pc->emit[1] = CVT_U32_F32; break;
 911       case NV_TYPE_S32: pc->emit[1] = CVT_U32_S32; break;
 912       case NV_TYPE_U32: pc->emit[1] = CVT_U32_U32; break;
 913       }
 914       break;
 915    }
 916    if (pc->emit[1] == CVT_F32_F32 &&
 917        (nvi->opcode == NV_OP_CEIL || nvi->opcode == NV_OP_FLOOR ||
 918         nvi->opcode == NV_OP_TRUNC))
 919        pc->emit[1] |= CVT_RI;
 920
 921    switch (nvi->opcode) {
 922    case NV_OP_CEIL:  pc->emit[1] |= CVT_CEIL; break;
 923    case NV_OP_FLOOR: pc->emit[1] |= CVT_FLOOR; break;
 924    case NV_OP_TRUNC: pc->emit[1] |= CVT_TRUNC; break;
 925
 926    case NV_OP_ABS: pc->emit[1] |= CVT_ABS; break;
 927    case NV_OP_SAT: pc->emit[1] |= CVT_SAT; break;
 928    case NV_OP_NEG: pc->emit[1] |= CVT_NEG; break;
 929    default:
 930       assert(nvi->opcode == NV_OP_CVT);
 931       break;
 932    }
 933    assert(nvi->opcode != NV_OP_ABS || !(nvi->src[0]->mod & NV_MOD_NEG));
 934
 935    if (nvi->src[0]->mod & NV_MOD_NEG) pc->emit[1] ^= CVT_NEG;
 936    if (nvi->src[0]->mod & NV_MOD_ABS) pc->emit[1] |= CVT_ABS;
 937
 938    emit_form_MAD(pc, nvi);
 939 }
 940
 941 static void
 942 emit_tex(struct nv_pc *pc, struct nv_instruction *i)
 943 {
 944    pc->emit[0] = 0xf0000001;
 945    pc->emit[1] = 0x00000000;
 946
 947    DID(pc, i->def[0], 2);
 948
 949    set_pred(pc, i);
 950
 951    pc->emit[0] |= i->tex_t << 9;
 952    pc->emit[0] |= i->tex_s << 17;
 953
 954    pc->emit[0] |= i->tex_argc << 22;
 955
 956    pc->emit[0] |= (i->tex_mask & 0x3) << 25;
 957    pc->emit[1] |= (i->tex_mask & 0xc) << 12;
 958
 959    if (i->tex_live)
 960       pc->emit[1] |= 4;
 961
 962    if (i->tex_cube)
 963       pc->emit[0] |= 0x08000000;
 964
 965    if (i->opcode == NV_OP_TXB)
 966       pc->emit[1] |= 0x20000000;
 967    else
 968    if (i->opcode == NV_OP_TXL)
 969       pc->emit[1] |= 0x40000000;
 970    else
 971       pc->emit[0] -= 1 << 22;
 972 }
 973
 974 static void
 975 emit_cvt2fixed(struct nv_pc *pc, struct nv_instruction *i)
 976 {
 977    ubyte mod = i->src[0]->mod;
 978
 979    pc->emit[0] = 0xb0000000;
 980    pc->emit[1] = 0xc0000000;
 981
 982    if (i->opcode == NV_OP_PREEX2)
 983       pc->emit[1] |= 0x4000;
 984
 985    emit_form_MAD(pc, i);
 986
 987    if (mod & NV_MOD_NEG) pc->emit[1] |= 0x04000000;
 988    if (mod & NV_MOD_ABS) pc->emit[1] |= 0x00100000;
 989 }
 990
 991 static void
 992 emit_ddx(struct nv_pc *pc, struct nv_instruction *i)
 993 {
 994    assert(i->is_long && SFILE(i, 0) == NV_FILE_GPR);
 995
 996    pc->emit[0] = (i->src[0]->mod & NV_MOD_NEG) ? 0xc0240001 : 0xc0140001;
 997    pc->emit[1] = (i->src[0]->mod & NV_MOD_NEG) ? 0x86400000 : 0x89800000;
 998
 999    DID(pc, i->def[0], 2);
1000    SID(pc, i->src[0], 9);
1001    SID(pc, i->src[0], 32 + 14);
1002
1003    set_pred(pc, i);
1004    set_pred_wr(pc, i);
1005 }
1006
1007 static void
1008 emit_ddy(struct nv_pc *pc, struct nv_instruction *i)
1009 {
1010    assert(i->is_long && SFILE(i, 0) == NV_FILE_GPR);
1011
1012    pc->emit[0] = (i->src[0]->mod & NV_MOD_NEG) ? 0xc0250001 : 0xc0150001;
1013    pc->emit[1] = (i->src[0]->mod & NV_MOD_NEG) ? 0x85800000 : 0x8a400000;
1014
1015    DID(pc, i->def[0], 2);
1016    SID(pc, i->src[0], 9);
1017    SID(pc, i->src[0], 32 + 14);
1018
1019    set_pred(pc, i);
1020    set_pred_wr(pc, i);
1021 }
1022
1023 void
1024 nv50_emit_instruction(struct nv_pc *pc, struct nv_instruction *i)
1025 {
1026    // nv_print_instruction(i);
1027
1028    switch (i->opcode) {
1029    case NV_OP_MOV:
1030       if (DFILE(i, 0) == NV_FILE_ADDR)
1031          emit_add_a16(pc, i);
1032       else
1033          emit_mov(pc, i);
1034       break;
1035    case NV_OP_LDA:
1036       emit_mov(pc, i);
1037       break;
1038    case NV_OP_STA:
1039       emit_st(pc, i);
1040       break;
1041    case NV_OP_LINTERP:
1042    case NV_OP_PINTERP:
1043       emit_interp(pc, i);
1044       break;
1045    case NV_OP_ADD:
1046       emit_add(pc, i);
1047       break;
1048    case NV_OP_AND:
1049    case NV_OP_OR:
1050    case NV_OP_XOR:
1051       emit_bitop2(pc, i);
1052       break;
1053    case NV_OP_CVT:
1054    case NV_OP_ABS:
1055    case NV_OP_NEG:
1056    case NV_OP_SAT:
1057    case NV_OP_CEIL:
1058    case NV_OP_FLOOR:
1059    case NV_OP_TRUNC:
1060       emit_cvt(pc, i);
1061       break;
1062    case NV_OP_DFDX:
1063       emit_ddx(pc, i);
1064       break;
1065    case NV_OP_DFDY:
1066       emit_ddy(pc, i);
1067       break;
1068    case NV_OP_RCP:
1069    case NV_OP_RSQ:
1070    case NV_OP_LG2:
1071    case NV_OP_SIN:
1072    case NV_OP_COS:
1073    case NV_OP_EX2:
1074       emit_flop(pc, i);
1075       break;
1076    case NV_OP_PRESIN:
1077    case NV_OP_PREEX2:
1078       emit_cvt2fixed(pc, i);
1079       break;
1080    case NV_OP_MAD:
1081       emit_mad(pc, i);
1082       break;
1083    case NV_OP_MAX:
1084    case NV_OP_MIN:
1085       emit_minmax(pc, i);
1086       break;
1087    case NV_OP_MUL:
1088       emit_mul_f32(pc, i);
1089       break;
1090    case NV_OP_SET:
1091       emit_set(pc, i);
1092       break;
1093    case NV_OP_SHL:
1094    case NV_OP_SHR:
1095       emit_shift(pc, i);
1096       break;
1097    case NV_OP_TEX:
1098    case NV_OP_TXB:
1099    case NV_OP_TXL:
1100       emit_tex(pc, i);
1101       break;
1102    case NV_OP_KIL:
1103       emit_flow(pc, i, 0x0);
1104       break;
1105    case NV_OP_BRA:
1106       emit_flow(pc, i, 0x1);
1107       break;
1108    case NV_OP_CALL:
1109       emit_flow(pc, i, 0x2);
1110       break;
1111    case NV_OP_RET:
1112       emit_flow(pc, i, 0x3);
1113       break;
1114    case NV_OP_BREAKADDR:
1115       emit_flow(pc, i, 0x4);
1116       break;
1117    case NV_OP_BREAK:
1118       emit_flow(pc, i, 0x5);
1119       break;
1120    case NV_OP_JOINAT:
1121       emit_flow(pc, i, 0xa);
1122       break;
1123    case NV_OP_NOP:
1124       pc->emit[0] = 0xf0000001;
1125       pc->emit[1] = 0xe0000000;
1126       break;
1127    case NV_OP_PHI:
1128    case NV_OP_SUB:
1129       NOUVEAU_ERR("operation \"%s\" should have been eliminated\n",
1130                   nv_opcode_name(i->opcode));
1131       break;
1132    default:
1133       NOUVEAU_ERR("unhandled NV_OP: %d\n", i->opcode);
1134       abort();
1135       break;
1136    }
1137
1138    assert((pc->emit[0] & 1) == i->is_long);
1139 }