src/amd/compiler/aco_assembler.cpp

   1 #include <vector>
   2 #include <algorithm>
   3
   4 #include "aco_ir.h"
   5 #include "common/sid.h"
   6 #include "ac_shader_util.h"
   7 #include "util/u_math.h"
   8
   9 namespace aco {
  10
  11 struct asm_context {
  12    Program *program;
  13    enum chip_class chip_class;
  14    std::vector<std::pair<int, SOPP_instruction*>> branches;
  15    std::vector<unsigned> constaddrs;
  16    const int16_t* opcode;
  17    // TODO: keep track of branch instructions referring blocks
  18    // and, when emitting the block, correct the offset in instr
  19    asm_context(Program* program) : program(program), chip_class(program->chip_class) {
  20       if (chip_class <= GFX7)
  21          opcode = &instr_info.opcode_gfx7[0];
  22       else if (chip_class <= GFX9)
  23          opcode = &instr_info.opcode_gfx9[0];
  24       else if (chip_class == GFX10)
  25          opcode = &instr_info.opcode_gfx10[0];
  26    }
  27
  28    int subvector_begin_pos = -1;
  29 };
  30
  31 static uint32_t get_sdwa_sel(unsigned sel, PhysReg reg)
  32 {
  33    if (sel & sdwa_isra) {
  34       unsigned size = sdwa_rasize & sel;
  35       if (size == 1)
  36          return reg.byte();
  37       else /* size == 2 */
  38          return sdwa_isword | (reg.byte() >> 1);
  39    }
  40    return sel & sdwa_asuint;
  41 }
  42
  43 void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
  44 {
  45    uint32_t instr_offset = out.size() * 4u;
  46
  47    /* lower remaining pseudo-instructions */
  48    if (instr->opcode == aco_opcode::p_constaddr) {
  49       unsigned dest = instr->definitions[0].physReg();
  50       unsigned offset = instr->operands[0].constantValue();
  51
  52       /* s_getpc_b64 dest[0:1] */
  53       uint32_t encoding = (0b101111101 << 23);
  54       uint32_t opcode = ctx.opcode[(int)aco_opcode::s_getpc_b64];
  55       if (opcode >= 55 && ctx.chip_class <= GFX9) {
  56          assert(ctx.chip_class == GFX9 && opcode < 60);
  57          opcode = opcode - 4;
  58       }
  59       encoding |= dest << 16;
  60       encoding |= opcode << 8;
  61       out.push_back(encoding);
  62
  63       /* s_add_u32 dest[0], dest[0], ... */
  64       encoding = (0b10 << 30);
  65       encoding |= ctx.opcode[(int)aco_opcode::s_add_u32] << 23;
  66       encoding |= dest << 16;
  67       encoding |= dest;
  68       encoding |= 255 << 8;
  69       out.push_back(encoding);
  70       ctx.constaddrs.push_back(out.size());
  71       out.push_back(-(instr_offset + 4) + offset);
  72
  73       /* s_addc_u32 dest[1], dest[1], 0 */
  74       encoding = (0b10 << 30);
  75       encoding |= ctx.opcode[(int)aco_opcode::s_addc_u32] << 23;
  76       encoding |= (dest + 1) << 16;
  77       encoding |= dest + 1;
  78       encoding |= 128 << 8;
  79       out.push_back(encoding);
  80       return;
  81    }
  82
  83    uint32_t opcode = ctx.opcode[(int)instr->opcode];
  84    if (opcode == (uint32_t)-1) {
  85       fprintf(stderr, "Unsupported opcode: ");
  86       aco_print_instr(instr, stderr);
  87       abort();
  88    }
  89
  90    switch (instr->format) {
  91    case Format::SOP2: {
  92       uint32_t encoding = (0b10 << 30);
  93       encoding |= opcode << 23;
  94       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
  95       encoding |= instr->operands.size() >= 2 ? instr->operands[1].physReg() << 8 : 0;
  96       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
  97       out.push_back(encoding);
  98       break;
  99    }
 100    case Format::SOPK: {
 101       SOPK_instruction *sopk = static_cast<SOPK_instruction*>(instr);
 102
 103       if (instr->opcode == aco_opcode::s_subvector_loop_begin) {
 104          assert(ctx.chip_class >= GFX10);
 105          assert(ctx.subvector_begin_pos == -1);
 106          ctx.subvector_begin_pos = out.size();
 107       } else if (instr->opcode == aco_opcode::s_subvector_loop_end) {
 108          assert(ctx.chip_class >= GFX10);
 109          assert(ctx.subvector_begin_pos != -1);
 110          /* Adjust s_subvector_loop_begin instruction to the address after the end  */
 111          out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos);
 112          /* Adjust s_subvector_loop_end instruction to the address after the beginning  */
 113          sopk->imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size());
 114          ctx.subvector_begin_pos = -1;
 115       }
 116
 117       uint32_t encoding = (0b1011 << 28);
 118       encoding |= opcode << 23;
 119       encoding |=
 120          !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
 121          instr->definitions[0].physReg() << 16 :
 122          !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
 123          instr->operands[0].physReg() << 16 : 0;
 124       encoding |= sopk->imm;
 125       out.push_back(encoding);
 126       break;
 127    }
 128    case Format::SOP1: {
 129       uint32_t encoding = (0b101111101 << 23);
 130       if (opcode >= 55 && ctx.chip_class <= GFX9) {
 131          assert(ctx.chip_class == GFX9 && opcode < 60);
 132          opcode = opcode - 4;
 133       }
 134       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
 135       encoding |= opcode << 8;
 136       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 137       out.push_back(encoding);
 138       break;
 139    }
 140    case Format::SOPC: {
 141       uint32_t encoding = (0b101111110 << 23);
 142       encoding |= opcode << 16;
 143       encoding |= instr->operands.size() == 2 ? instr->operands[1].physReg() << 8 : 0;
 144       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 145       out.push_back(encoding);
 146       break;
 147    }
 148    case Format::SOPP: {
 149       SOPP_instruction* sopp = static_cast<SOPP_instruction*>(instr);
 150       uint32_t encoding = (0b101111111 << 23);
 151       encoding |= opcode << 16;
 152       encoding |= (uint16_t) sopp->imm;
 153       if (sopp->block != -1)
 154          ctx.branches.emplace_back(out.size(), sopp);
 155       out.push_back(encoding);
 156       break;
 157    }
 158    case Format::SMEM: {
 159       SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
 160       bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
 161       bool is_load = !instr->definitions.empty();
 162       uint32_t encoding = 0;
 163
 164       if (ctx.chip_class <= GFX7) {
 165          encoding = (0b11000 << 27);
 166          encoding |= opcode << 22;
 167          encoding |= instr->definitions.size() ? instr->definitions[0].physReg() << 15 : 0;
 168          encoding |= instr->operands.size() ? (instr->operands[0].physReg() >> 1) << 9 : 0;
 169          if (instr->operands.size() >= 2) {
 170             if (!instr->operands[1].isConstant() || instr->operands[1].constantValue() >= 1024) {
 171                encoding |= instr->operands[1].physReg().reg();
 172             } else {
 173                encoding |= instr->operands[1].constantValue() >> 2;
 174                encoding |= 1 << 8;
 175             }
 176          }
 177          out.push_back(encoding);
 178          /* SMRD instructions can take a literal on GFX6 & GFX7 */
 179          if (instr->operands.size() >= 2 && instr->operands[1].isConstant() && instr->operands[1].constantValue() >= 1024)
 180             out.push_back(instr->operands[1].constantValue() >> 2);
 181          return;
 182       }
 183
 184       if (ctx.chip_class <= GFX9) {
 185          encoding = (0b110000 << 26);
 186          assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 187          encoding |= smem->nv ? 1 << 15 : 0;
 188       } else {
 189          encoding = (0b111101 << 26);
 190          assert(!smem->nv); /* Non-volatile is not supported on GFX10 */
 191          encoding |= smem->dlc ? 1 << 14 : 0;
 192       }
 193
 194       encoding |= opcode << 18;
 195       encoding |= smem->glc ? 1 << 16 : 0;
 196
 197       if (ctx.chip_class <= GFX9) {
 198          if (instr->operands.size() >= 2)
 199             encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
 200       }
 201       if (ctx.chip_class == GFX9) {
 202          encoding |= soe ? 1 << 14 : 0;
 203       }
 204
 205       if (is_load || instr->operands.size() >= 3) { /* SDATA */
 206          encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
 207       }
 208       if (instr->operands.size() >= 1) { /* SBASE */
 209          encoding |= instr->operands[0].physReg() >> 1;
 210       }
 211
 212       out.push_back(encoding);
 213       encoding = 0;
 214
 215       int32_t offset = 0;
 216       uint32_t soffset = ctx.chip_class >= GFX10
 217                          ? sgpr_null /* On GFX10 this is disabled by specifying SGPR_NULL */
 218                          : 0;        /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
 219       if (instr->operands.size() >= 2) {
 220          const Operand &op_off1 = instr->operands[1];
 221          if (ctx.chip_class <= GFX9) {
 222             offset = op_off1.isConstant() ? op_off1.constantValue() : op_off1.physReg();
 223          } else {
 224             /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
 225             if (op_off1.isConstant()) {
 226                offset = op_off1.constantValue();
 227             } else {
 228                soffset = op_off1.physReg();
 229                assert(!soe); /* There is no place to put the other SGPR offset, if any */
 230             }
 231          }
 232
 233          if (soe) {
 234             const Operand &op_off2 = instr->operands.back();
 235             assert(ctx.chip_class >= GFX9); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
 236             assert(!op_off2.isConstant());
 237             soffset = op_off2.physReg();
 238          }
 239       }
 240       encoding |= offset;
 241       encoding |= soffset << 25;
 242
 243       out.push_back(encoding);
 244       return;
 245    }
 246    case Format::VOP2: {
 247       uint32_t encoding = 0;
 248       encoding |= opcode << 25;
 249       encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
 250       encoding |= (0xFF & instr->operands[1].physReg()) << 9;
 251       encoding |= instr->operands[0].physReg();
 252       out.push_back(encoding);
 253       break;
 254    }
 255    case Format::VOP1: {
 256       uint32_t encoding = (0b0111111 << 25);
 257       if (!instr->definitions.empty())
 258          encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
 259       encoding |= opcode << 9;
 260       if (!instr->operands.empty())
 261          encoding |= instr->operands[0].physReg();
 262       out.push_back(encoding);
 263       break;
 264    }
 265    case Format::VOPC: {
 266       uint32_t encoding = (0b0111110 << 25);
 267       encoding |= opcode << 17;
 268       encoding |= (0xFF & instr->operands[1].physReg()) << 9;
 269       encoding |= instr->operands[0].physReg();
 270       out.push_back(encoding);
 271       break;
 272    }
 273    case Format::VINTRP: {
 274       Interp_instruction* interp = static_cast<Interp_instruction*>(instr);
 275       uint32_t encoding = 0;
 276
 277       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 278          encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
 279       } else {
 280          encoding = (0b110010 << 26);
 281       }
 282
 283       assert(encoding);
 284       encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
 285       encoding |= opcode << 16;
 286       encoding |= interp->attribute << 10;
 287       encoding |= interp->component << 8;
 288       if (instr->opcode == aco_opcode::v_interp_mov_f32)
 289          encoding |= (0x3 & instr->operands[0].constantValue());
 290       else
 291          encoding |= (0xFF & instr->operands[0].physReg());
 292       out.push_back(encoding);
 293       break;
 294    }
 295    case Format::DS: {
 296       DS_instruction* ds = static_cast<DS_instruction*>(instr);
 297       uint32_t encoding = (0b110110 << 26);
 298       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 299          encoding |= opcode << 17;
 300          encoding |= (ds->gds ? 1 : 0) << 16;
 301       } else {
 302          encoding |= opcode << 18;
 303          encoding |= (ds->gds ? 1 : 0) << 17;
 304       }
 305       encoding |= ((0xFF & ds->offset1) << 8);
 306       encoding |= (0xFFFF & ds->offset0);
 307       out.push_back(encoding);
 308       encoding = 0;
 309       unsigned reg = !instr->definitions.empty() ? instr->definitions[0].physReg() : 0;
 310       encoding |= (0xFF & reg) << 24;
 311       reg = instr->operands.size() >= 3 && !(instr->operands[2].physReg() == m0)  ? instr->operands[2].physReg() : 0;
 312       encoding |= (0xFF & reg) << 16;
 313       reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
 314       encoding |= (0xFF & reg) << 8;
 315       encoding |= (0xFF & instr->operands[0].physReg());
 316       out.push_back(encoding);
 317       break;
 318    }
 319    case Format::MUBUF: {
 320       MUBUF_instruction* mubuf = static_cast<MUBUF_instruction*>(instr);
 321       uint32_t encoding = (0b111000 << 26);
 322       encoding |= opcode << 18;
 323       encoding |= (mubuf->lds ? 1 : 0) << 16;
 324       encoding |= (mubuf->glc ? 1 : 0) << 14;
 325       encoding |= (mubuf->idxen ? 1 : 0) << 13;
 326       assert(!mubuf->addr64 || ctx.chip_class <= GFX7);
 327       if (ctx.chip_class == GFX6 || ctx.chip_class == GFX7)
 328          encoding |= (mubuf->addr64 ? 1 : 0) << 15;
 329       encoding |= (mubuf->offen ? 1 : 0) << 12;
 330       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 331          assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 332          encoding |= (mubuf->slc ? 1 : 0) << 17;
 333       } else if (ctx.chip_class >= GFX10) {
 334          encoding |= (mubuf->dlc ? 1 : 0) << 15;
 335       }
 336       encoding |= 0x0FFF & mubuf->offset;
 337       out.push_back(encoding);
 338       encoding = 0;
 339       if (ctx.chip_class <= GFX7 || ctx.chip_class >= GFX10) {
 340          encoding |= (mubuf->slc ? 1 : 0) << 22;
 341       }
 342       encoding |= instr->operands[2].physReg() << 24;
 343       encoding |= (mubuf->tfe ? 1 : 0) << 23;
 344       encoding |= (instr->operands[0].physReg() >> 2) << 16;
 345       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
 346       encoding |= (0xFF & reg) << 8;
 347       encoding |= (0xFF & instr->operands[1].physReg());
 348       out.push_back(encoding);
 349       break;
 350    }
 351    case Format::MTBUF: {
 352       MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
 353
 354       uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
 355       uint32_t encoding = (0b111010 << 26);
 356       assert(img_format <= 0x7F);
 357       assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
 358       encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
 359       encoding |= (mtbuf->glc ? 1 : 0) << 14;
 360       encoding |= (mtbuf->idxen ? 1 : 0) << 13;
 361       encoding |= (mtbuf->offen ? 1 : 0) << 12;
 362       encoding |= 0x0FFF & mtbuf->offset;
 363       encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
 364
 365       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 366          encoding |= opcode << 15;
 367       } else {
 368          encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
 369       }
 370
 371       out.push_back(encoding);
 372       encoding = 0;
 373
 374       encoding |= instr->operands[2].physReg() << 24;
 375       encoding |= (mtbuf->tfe ? 1 : 0) << 23;
 376       encoding |= (mtbuf->slc ? 1 : 0) << 22;
 377       encoding |= (instr->operands[0].physReg() >> 2) << 16;
 378       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
 379       encoding |= (0xFF & reg) << 8;
 380       encoding |= (0xFF & instr->operands[1].physReg());
 381
 382       if (ctx.chip_class >= GFX10) {
 383          encoding |= (((opcode & 0x08) >> 4) << 21); /* MSB of 4-bit OPCODE */
 384       }
 385
 386       out.push_back(encoding);
 387       break;
 388    }
 389    case Format::MIMG: {
 390       MIMG_instruction* mimg = static_cast<MIMG_instruction*>(instr);
 391       uint32_t encoding = (0b111100 << 26);
 392       encoding |= mimg->slc ? 1 << 25 : 0;
 393       encoding |= opcode << 18;
 394       encoding |= mimg->lwe ? 1 << 17 : 0;
 395       encoding |= mimg->tfe ? 1 << 16 : 0;
 396       encoding |= mimg->glc ? 1 << 13 : 0;
 397       encoding |= mimg->unrm ? 1 << 12 : 0;
 398       if (ctx.chip_class <= GFX9) {
 399          assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 400          assert(!mimg->r128);
 401          encoding |= mimg->a16 ? 1 << 15 : 0;
 402          encoding |= mimg->da ? 1 << 14 : 0;
 403       } else {
 404          encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
 405          encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
 406          encoding |= mimg->dlc ? 1 << 7 : 0;
 407       }
 408       encoding |= (0xF & mimg->dmask) << 8;
 409       out.push_back(encoding);
 410       encoding = (0xFF & instr->operands[2].physReg()); /* VADDR */
 411       if (!instr->definitions.empty()) {
 412          encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
 413       } else if (instr->operands[1].regClass().type() == RegType::vgpr) {
 414          encoding |= (0xFF & instr->operands[1].physReg()) << 8; /* VDATA */
 415       }
 416       encoding |= (0x1F & (instr->operands[0].physReg() >> 2)) << 16; /* T# (resource) */
 417       if (instr->operands[1].regClass().type() == RegType::sgpr)
 418          encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 21; /* sampler */
 419
 420       assert(!mimg->d16 || ctx.chip_class >= GFX9);
 421       encoding |= mimg->d16 ? 1 << 15 : 0;
 422       if (ctx.chip_class >= GFX10) {
 423          encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
 424       }
 425
 426       out.push_back(encoding);
 427       break;
 428    }
 429    case Format::FLAT:
 430    case Format::SCRATCH:
 431    case Format::GLOBAL: {
 432       FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
 433       uint32_t encoding = (0b110111 << 26);
 434       encoding |= opcode << 18;
 435       if (ctx.chip_class <= GFX9) {
 436          assert(flat->offset <= 0x1fff);
 437          encoding |= flat->offset & 0x1fff;
 438       } else if (instr->format == Format::FLAT) {
 439          /* GFX10 has a 12-bit immediate OFFSET field,
 440           * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
 441           */
 442          assert(flat->offset == 0);
 443       } else {
 444          assert(flat->offset <= 0xfff);
 445          encoding |= flat->offset & 0xfff;
 446       }
 447       if (instr->format == Format::SCRATCH)
 448          encoding |= 1 << 14;
 449       else if (instr->format == Format::GLOBAL)
 450          encoding |= 2 << 14;
 451       encoding |= flat->lds ? 1 << 13 : 0;
 452       encoding |= flat->glc ? 1 << 16 : 0;
 453       encoding |= flat->slc ? 1 << 17 : 0;
 454       if (ctx.chip_class >= GFX10) {
 455          assert(!flat->nv);
 456          encoding |= flat->dlc ? 1 << 12 : 0;
 457       } else {
 458          assert(!flat->dlc);
 459       }
 460       out.push_back(encoding);
 461       encoding = (0xFF & instr->operands[0].physReg());
 462       if (!instr->definitions.empty())
 463          encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
 464       if (instr->operands.size() >= 3)
 465          encoding |= (0xFF & instr->operands[2].physReg()) << 8;
 466       if (!instr->operands[1].isUndefined()) {
 467          assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
 468          assert(instr->format != Format::FLAT);
 469          encoding |= instr->operands[1].physReg() << 16;
 470       } else if (instr->format != Format::FLAT || ctx.chip_class >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */
 471          if (ctx.chip_class <= GFX9)
 472             encoding |= 0x7F << 16;
 473          else
 474             encoding |= sgpr_null << 16;
 475       }
 476       encoding |= flat->nv ? 1 << 23 : 0;
 477       out.push_back(encoding);
 478       break;
 479    }
 480    case Format::EXP: {
 481       Export_instruction* exp = static_cast<Export_instruction*>(instr);
 482       uint32_t encoding;
 483       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 484          encoding = (0b110001 << 26);
 485       } else {
 486          encoding = (0b111110 << 26);
 487       }
 488
 489       encoding |= exp->valid_mask ? 0b1 << 12 : 0;
 490       encoding |= exp->done ? 0b1 << 11 : 0;
 491       encoding |= exp->compressed ? 0b1 << 10 : 0;
 492       encoding |= exp->dest << 4;
 493       encoding |= exp->enabled_mask;
 494       out.push_back(encoding);
 495       encoding = 0xFF & exp->operands[0].physReg();
 496       encoding |= (0xFF & exp->operands[1].physReg()) << 8;
 497       encoding |= (0xFF & exp->operands[2].physReg()) << 16;
 498       encoding |= (0xFF & exp->operands[3].physReg()) << 24;
 499       out.push_back(encoding);
 500       break;
 501    }
 502    case Format::PSEUDO:
 503    case Format::PSEUDO_BARRIER:
 504       unreachable("Pseudo instructions should be lowered before assembly.");
 505    default:
 506       if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
 507          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
 508
 509          if ((uint16_t) instr->format & (uint16_t) Format::VOP2) {
 510             opcode = opcode + 0x100;
 511          } else if ((uint16_t) instr->format & (uint16_t) Format::VOP1) {
 512             if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9)
 513                opcode = opcode + 0x140;
 514             else
 515                opcode = opcode + 0x180;
 516          } else if ((uint16_t) instr->format & (uint16_t) Format::VOPC) {
 517             opcode = opcode + 0x0;
 518          } else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP) {
 519             opcode = opcode + 0x270;
 520          }
 521
 522          uint32_t encoding;
 523          if (ctx.chip_class <= GFX9) {
 524             encoding = (0b110100 << 26);
 525          } else if (ctx.chip_class == GFX10) {
 526             encoding = (0b110101 << 26);
 527          } else {
 528             unreachable("Unknown chip_class.");
 529          }
 530
 531          if (ctx.chip_class <= GFX7) {
 532             encoding |= opcode << 17;
 533             encoding |= (vop3->clamp ? 1 : 0) << 11;
 534          } else {
 535             encoding |= opcode << 16;
 536             encoding |= (vop3->clamp ? 1 : 0) << 15;
 537          }
 538          encoding |= vop3->opsel << 11;
 539          for (unsigned i = 0; i < 3; i++)
 540             encoding |= vop3->abs[i] << (8+i);
 541          if (instr->definitions.size() == 2)
 542             encoding |= instr->definitions[1].physReg() << 8;
 543          encoding |= (0xFF & instr->definitions[0].physReg());
 544          out.push_back(encoding);
 545          encoding = 0;
 546          if (instr->opcode == aco_opcode::v_interp_mov_f32) {
 547             encoding = 0x3 & instr->operands[0].constantValue();
 548          } else {
 549             for (unsigned i = 0; i < instr->operands.size(); i++)
 550                encoding |= instr->operands[i].physReg() << (i * 9);
 551          }
 552          encoding |= vop3->omod << 27;
 553          for (unsigned i = 0; i < 3; i++)
 554             encoding |= vop3->neg[i] << (29+i);
 555          out.push_back(encoding);
 556
 557       } else if (instr->format == Format::VOP3P) {
 558          VOP3P_instruction* vop3 = static_cast<VOP3P_instruction*>(instr);
 559
 560          uint32_t encoding;
 561          if (ctx.chip_class == GFX9) {
 562             encoding = (0b110100111 << 23);
 563          } else if (ctx.chip_class == GFX10) {
 564             encoding = (0b110011 << 26);
 565          } else {
 566             unreachable("Unknown chip_class.");
 567          }
 568
 569          encoding |= opcode << 16;
 570          encoding |= (vop3->clamp ? 1 : 0) << 15;
 571          encoding |= vop3->opsel_lo << 11;
 572          encoding |= (vop3->opsel_hi & 0x4) ? 1 : 0 << 14;
 573          for (unsigned i = 0; i < 3; i++)
 574             encoding |= vop3->neg_hi[i] << (8+i);
 575          encoding |= (0xFF & instr->definitions[0].physReg());
 576          out.push_back(encoding);
 577          encoding = 0;
 578          for (unsigned i = 0; i < instr->operands.size(); i++)
 579             encoding |= instr->operands[i].physReg() << (i * 9);
 580          encoding |= vop3->opsel_hi & 0x3 << 27;
 581          for (unsigned i = 0; i < 3; i++)
 582             encoding |= vop3->neg_lo[i] << (29+i);
 583          out.push_back(encoding);
 584
 585       } else if (instr->isDPP()){
 586          assert(ctx.chip_class >= GFX8);
 587          /* first emit the instruction without the DPP operand */
 588          Operand dpp_op = instr->operands[0];
 589          instr->operands[0] = Operand(PhysReg{250}, v1);
 590          instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::DPP);
 591          emit_instruction(ctx, out, instr);
 592          DPP_instruction* dpp = static_cast<DPP_instruction*>(instr);
 593          uint32_t encoding = (0xF & dpp->row_mask) << 28;
 594          encoding |= (0xF & dpp->bank_mask) << 24;
 595          encoding |= dpp->abs[1] << 23;
 596          encoding |= dpp->neg[1] << 22;
 597          encoding |= dpp->abs[0] << 21;
 598          encoding |= dpp->neg[0] << 20;
 599          encoding |= dpp->bound_ctrl << 19;
 600          encoding |= dpp->dpp_ctrl << 8;
 601          encoding |= (0xFF) & dpp_op.physReg();
 602          out.push_back(encoding);
 603          return;
 604       } else if (instr->isSDWA()) {
 605          /* first emit the instruction without the SDWA operand */
 606          Operand sdwa_op = instr->operands[0];
 607          instr->operands[0] = Operand(PhysReg{249}, v1);
 608          instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::SDWA);
 609          emit_instruction(ctx, out, instr);
 610
 611          SDWA_instruction* sdwa = static_cast<SDWA_instruction*>(instr);
 612          uint32_t encoding = 0;
 613
 614          if ((uint16_t)instr->format & (uint16_t)Format::VOPC) {
 615             if (instr->definitions[0].physReg() != vcc) {
 616                encoding |= instr->definitions[0].physReg() << 8;
 617                encoding |= 1 << 15;
 618             }
 619             encoding |= (sdwa->clamp ? 1 : 0) << 13;
 620          } else {
 621             encoding |= get_sdwa_sel(sdwa->dst_sel, instr->definitions[0].physReg()) << 8;
 622             uint32_t dst_u = sdwa->dst_sel & sdwa_sext ? 1 : 0;
 623             if (sdwa->dst_preserve || (sdwa->dst_sel & sdwa_isra))
 624                dst_u = 2;
 625             encoding |= dst_u << 11;
 626             encoding |= (sdwa->clamp ? 1 : 0) << 13;
 627             encoding |= sdwa->omod << 14;
 628          }
 629
 630          encoding |= get_sdwa_sel(sdwa->sel[0], sdwa_op.physReg()) << 16;
 631          encoding |= sdwa->sel[0] & sdwa_sext ? 1 << 19 : 0;
 632          encoding |= sdwa->abs[0] << 21;
 633          encoding |= sdwa->neg[0] << 20;
 634
 635          if (instr->operands.size() >= 2) {
 636             encoding |= get_sdwa_sel(sdwa->sel[1], instr->operands[1].physReg()) << 24;
 637             encoding |= sdwa->sel[1] & sdwa_sext ? 1 << 27 : 0;
 638             encoding |= sdwa->abs[1] << 29;
 639             encoding |= sdwa->neg[1] << 28;
 640          }
 641
 642          encoding |= 0xFF & sdwa_op.physReg();
 643          encoding |= (sdwa_op.physReg() < 256) << 23;
 644          if (instr->operands.size() >= 2)
 645             encoding |= (instr->operands[1].physReg() < 256) << 31;
 646          out.push_back(encoding);
 647       } else {
 648          unreachable("unimplemented instruction format");
 649       }
 650       break;
 651    }
 652
 653    /* append literal dword */
 654    for (const Operand& op : instr->operands) {
 655       if (op.isLiteral()) {
 656          out.push_back(op.constantValue());
 657          break;
 658       }
 659    }
 660 }
 661
 662 void emit_block(asm_context& ctx, std::vector<uint32_t>& out, Block& block)
 663 {
 664    for (aco_ptr<Instruction>& instr : block.instructions) {
 665 #if 0
 666       int start_idx = out.size();
 667       std::cerr << "Encoding:\t" << std::endl;
 668       aco_print_instr(&*instr, stderr);
 669       std::cerr << std::endl;
 670 #endif
 671       emit_instruction(ctx, out, instr.get());
 672 #if 0
 673       for (int i = start_idx; i < out.size(); i++)
 674          std::cerr << "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex << out[i] << std::endl;
 675 #endif
 676    }
 677 }
 678
 679 void fix_exports(asm_context& ctx, std::vector<uint32_t>& out, Program* program)
 680 {
 681    bool exported = false;
 682    for (Block& block : program->blocks) {
 683       if (!(block.kind & block_kind_export_end))
 684          continue;
 685       std::vector<aco_ptr<Instruction>>::reverse_iterator it = block.instructions.rbegin();
 686       while ( it != block.instructions.rend())
 687       {
 688          if ((*it)->format == Format::EXP) {
 689             Export_instruction* exp = static_cast<Export_instruction*>((*it).get());
 690             if (program->stage & (hw_vs | hw_ngg_gs)) {
 691                if (exp->dest >= V_008DFC_SQ_EXP_POS && exp->dest <= (V_008DFC_SQ_EXP_POS + 3)) {
 692                   exp->done = true;
 693                   exported = true;
 694                   break;
 695                }
 696             } else {
 697                exp->done = true;
 698                exp->valid_mask = true;
 699                exported = true;
 700                break;
 701             }
 702          } else if ((*it)->definitions.size() && (*it)->definitions[0].physReg() == exec)
 703             break;
 704          ++it;
 705       }
 706    }
 707
 708    if (!exported) {
 709       /* Abort in order to avoid a GPU hang. */
 710       fprintf(stderr, "Missing export in %s shader:\n", (program->stage & hw_vs) ? "vertex" : "fragment");
 711       aco_print_program(program, stderr);
 712       abort();
 713    }
 714 }
 715
 716 static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
 717 {
 718    /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
 719    bool gfx10_3f_bug = false;
 720
 721    do {
 722       auto buggy_branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [&ctx](const auto &branch) -> bool {
 723          return ((int)ctx.program->blocks[branch.second->block].offset - branch.first - 1) == 0x3f;
 724       });
 725
 726       gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
 727
 728       if (gfx10_3f_bug) {
 729          /* Insert an s_nop after the branch */
 730          constexpr uint32_t s_nop_0 = 0xbf800000u;
 731          int s_nop_pos = buggy_branch_it->first + 1;
 732          auto out_pos = std::next(out.begin(), s_nop_pos);
 733          out.insert(out_pos, s_nop_0);
 734
 735          /* Update the offset of each affected block */
 736          for (Block& block : ctx.program->blocks) {
 737             if (block.offset > (unsigned)buggy_branch_it->first)
 738                block.offset++;
 739          }
 740
 741          /* Update the branches following the current one */
 742          for (auto branch_it = std::next(buggy_branch_it); branch_it != ctx.branches.end(); ++branch_it)
 743             branch_it->first++;
 744
 745          /* Find first constant address after the inserted instruction */
 746          auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [s_nop_pos](const int &caddr_pos) -> bool {
 747             return caddr_pos >= s_nop_pos;
 748          });
 749
 750          /* Update the locations of constant addresses */
 751          for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
 752             (*caddr_it)++;
 753
 754       }
 755    } while (gfx10_3f_bug);
 756 }
 757
 758 void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
 759 {
 760    if (ctx.chip_class >= GFX10)
 761       fix_branches_gfx10(ctx, out);
 762
 763    for (std::pair<int, SOPP_instruction*> &branch : ctx.branches) {
 764       int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
 765       out[branch.first] |= (uint16_t) offset;
 766    }
 767 }
 768
 769 void fix_constaddrs(asm_context& ctx, std::vector<uint32_t>& out)
 770 {
 771    for (unsigned addr : ctx.constaddrs)
 772       out[addr] += out.size() * 4u;
 773 }
 774
 775 unsigned emit_program(Program* program,
 776                       std::vector<uint32_t>& code)
 777 {
 778    asm_context ctx(program);
 779
 780    if (program->stage & (hw_vs | hw_fs | hw_ngg_gs))
 781       fix_exports(ctx, code, program);
 782
 783    for (Block& block : program->blocks) {
 784       block.offset = code.size();
 785       emit_block(ctx, code, block);
 786    }
 787
 788    fix_branches(ctx, code);
 789
 790    unsigned exec_size = code.size() * sizeof(uint32_t);
 791
 792    if (program->chip_class >= GFX10) {
 793       /* Pad output with s_code_end so instruction prefetching doesn't cause
 794        * page faults */
 795       unsigned final_size = align(code.size() + 3 * 16, 16);
 796       while (code.size() < final_size)
 797          code.push_back(0xbf9f0000u);
 798    }
 799
 800    fix_constaddrs(ctx, code);
 801
 802    while (program->constant_data.size() % 4u)
 803       program->constant_data.push_back(0);
 804    /* Copy constant data */
 805    code.insert(code.end(), (uint32_t*)program->constant_data.data(),
 806                (uint32_t*)(program->constant_data.data() + program->constant_data.size()));
 807
 808    return exec_size;
 809 }
 810
 811 }