src/amd/compiler/aco_assembler.cpp

   1 #include <vector>
   2 #include <algorithm>
   3
   4 #include "aco_ir.h"
   5 #include "common/sid.h"
   6 #include "ac_shader_util.h"
   7 #include "util/u_math.h"
   8
   9 namespace aco {
  10
  11 struct asm_context {
  12    Program *program;
  13    enum chip_class chip_class;
  14    std::vector<std::pair<int, SOPP_instruction*>> branches;
  15    std::vector<unsigned> constaddrs;
  16    const int16_t* opcode;
  17    // TODO: keep track of branch instructions referring blocks
  18    // and, when emitting the block, correct the offset in instr
  19    asm_context(Program* program) : program(program), chip_class(program->chip_class) {
  20       if (chip_class <= GFX9)
  21          opcode = &instr_info.opcode_gfx9[0];
  22       else if (chip_class == GFX10)
  23          opcode = &instr_info.opcode_gfx10[0];
  24    }
  25
  26    int subvector_begin_pos = -1;
  27 };
  28
  29 void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
  30 {
  31    uint32_t instr_offset = out.size() * 4u;
  32
  33    /* lower remaining pseudo-instructions */
  34    if (instr->opcode == aco_opcode::p_constaddr) {
  35       unsigned dest = instr->definitions[0].physReg();
  36       unsigned offset = instr->operands[0].constantValue();
  37
  38       /* s_getpc_b64 dest[0:1] */
  39       uint32_t encoding = (0b101111101 << 23);
  40       uint32_t opcode = ctx.opcode[(int)aco_opcode::s_getpc_b64];
  41       if (opcode >= 55 && ctx.chip_class <= GFX9) {
  42          assert(ctx.chip_class == GFX9 && opcode < 60);
  43          opcode = opcode - 4;
  44       }
  45       encoding |= dest << 16;
  46       encoding |= opcode << 8;
  47       out.push_back(encoding);
  48
  49       /* s_add_u32 dest[0], dest[0], ... */
  50       encoding = (0b10 << 30);
  51       encoding |= ctx.opcode[(int)aco_opcode::s_add_u32] << 23;
  52       encoding |= dest << 16;
  53       encoding |= dest;
  54       encoding |= 255 << 8;
  55       out.push_back(encoding);
  56       ctx.constaddrs.push_back(out.size());
  57       out.push_back(-(instr_offset + 4) + offset);
  58
  59       /* s_addc_u32 dest[1], dest[1], 0 */
  60       encoding = (0b10 << 30);
  61       encoding |= ctx.opcode[(int)aco_opcode::s_addc_u32] << 23;
  62       encoding |= (dest + 1) << 16;
  63       encoding |= dest + 1;
  64       encoding |= 128 << 8;
  65       out.push_back(encoding);
  66       return;
  67    }
  68
  69    uint32_t opcode = ctx.opcode[(int)instr->opcode];
  70    if (opcode == (uint32_t)-1) {
  71       fprintf(stderr, "Unsupported opcode: ");
  72       aco_print_instr(instr, stderr);
  73       abort();
  74    }
  75
  76    switch (instr->format) {
  77    case Format::SOP2: {
  78       uint32_t encoding = (0b10 << 30);
  79       encoding |= opcode << 23;
  80       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
  81       encoding |= instr->operands.size() >= 2 ? instr->operands[1].physReg() << 8 : 0;
  82       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
  83       out.push_back(encoding);
  84       break;
  85    }
  86    case Format::SOPK: {
  87       SOPK_instruction *sopk = static_cast<SOPK_instruction*>(instr);
  88
  89       if (instr->opcode == aco_opcode::s_subvector_loop_begin) {
  90          assert(ctx.chip_class >= GFX10);
  91          assert(ctx.subvector_begin_pos == -1);
  92          ctx.subvector_begin_pos = out.size();
  93       } else if (instr->opcode == aco_opcode::s_subvector_loop_end) {
  94          assert(ctx.chip_class >= GFX10);
  95          assert(ctx.subvector_begin_pos != -1);
  96          /* Adjust s_subvector_loop_begin instruction to the address after the end  */
  97          out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos);
  98          /* Adjust s_subvector_loop_end instruction to the address after the beginning  */
  99          sopk->imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size());
 100          ctx.subvector_begin_pos = -1;
 101       }
 102
 103       uint32_t encoding = (0b1011 << 28);
 104       encoding |= opcode << 23;
 105       encoding |=
 106          !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
 107          instr->definitions[0].physReg() << 16 :
 108          !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
 109          instr->operands[0].physReg() << 16 : 0;
 110       encoding |= sopk->imm;
 111       out.push_back(encoding);
 112       break;
 113    }
 114    case Format::SOP1: {
 115       uint32_t encoding = (0b101111101 << 23);
 116       if (opcode >= 55 && ctx.chip_class <= GFX9) {
 117          assert(ctx.chip_class == GFX9 && opcode < 60);
 118          opcode = opcode - 4;
 119       }
 120       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
 121       encoding |= opcode << 8;
 122       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 123       out.push_back(encoding);
 124       break;
 125    }
 126    case Format::SOPC: {
 127       uint32_t encoding = (0b101111110 << 23);
 128       encoding |= opcode << 16;
 129       encoding |= instr->operands.size() == 2 ? instr->operands[1].physReg() << 8 : 0;
 130       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 131       out.push_back(encoding);
 132       break;
 133    }
 134    case Format::SOPP: {
 135       SOPP_instruction* sopp = static_cast<SOPP_instruction*>(instr);
 136       uint32_t encoding = (0b101111111 << 23);
 137       encoding |= opcode << 16;
 138       encoding |= (uint16_t) sopp->imm;
 139       if (sopp->block != -1)
 140          ctx.branches.emplace_back(out.size(), sopp);
 141       out.push_back(encoding);
 142       break;
 143    }
 144    case Format::SMEM: {
 145       SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
 146       bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
 147       bool is_load = !instr->definitions.empty();
 148
 149       uint32_t encoding = 0;
 150
 151       if (ctx.chip_class <= GFX9) {
 152          encoding = (0b110000 << 26);
 153          assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 154          encoding |= smem->nv ? 1 << 15 : 0;
 155       } else {
 156          encoding = (0b111101 << 26);
 157          assert(!smem->nv); /* Non-volatile is not supported on GFX10 */
 158          encoding |= smem->dlc ? 1 << 14 : 0;
 159       }
 160
 161       encoding |= opcode << 18;
 162       encoding |= smem->glc ? 1 << 16 : 0;
 163
 164       if (ctx.chip_class <= GFX9) {
 165          if (instr->operands.size() >= 2)
 166             encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
 167       }
 168       if (ctx.chip_class == GFX9) {
 169          encoding |= soe ? 1 << 14 : 0;
 170       }
 171
 172       if (is_load || instr->operands.size() >= 3) { /* SDATA */
 173          encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
 174       }
 175       if (instr->operands.size() >= 1) { /* SBASE */
 176          encoding |= instr->operands[0].physReg() >> 1;
 177       }
 178
 179       out.push_back(encoding);
 180       encoding = 0;
 181
 182       int32_t offset = 0;
 183       uint32_t soffset = ctx.chip_class >= GFX10
 184                          ? sgpr_null /* On GFX10 this is disabled by specifying SGPR_NULL */
 185                          : 0;        /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
 186       if (instr->operands.size() >= 2) {
 187          const Operand &op_off1 = instr->operands[1];
 188          if (ctx.chip_class <= GFX9) {
 189             offset = op_off1.isConstant() ? op_off1.constantValue() : op_off1.physReg();
 190          } else {
 191             /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
 192             if (op_off1.isConstant()) {
 193                offset = op_off1.constantValue();
 194             } else {
 195                soffset = op_off1.physReg();
 196                assert(!soe); /* There is no place to put the other SGPR offset, if any */
 197             }
 198          }
 199
 200          if (soe) {
 201             const Operand &op_off2 = instr->operands.back();
 202             assert(ctx.chip_class >= GFX9); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
 203             assert(!op_off2.isConstant());
 204             soffset = op_off2.physReg();
 205          }
 206       }
 207       encoding |= offset;
 208       encoding |= soffset << 25;
 209
 210       out.push_back(encoding);
 211       return;
 212    }
 213    case Format::VOP2: {
 214       uint32_t encoding = 0;
 215       encoding |= opcode << 25;
 216       encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
 217       encoding |= (0xFF & instr->operands[1].physReg()) << 9;
 218       encoding |= instr->operands[0].physReg();
 219       out.push_back(encoding);
 220       break;
 221    }
 222    case Format::VOP1: {
 223       uint32_t encoding = (0b0111111 << 25);
 224       if (!instr->definitions.empty())
 225          encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
 226       encoding |= opcode << 9;
 227       if (!instr->operands.empty())
 228          encoding |= instr->operands[0].physReg();
 229       out.push_back(encoding);
 230       break;
 231    }
 232    case Format::VOPC: {
 233       uint32_t encoding = (0b0111110 << 25);
 234       encoding |= opcode << 17;
 235       encoding |= (0xFF & instr->operands[1].physReg()) << 9;
 236       encoding |= instr->operands[0].physReg();
 237       out.push_back(encoding);
 238       break;
 239    }
 240    case Format::VINTRP: {
 241       Interp_instruction* interp = static_cast<Interp_instruction*>(instr);
 242       uint32_t encoding = 0;
 243
 244       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 245          encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
 246       } else {
 247          encoding = (0b110010 << 26);
 248       }
 249
 250       assert(encoding);
 251       encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
 252       encoding |= opcode << 16;
 253       encoding |= interp->attribute << 10;
 254       encoding |= interp->component << 8;
 255       if (instr->opcode == aco_opcode::v_interp_mov_f32)
 256          encoding |= (0x3 & instr->operands[0].constantValue());
 257       else
 258          encoding |= (0xFF & instr->operands[0].physReg());
 259       out.push_back(encoding);
 260       break;
 261    }
 262    case Format::DS: {
 263       DS_instruction* ds = static_cast<DS_instruction*>(instr);
 264       uint32_t encoding = (0b110110 << 26);
 265       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 266          encoding |= opcode << 17;
 267          encoding |= (ds->gds ? 1 : 0) << 16;
 268       } else {
 269          encoding |= opcode << 18;
 270          encoding |= (ds->gds ? 1 : 0) << 17;
 271       }
 272       encoding |= ((0xFF & ds->offset1) << 8);
 273       encoding |= (0xFFFF & ds->offset0);
 274       out.push_back(encoding);
 275       encoding = 0;
 276       unsigned reg = !instr->definitions.empty() ? instr->definitions[0].physReg() : 0;
 277       encoding |= (0xFF & reg) << 24;
 278       reg = instr->operands.size() >= 3 && !(instr->operands[2].physReg() == m0)  ? instr->operands[2].physReg() : 0;
 279       encoding |= (0xFF & reg) << 16;
 280       reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
 281       encoding |= (0xFF & reg) << 8;
 282       encoding |= (0xFF & instr->operands[0].physReg());
 283       out.push_back(encoding);
 284       break;
 285    }
 286    case Format::MUBUF: {
 287       MUBUF_instruction* mubuf = static_cast<MUBUF_instruction*>(instr);
 288       uint32_t encoding = (0b111000 << 26);
 289       encoding |= opcode << 18;
 290       encoding |= (mubuf->lds ? 1 : 0) << 16;
 291       encoding |= (mubuf->glc ? 1 : 0) << 14;
 292       encoding |= (mubuf->idxen ? 1 : 0) << 13;
 293       encoding |= (mubuf->offen ? 1 : 0) << 12;
 294       if (ctx.chip_class <= GFX9) {
 295          assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 296          encoding |= (mubuf->slc ? 1 : 0) << 17;
 297       } else if (ctx.chip_class >= GFX10) {
 298          encoding |= (mubuf->dlc ? 1 : 0) << 15;
 299       }
 300       encoding |= 0x0FFF & mubuf->offset;
 301       out.push_back(encoding);
 302       encoding = 0;
 303       if (ctx.chip_class >= GFX10) {
 304          encoding |= (mubuf->slc ? 1 : 0) << 22;
 305       }
 306       encoding |= instr->operands[2].physReg() << 24;
 307       encoding |= (mubuf->tfe ? 1 : 0) << 23;
 308       encoding |= (instr->operands[1].physReg() >> 2) << 16;
 309       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
 310       encoding |= (0xFF & reg) << 8;
 311       encoding |= (0xFF & instr->operands[0].physReg());
 312       out.push_back(encoding);
 313       break;
 314    }
 315    case Format::MTBUF: {
 316       MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
 317
 318       uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
 319       uint32_t encoding = (0b111010 << 26);
 320       assert(img_format <= 0x7F);
 321       assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
 322       encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
 323       encoding |= (mtbuf->glc ? 1 : 0) << 14;
 324       encoding |= (mtbuf->idxen ? 1 : 0) << 13;
 325       encoding |= (mtbuf->offen ? 1 : 0) << 12;
 326       encoding |= 0x0FFF & mtbuf->offset;
 327       encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
 328
 329       if (ctx.chip_class <= GFX9) {
 330          encoding |= opcode << 15;
 331       } else {
 332          encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
 333       }
 334
 335       out.push_back(encoding);
 336       encoding = 0;
 337
 338       encoding |= instr->operands[2].physReg() << 24;
 339       encoding |= (mtbuf->tfe ? 1 : 0) << 23;
 340       encoding |= (mtbuf->slc ? 1 : 0) << 22;
 341       encoding |= (instr->operands[1].physReg() >> 2) << 16;
 342       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
 343       encoding |= (0xFF & reg) << 8;
 344       encoding |= (0xFF & instr->operands[0].physReg());
 345
 346       if (ctx.chip_class >= GFX10) {
 347          encoding |= (((opcode & 0x08) >> 4) << 21); /* MSB of 4-bit OPCODE */
 348       }
 349
 350       out.push_back(encoding);
 351       break;
 352    }
 353    case Format::MIMG: {
 354       MIMG_instruction* mimg = static_cast<MIMG_instruction*>(instr);
 355       uint32_t encoding = (0b111100 << 26);
 356       encoding |= mimg->slc ? 1 << 25 : 0;
 357       encoding |= opcode << 18;
 358       encoding |= mimg->lwe ? 1 << 17 : 0;
 359       encoding |= mimg->tfe ? 1 << 16 : 0;
 360       encoding |= mimg->glc ? 1 << 13 : 0;
 361       encoding |= mimg->unrm ? 1 << 12 : 0;
 362       if (ctx.chip_class <= GFX9) {
 363          assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 364          assert(!mimg->r128);
 365          encoding |= mimg->a16 ? 1 << 15 : 0;
 366          encoding |= mimg->da ? 1 << 14 : 0;
 367       } else {
 368          encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
 369          encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
 370          encoding |= mimg->dlc ? 1 << 7 : 0;
 371       }
 372       encoding |= (0xF & mimg->dmask) << 8;
 373       out.push_back(encoding);
 374       encoding = (0xFF & instr->operands[0].physReg()); /* VADDR */
 375       if (!instr->definitions.empty()) {
 376          encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
 377       } else if (instr->operands.size() == 4) {
 378          encoding |= (0xFF & instr->operands[3].physReg()) << 8; /* VDATA */
 379       }
 380       encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 16; /* T# (resource) */
 381       if (instr->operands.size() > 2)
 382          encoding |= (0x1F & (instr->operands[2].physReg() >> 2)) << 21; /* sampler */
 383
 384       assert(!mimg->d16 || ctx.chip_class >= GFX9);
 385       encoding |= mimg->d16 ? 1 << 15 : 0;
 386       if (ctx.chip_class >= GFX10) {
 387          encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
 388       }
 389
 390       out.push_back(encoding);
 391       break;
 392    }
 393    case Format::FLAT:
 394    case Format::SCRATCH:
 395    case Format::GLOBAL: {
 396       FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
 397       uint32_t encoding = (0b110111 << 26);
 398       encoding |= opcode << 18;
 399       if (ctx.chip_class <= GFX9) {
 400          assert(flat->offset <= 0x1fff);
 401          encoding |= flat->offset & 0x1fff;
 402       } else if (instr->format == Format::FLAT) {
 403          /* GFX10 has a 12-bit immediate OFFSET field,
 404           * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
 405           */
 406          assert(flat->offset == 0);
 407       } else {
 408          assert(flat->offset <= 0xfff);
 409          encoding |= flat->offset & 0xfff;
 410       }
 411       if (instr->format == Format::SCRATCH)
 412          encoding |= 1 << 14;
 413       else if (instr->format == Format::GLOBAL)
 414          encoding |= 2 << 14;
 415       encoding |= flat->lds ? 1 << 13 : 0;
 416       encoding |= flat->glc ? 1 << 16 : 0;
 417       encoding |= flat->slc ? 1 << 17 : 0;
 418       if (ctx.chip_class >= GFX10) {
 419          assert(!flat->nv);
 420          encoding |= flat->dlc ? 1 << 12 : 0;
 421       } else {
 422          assert(!flat->dlc);
 423       }
 424       out.push_back(encoding);
 425       encoding = (0xFF & instr->operands[0].physReg());
 426       if (!instr->definitions.empty())
 427          encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
 428       if (instr->operands.size() >= 3)
 429          encoding |= (0xFF & instr->operands[2].physReg()) << 8;
 430       if (!instr->operands[1].isUndefined()) {
 431          assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
 432          assert(instr->format != Format::FLAT);
 433          encoding |= instr->operands[1].physReg() << 16;
 434       } else if (instr->format != Format::FLAT) {
 435          if (ctx.chip_class <= GFX9)
 436             encoding |= 0x7F << 16;
 437          else
 438             encoding |= sgpr_null << 16;
 439       }
 440       encoding |= flat->nv ? 1 << 23 : 0;
 441       out.push_back(encoding);
 442       break;
 443    }
 444    case Format::EXP: {
 445       Export_instruction* exp = static_cast<Export_instruction*>(instr);
 446       uint32_t encoding;
 447       if (ctx.chip_class <= GFX9) {
 448          encoding = (0b110001 << 26);
 449       } else if (ctx.chip_class >= GFX10) {
 450          encoding = (0b111110 << 26);
 451       }
 452
 453       encoding |= exp->valid_mask ? 0b1 << 12 : 0;
 454       encoding |= exp->done ? 0b1 << 11 : 0;
 455       encoding |= exp->compressed ? 0b1 << 10 : 0;
 456       encoding |= exp->dest << 4;
 457       encoding |= exp->enabled_mask;
 458       out.push_back(encoding);
 459       encoding = 0xFF & exp->operands[0].physReg();
 460       encoding |= (0xFF & exp->operands[1].physReg()) << 8;
 461       encoding |= (0xFF & exp->operands[2].physReg()) << 16;
 462       encoding |= (0xFF & exp->operands[3].physReg()) << 24;
 463       out.push_back(encoding);
 464       break;
 465    }
 466    case Format::PSEUDO:
 467    case Format::PSEUDO_BARRIER:
 468       unreachable("Pseudo instructions should be lowered before assembly.");
 469    default:
 470       if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
 471          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
 472
 473          if ((uint16_t) instr->format & (uint16_t) Format::VOP2) {
 474             opcode = opcode + 0x100;
 475          } else if ((uint16_t) instr->format & (uint16_t) Format::VOP1) {
 476             if (ctx.chip_class <= GFX9) {
 477                opcode = opcode + 0x140;
 478             } else {
 479                /* RDNA ISA doc says this is 0x140, but that doesn't work  */
 480                opcode = opcode + 0x180;
 481             }
 482          } else if ((uint16_t) instr->format & (uint16_t) Format::VOPC) {
 483             opcode = opcode + 0x0;
 484          } else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP) {
 485             opcode = opcode + 0x270;
 486          }
 487
 488          uint32_t encoding;
 489          if (ctx.chip_class <= GFX9) {
 490             encoding = (0b110100 << 26);
 491          } else if (ctx.chip_class == GFX10) {
 492             encoding = (0b110101 << 26);
 493          }
 494
 495          encoding |= opcode << 16;
 496          encoding |= (vop3->clamp ? 1 : 0) << 15;
 497          for (unsigned i = 0; i < 3; i++)
 498             encoding |= vop3->abs[i] << (8+i);
 499          for (unsigned i = 0; i < 4; i++)
 500             encoding |= vop3->opsel[i] << (11+i);
 501          if (instr->definitions.size() == 2)
 502             encoding |= instr->definitions[1].physReg() << 8;
 503          encoding |= (0xFF & instr->definitions[0].physReg());
 504          out.push_back(encoding);
 505          encoding = 0;
 506          if (instr->opcode == aco_opcode::v_interp_mov_f32) {
 507             encoding = 0x3 & instr->operands[0].constantValue();
 508          } else {
 509             for (unsigned i = 0; i < instr->operands.size(); i++)
 510                encoding |= instr->operands[i].physReg() << (i * 9);
 511          }
 512          encoding |= vop3->omod << 27;
 513          for (unsigned i = 0; i < 3; i++)
 514             encoding |= vop3->neg[i] << (29+i);
 515          out.push_back(encoding);
 516
 517       } else if (instr->isDPP()){
 518          /* first emit the instruction without the DPP operand */
 519          Operand dpp_op = instr->operands[0];
 520          instr->operands[0] = Operand(PhysReg{250}, v1);
 521          instr->format = (Format) ((uint32_t) instr->format & ~(1 << 14));
 522          emit_instruction(ctx, out, instr);
 523          DPP_instruction* dpp = static_cast<DPP_instruction*>(instr);
 524          uint32_t encoding = (0xF & dpp->row_mask) << 28;
 525          encoding |= (0xF & dpp->bank_mask) << 24;
 526          encoding |= dpp->abs[1] << 23;
 527          encoding |= dpp->neg[1] << 22;
 528          encoding |= dpp->abs[0] << 21;
 529          encoding |= dpp->neg[0] << 20;
 530          encoding |= dpp->bound_ctrl << 19;
 531          encoding |= dpp->dpp_ctrl << 8;
 532          encoding |= (0xFF) & dpp_op.physReg();
 533          out.push_back(encoding);
 534          return;
 535       } else {
 536          unreachable("unimplemented instruction format");
 537       }
 538       break;
 539    }
 540
 541    /* append literal dword */
 542    for (const Operand& op : instr->operands) {
 543       if (op.isLiteral()) {
 544          out.push_back(op.constantValue());
 545          break;
 546       }
 547    }
 548 }
 549
 550 void emit_block(asm_context& ctx, std::vector<uint32_t>& out, Block& block)
 551 {
 552    for (aco_ptr<Instruction>& instr : block.instructions) {
 553 #if 0
 554       int start_idx = out.size();
 555       std::cerr << "Encoding:\t" << std::endl;
 556       aco_print_instr(&*instr, stderr);
 557       std::cerr << std::endl;
 558 #endif
 559       emit_instruction(ctx, out, instr.get());
 560 #if 0
 561       for (int i = start_idx; i < out.size(); i++)
 562          std::cerr << "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex << out[i] << std::endl;
 563 #endif
 564    }
 565 }
 566
 567 void fix_exports(asm_context& ctx, std::vector<uint32_t>& out, Program* program)
 568 {
 569    for (int idx = program->blocks.size() - 1; idx >= 0; idx--) {
 570       Block& block = program->blocks[idx];
 571       std::vector<aco_ptr<Instruction>>::reverse_iterator it = block.instructions.rbegin();
 572       bool endBlock = false;
 573       bool exported = false;
 574       while ( it != block.instructions.rend())
 575       {
 576          if ((*it)->format == Format::EXP && endBlock) {
 577             Export_instruction* exp = static_cast<Export_instruction*>((*it).get());
 578             if (program->stage & hw_vs) {
 579                if (exp->dest >= V_008DFC_SQ_EXP_POS && exp->dest <= (V_008DFC_SQ_EXP_POS + 3)) {
 580                   exp->done = true;
 581                   exported = true;
 582                   break;
 583                }
 584             } else {
 585                exp->done = true;
 586                exp->valid_mask = true;
 587                exported = true;
 588                break;
 589             }
 590          } else if ((*it)->definitions.size() && (*it)->definitions[0].physReg() == exec)
 591             break;
 592          else if ((*it)->opcode == aco_opcode::s_endpgm) {
 593             if (endBlock)
 594                break;
 595             endBlock = true;
 596          }
 597          ++it;
 598       }
 599       if (!endBlock || exported)
 600          continue;
 601       /* we didn't find an Export instruction and have to insert a null export */
 602       aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
 603       for (unsigned i = 0; i < 4; i++)
 604          exp->operands[i] = Operand(v1);
 605       exp->enabled_mask = 0;
 606       exp->compressed = false;
 607       exp->done = true;
 608       exp->valid_mask = program->stage & hw_fs;
 609       if (program->stage & hw_fs)
 610          exp->dest = 9; /* NULL */
 611       else
 612          exp->dest = V_008DFC_SQ_EXP_POS;
 613       /* insert the null export 1 instruction before endpgm */
 614       block.instructions.insert(block.instructions.end() - 1, std::move(exp));
 615    }
 616 }
 617
 618 static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
 619 {
 620    /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
 621    bool gfx10_3f_bug = false;
 622
 623    do {
 624       auto buggy_branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [&ctx](const auto &branch) -> bool {
 625          return ((int)ctx.program->blocks[branch.second->block].offset - branch.first - 1) == 0x3f;
 626       });
 627
 628       gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
 629
 630       if (gfx10_3f_bug) {
 631          /* Insert an s_nop after the branch */
 632          constexpr uint32_t s_nop_0 = 0xbf800000u;
 633          int s_nop_pos = buggy_branch_it->first + 1;
 634          auto out_pos = std::next(out.begin(), s_nop_pos);
 635          out.insert(out_pos, s_nop_0);
 636
 637          /* Update the offset of each affected block */
 638          for (Block& block : ctx.program->blocks) {
 639             if (block.offset > (unsigned)buggy_branch_it->first)
 640                block.offset++;
 641          }
 642
 643          /* Update the branches following the current one */
 644          for (auto branch_it = std::next(buggy_branch_it); branch_it != ctx.branches.end(); ++branch_it)
 645             branch_it->first++;
 646
 647          /* Find first constant address after the inserted instruction */
 648          auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [s_nop_pos](const int &caddr_pos) -> bool {
 649             return caddr_pos >= s_nop_pos;
 650          });
 651
 652          /* Update the locations of constant addresses */
 653          for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
 654             (*caddr_it)++;
 655
 656       }
 657    } while (gfx10_3f_bug);
 658 }
 659
 660 void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
 661 {
 662    if (ctx.chip_class >= GFX10)
 663       fix_branches_gfx10(ctx, out);
 664
 665    for (std::pair<int, SOPP_instruction*> &branch : ctx.branches) {
 666       int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
 667       out[branch.first] |= (uint16_t) offset;
 668    }
 669 }
 670
 671 void fix_constaddrs(asm_context& ctx, std::vector<uint32_t>& out)
 672 {
 673    for (unsigned addr : ctx.constaddrs)
 674       out[addr] += out.size() * 4u;
 675 }
 676
 677 unsigned emit_program(Program* program,
 678                       std::vector<uint32_t>& code)
 679 {
 680    asm_context ctx(program);
 681
 682    if (program->stage & (hw_vs | hw_fs))
 683       fix_exports(ctx, code, program);
 684
 685    for (Block& block : program->blocks) {
 686       block.offset = code.size();
 687       emit_block(ctx, code, block);
 688    }
 689
 690    fix_branches(ctx, code);
 691
 692    unsigned exec_size = code.size() * sizeof(uint32_t);
 693
 694    if (program->chip_class >= GFX10) {
 695       /* Pad output with s_code_end so instruction prefetching doesn't cause
 696        * page faults */
 697       unsigned final_size = align(code.size() + 3 * 16, 16);
 698       while (code.size() < final_size)
 699          code.push_back(0xbf9f0000u);
 700    }
 701
 702    fix_constaddrs(ctx, code);
 703
 704    while (program->constant_data.size() % 4u)
 705       program->constant_data.push_back(0);
 706    /* Copy constant data */
 707    code.insert(code.end(), (uint32_t*)program->constant_data.data(),
 708                (uint32_t*)(program->constant_data.data() + program->constant_data.size()));
 709
 710    return exec_size;
 711 }
 712
 713 }