src/amd/compiler/aco_assembler.cpp

   1 #include <map>
   2
   3 #include "aco_ir.h"
   4 #include "common/sid.h"
   5 #include "ac_shader_util.h"
   6
   7 namespace aco {
   8
   9 struct asm_context {
  10    Program *program;
  11    enum chip_class chip_class;
  12    std::map<int, SOPP_instruction*> branches;
  13    std::vector<unsigned> constaddrs;
  14    const int16_t* opcode;
  15    // TODO: keep track of branch instructions referring blocks
  16    // and, when emitting the block, correct the offset in instr
  17    asm_context(Program* program) : program(program), chip_class(program->chip_class) {
  18       if (chip_class <= GFX9)
  19          opcode = &instr_info.opcode_gfx9[0];
  20       else if (chip_class == GFX10)
  21          opcode = &instr_info.opcode_gfx10[0];
  22    }
  23 };
  24
  25 void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
  26 {
  27    uint32_t instr_offset = out.size() * 4u;
  28
  29    /* lower remaining pseudo-instructions */
  30    if (instr->opcode == aco_opcode::p_constaddr) {
  31       unsigned dest = instr->definitions[0].physReg();
  32       unsigned offset = instr->operands[0].constantValue();
  33
  34       /* s_getpc_b64 dest[0:1] */
  35       uint32_t encoding = (0b101111101 << 23);
  36       uint32_t opcode = ctx.opcode[(int)aco_opcode::s_getpc_b64];
  37       if (opcode >= 55 && ctx.chip_class <= GFX9) {
  38          assert(ctx.chip_class == GFX9 && opcode < 60);
  39          opcode = opcode - 4;
  40       }
  41       encoding |= dest << 16;
  42       encoding |= opcode << 8;
  43       out.push_back(encoding);
  44
  45       /* s_add_u32 dest[0], dest[0], ... */
  46       encoding = (0b10 << 30);
  47       encoding |= ctx.opcode[(int)aco_opcode::s_add_u32] << 23;
  48       encoding |= dest << 16;
  49       encoding |= dest;
  50       encoding |= 255 << 8;
  51       out.push_back(encoding);
  52       ctx.constaddrs.push_back(out.size());
  53       out.push_back(-(instr_offset + 4) + offset);
  54
  55       /* s_addc_u32 dest[1], dest[1], 0 */
  56       encoding = (0b10 << 30);
  57       encoding |= ctx.opcode[(int)aco_opcode::s_addc_u32] << 23;
  58       encoding |= (dest + 1) << 16;
  59       encoding |= dest + 1;
  60       encoding |= 128 << 8;
  61       out.push_back(encoding);
  62       return;
  63    }
  64
  65    uint32_t opcode = ctx.opcode[(int)instr->opcode];
  66    if (opcode == (uint32_t)-1) {
  67       fprintf(stderr, "Unsupported opcode: ");
  68       aco_print_instr(instr, stderr);
  69       abort();
  70    }
  71
  72    switch (instr->format) {
  73    case Format::SOP2: {
  74       uint32_t encoding = (0b10 << 30);
  75       encoding |= opcode << 23;
  76       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
  77       encoding |= instr->operands.size() >= 2 ? instr->operands[1].physReg() << 8 : 0;
  78       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
  79       out.push_back(encoding);
  80       break;
  81    }
  82    case Format::SOPK: {
  83       uint32_t encoding = (0b1011 << 28);
  84       encoding |= opcode << 23;
  85       encoding |=
  86          !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
  87          instr->definitions[0].physReg() << 16 :
  88          !instr->operands.empty() && !(instr->operands[0].physReg() == scc) ?
  89          instr->operands[0].physReg() << 16 : 0;
  90       encoding |= static_cast<SOPK_instruction*>(instr)->imm;
  91       out.push_back(encoding);
  92       break;
  93    }
  94    case Format::SOP1: {
  95       uint32_t encoding = (0b101111101 << 23);
  96       if (opcode >= 55 && ctx.chip_class <= GFX9) {
  97          assert(ctx.chip_class == GFX9 && opcode < 60);
  98          opcode = opcode - 4;
  99       }
 100       encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
 101       encoding |= opcode << 8;
 102       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 103       out.push_back(encoding);
 104       break;
 105    }
 106    case Format::SOPC: {
 107       uint32_t encoding = (0b101111110 << 23);
 108       encoding |= opcode << 16;
 109       encoding |= instr->operands.size() == 2 ? instr->operands[1].physReg() << 8 : 0;
 110       encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
 111       out.push_back(encoding);
 112       break;
 113    }
 114    case Format::SOPP: {
 115       SOPP_instruction* sopp = static_cast<SOPP_instruction*>(instr);
 116       uint32_t encoding = (0b101111111 << 23);
 117       encoding |= opcode << 16;
 118       encoding |= (uint16_t) sopp->imm;
 119       if (sopp->block != -1)
 120          ctx.branches.insert({out.size(), sopp});
 121       out.push_back(encoding);
 122       break;
 123    }
 124    case Format::SMEM: {
 125       SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
 126       bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
 127       bool is_load = !instr->definitions.empty();
 128
 129       uint32_t encoding = 0;
 130
 131       if (ctx.chip_class <= GFX9) {
 132          encoding = (0b110000 << 26);
 133          assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 134          encoding |= smem->nv ? 1 << 15 : 0;
 135       } else {
 136          encoding = (0b111101 << 26);
 137          assert(!smem->nv); /* Non-volatile is not supported on GFX10 */
 138          encoding |= smem->dlc ? 1 << 14 : 0;
 139       }
 140
 141       encoding |= opcode << 18;
 142       encoding |= smem->glc ? 1 << 16 : 0;
 143
 144       if (ctx.chip_class <= GFX9) {
 145          if (instr->operands.size() >= 2)
 146             encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
 147       }
 148       if (ctx.chip_class == GFX9) {
 149          encoding |= soe ? 1 << 14 : 0;
 150       }
 151
 152       if (is_load || instr->operands.size() >= 3) { /* SDATA */
 153          encoding |= (is_load ? instr->definitions[0].physReg().reg : instr->operands[2].physReg().reg) << 6;
 154       }
 155       if (instr->operands.size() >= 1) { /* SBASE */
 156          encoding |= instr->operands[0].physReg().reg >> 1;
 157       }
 158
 159       out.push_back(encoding);
 160       encoding = 0;
 161
 162       int32_t offset = 0;
 163       uint32_t soffset = ctx.chip_class >= GFX10
 164                          ? sgpr_null /* On GFX10 this is disabled by specifying SGPR_NULL */
 165                          : 0;        /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
 166       if (instr->operands.size() >= 2) {
 167          const Operand &op_off1 = instr->operands[1];
 168          if (ctx.chip_class <= GFX9) {
 169             offset = op_off1.isConstant() ? op_off1.constantValue() : op_off1.physReg();
 170          } else {
 171             /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
 172             if (op_off1.isConstant()) {
 173                offset = op_off1.constantValue();
 174             } else {
 175                soffset = op_off1.physReg();
 176                assert(!soe); /* There is no place to put the other SGPR offset, if any */
 177             }
 178          }
 179
 180          if (soe) {
 181             const Operand &op_off2 = instr->operands.back();
 182             assert(ctx.chip_class >= GFX9); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
 183             assert(!op_off2.isConstant());
 184             soffset = op_off2.physReg();
 185          }
 186       }
 187       encoding |= offset;
 188       encoding |= soffset << 25;
 189
 190       out.push_back(encoding);
 191       return;
 192    }
 193    case Format::VOP2: {
 194       uint32_t encoding = 0;
 195       encoding |= opcode << 25;
 196       encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
 197       encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
 198       encoding |= instr->operands[0].physReg().reg;
 199       out.push_back(encoding);
 200       break;
 201    }
 202    case Format::VOP1: {
 203       uint32_t encoding = (0b0111111 << 25);
 204       encoding |= (0xFF & instr->definitions[0].physReg().reg) << 17;
 205       encoding |= opcode << 9;
 206       encoding |= instr->operands[0].physReg().reg;
 207       out.push_back(encoding);
 208       break;
 209    }
 210    case Format::VOPC: {
 211       uint32_t encoding = (0b0111110 << 25);
 212       encoding |= opcode << 17;
 213       encoding |= (0xFF & instr->operands[1].physReg().reg) << 9;
 214       encoding |= instr->operands[0].physReg().reg;
 215       out.push_back(encoding);
 216       break;
 217    }
 218    case Format::VINTRP: {
 219       Interp_instruction* interp = static_cast<Interp_instruction*>(instr);
 220       uint32_t encoding = 0;
 221
 222       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 223          encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
 224       } else {
 225          encoding = (0b110010 << 26);
 226       }
 227
 228       assert(encoding);
 229       encoding |= (0xFF & instr->definitions[0].physReg().reg) << 18;
 230       encoding |= opcode << 16;
 231       encoding |= interp->attribute << 10;
 232       encoding |= interp->component << 8;
 233       if (instr->opcode == aco_opcode::v_interp_mov_f32)
 234          encoding |= (0x3 & instr->operands[0].constantValue());
 235       else
 236          encoding |= (0xFF & instr->operands[0].physReg().reg);
 237       out.push_back(encoding);
 238       break;
 239    }
 240    case Format::DS: {
 241       DS_instruction* ds = static_cast<DS_instruction*>(instr);
 242       uint32_t encoding = (0b110110 << 26);
 243       if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
 244          encoding |= opcode << 17;
 245          encoding |= (ds->gds ? 1 : 0) << 16;
 246       } else {
 247          encoding |= opcode << 18;
 248          encoding |= (ds->gds ? 1 : 0) << 17;
 249       }
 250       encoding |= ((0xFF & ds->offset1) << 8);
 251       encoding |= (0xFFFF & ds->offset0);
 252       out.push_back(encoding);
 253       encoding = 0;
 254       unsigned reg = !instr->definitions.empty() ? instr->definitions[0].physReg() : 0;
 255       encoding |= (0xFF & reg) << 24;
 256       reg = instr->operands.size() >= 3 && !(instr->operands[2].physReg() == m0)  ? instr->operands[2].physReg() : 0;
 257       encoding |= (0xFF & reg) << 16;
 258       reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
 259       encoding |= (0xFF & reg) << 8;
 260       encoding |= (0xFF & instr->operands[0].physReg().reg);
 261       out.push_back(encoding);
 262       break;
 263    }
 264    case Format::MUBUF: {
 265       MUBUF_instruction* mubuf = static_cast<MUBUF_instruction*>(instr);
 266       uint32_t encoding = (0b111000 << 26);
 267       encoding |= opcode << 18;
 268       encoding |= (mubuf->lds ? 1 : 0) << 16;
 269       encoding |= (mubuf->glc ? 1 : 0) << 14;
 270       encoding |= (mubuf->idxen ? 1 : 0) << 13;
 271       encoding |= (mubuf->offen ? 1 : 0) << 12;
 272       if (ctx.chip_class <= GFX9) {
 273          assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 274          encoding |= (mubuf->slc ? 1 : 0) << 17;
 275       } else if (ctx.chip_class >= GFX10) {
 276          encoding |= (mubuf->dlc ? 1 : 0) << 15;
 277       }
 278       encoding |= 0x0FFF & mubuf->offset;
 279       out.push_back(encoding);
 280       encoding = 0;
 281       if (ctx.chip_class >= GFX10) {
 282          encoding |= (mubuf->slc ? 1 : 0) << 22;
 283       }
 284       encoding |= instr->operands[2].physReg() << 24;
 285       encoding |= (mubuf->tfe ? 1 : 0) << 23;
 286       encoding |= (instr->operands[1].physReg() >> 2) << 16;
 287       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg().reg;
 288       encoding |= (0xFF & reg) << 8;
 289       encoding |= (0xFF & instr->operands[0].physReg().reg);
 290       out.push_back(encoding);
 291       break;
 292    }
 293    case Format::MTBUF: {
 294       MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
 295
 296       uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
 297       uint32_t encoding = (0b111010 << 26);
 298       assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
 299       encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
 300       encoding |= (mtbuf->glc ? 1 : 0) << 14;
 301       encoding |= (mtbuf->idxen ? 1 : 0) << 13;
 302       encoding |= (mtbuf->offen ? 1 : 0) << 12;
 303       encoding |= 0x0FFF & mtbuf->offset;
 304       encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
 305
 306       if (ctx.chip_class <= GFX9) {
 307          encoding |= opcode << 15;
 308       } else {
 309          encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
 310       }
 311
 312       out.push_back(encoding);
 313       encoding = 0;
 314
 315       encoding |= instr->operands[2].physReg().reg << 24;
 316       encoding |= (mtbuf->tfe ? 1 : 0) << 23;
 317       encoding |= (mtbuf->slc ? 1 : 0) << 22;
 318       encoding |= (instr->operands[1].physReg().reg >> 2) << 16;
 319       unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg().reg : instr->definitions[0].physReg().reg;
 320       encoding |= (0xFF & reg) << 8;
 321       encoding |= (0xFF & instr->operands[0].physReg().reg);
 322
 323       if (ctx.chip_class >= GFX10) {
 324          encoding |= (((opcode & 0x08) >> 4) << 21); /* MSB of 4-bit OPCODE */
 325       }
 326
 327       out.push_back(encoding);
 328       break;
 329    }
 330    case Format::MIMG: {
 331       MIMG_instruction* mimg = static_cast<MIMG_instruction*>(instr);
 332       uint32_t encoding = (0b111100 << 26);
 333       encoding |= mimg->slc ? 1 << 25 : 0;
 334       encoding |= opcode << 18;
 335       encoding |= mimg->lwe ? 1 << 17 : 0;
 336       encoding |= mimg->tfe ? 1 << 16 : 0;
 337       encoding |= mimg->glc ? 1 << 13 : 0;
 338       encoding |= mimg->unrm ? 1 << 12 : 0;
 339       if (ctx.chip_class <= GFX9) {
 340          assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
 341          assert(!mimg->r128);
 342          encoding |= mimg->a16 ? 1 << 15 : 0;
 343          encoding |= mimg->da ? 1 << 14 : 0;
 344       } else {
 345          encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
 346          encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
 347          encoding |= mimg->dlc ? 1 << 7 : 0;
 348       }
 349       encoding |= (0xF & mimg->dmask) << 8;
 350       out.push_back(encoding);
 351       encoding = (0xFF & instr->operands[0].physReg().reg); /* VADDR */
 352       if (!instr->definitions.empty()) {
 353          encoding |= (0xFF & instr->definitions[0].physReg().reg) << 8; /* VDATA */
 354       } else if (instr->operands.size() == 4) {
 355          encoding |= (0xFF & instr->operands[3].physReg().reg) << 8; /* VDATA */
 356       }
 357       encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 16; /* T# (resource) */
 358       if (instr->operands.size() > 2)
 359          encoding |= (0x1F & (instr->operands[2].physReg() >> 2)) << 21; /* sampler */
 360
 361       assert(!mimg->d16 || ctx.chip_class >= GFX9);
 362       encoding |= mimg->d16 ? 1 << 15 : 0;
 363       if (ctx.chip_class >= GFX10) {
 364          encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
 365       }
 366
 367       out.push_back(encoding);
 368       break;
 369    }
 370    case Format::FLAT:
 371    case Format::SCRATCH:
 372    case Format::GLOBAL: {
 373       FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
 374       uint32_t encoding = (0b110111 << 26);
 375       encoding |= opcode << 18;
 376       if (ctx.chip_class <= GFX9) {
 377          assert(flat->offset <= 0x1fff);
 378          encoding |= flat->offset & 0x1fff;
 379       } else {
 380          assert(flat->offset <= 0x0fff);
 381          encoding |= flat->offset & 0x0fff;
 382       }
 383       if (instr->format == Format::SCRATCH)
 384          encoding |= 1 << 14;
 385       else if (instr->format == Format::GLOBAL)
 386          encoding |= 2 << 14;
 387       encoding |= flat->lds ? 1 << 13 : 0;
 388       encoding |= flat->glc ? 1 << 16 : 0;
 389       encoding |= flat->slc ? 1 << 17 : 0;
 390       if (ctx.chip_class >= GFX10) {
 391          assert(!flat->nv);
 392          encoding |= flat->dlc ? 1 << 12 : 0;
 393       } else {
 394          assert(!flat->dlc);
 395       }
 396       out.push_back(encoding);
 397       encoding = (0xFF & instr->operands[0].physReg());
 398       if (!instr->definitions.empty())
 399          encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
 400       else
 401          encoding |= (0xFF & instr->operands[2].physReg()) << 8;
 402       if (!instr->operands[1].isUndefined()) {
 403          assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
 404          assert(instr->format != Format::FLAT);
 405          encoding |= instr->operands[1].physReg() << 16;
 406       } else if (instr->format != Format::FLAT) {
 407          if (ctx.chip_class <= GFX9)
 408             encoding |= 0x7F << 16;
 409          else
 410             encoding |= sgpr_null << 16;
 411       }
 412       encoding |= flat->nv ? 1 << 23 : 0;
 413       out.push_back(encoding);
 414       break;
 415    }
 416    case Format::EXP: {
 417       Export_instruction* exp = static_cast<Export_instruction*>(instr);
 418       uint32_t encoding = (0b110001 << 26);
 419       encoding |= exp->valid_mask ? 0b1 << 12 : 0;
 420       encoding |= exp->done ? 0b1 << 11 : 0;
 421       encoding |= exp->compressed ? 0b1 << 10 : 0;
 422       encoding |= exp->dest << 4;
 423       encoding |= exp->enabled_mask;
 424       out.push_back(encoding);
 425       encoding = 0xFF & exp->operands[0].physReg().reg;
 426       encoding |= (0xFF & exp->operands[1].physReg().reg) << 8;
 427       encoding |= (0xFF & exp->operands[2].physReg().reg) << 16;
 428       encoding |= (0xFF & exp->operands[3].physReg().reg) << 24;
 429       out.push_back(encoding);
 430       break;
 431    }
 432    case Format::PSEUDO:
 433    case Format::PSEUDO_BARRIER:
 434       unreachable("Pseudo instructions should be lowered before assembly.");
 435    default:
 436       if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
 437          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
 438
 439          if ((uint16_t) instr->format & (uint16_t) Format::VOP2)
 440             opcode = opcode + 0x100;
 441          else if ((uint16_t) instr->format & (uint16_t) Format::VOP1)
 442             opcode = opcode + 0x140;
 443          else if ((uint16_t) instr->format & (uint16_t) Format::VOPC)
 444             opcode = opcode + 0x0;
 445          else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP)
 446             opcode = opcode + 0x270;
 447
 448          // TODO: op_sel
 449          uint32_t encoding = (0b110100 << 26);
 450          encoding |= opcode << 16;
 451          encoding |= (vop3->clamp ? 1 : 0) << 15;
 452          for (unsigned i = 0; i < 3; i++)
 453             encoding |= vop3->abs[i] << (8+i);
 454          if (instr->definitions.size() == 2)
 455             encoding |= instr->definitions[1].physReg() << 8;
 456          encoding |= (0xFF & instr->definitions[0].physReg().reg);
 457          out.push_back(encoding);
 458          encoding = 0;
 459          if (instr->opcode == aco_opcode::v_interp_mov_f32) {
 460             encoding = 0x3 & instr->operands[0].constantValue();
 461          } else {
 462             for (unsigned i = 0; i < instr->operands.size(); i++)
 463                encoding |= instr->operands[i].physReg() << (i * 9);
 464          }
 465          encoding |= vop3->omod << 27;
 466          for (unsigned i = 0; i < 3; i++)
 467             encoding |= vop3->neg[i] << (29+i);
 468          out.push_back(encoding);
 469          return;
 470
 471       } else if (instr->isDPP()){
 472          /* first emit the instruction without the DPP operand */
 473          Operand dpp_op = instr->operands[0];
 474          instr->operands[0] = Operand(PhysReg{250}, v1);
 475          instr->format = (Format) ((uint32_t) instr->format & ~(1 << 14));
 476          emit_instruction(ctx, out, instr);
 477          DPP_instruction* dpp = static_cast<DPP_instruction*>(instr);
 478          uint32_t encoding = (0xF & dpp->row_mask) << 28;
 479          encoding |= (0xF & dpp->bank_mask) << 24;
 480          encoding |= dpp->abs[1] << 23;
 481          encoding |= dpp->neg[1] << 22;
 482          encoding |= dpp->abs[0] << 21;
 483          encoding |= dpp->neg[0] << 20;
 484          encoding |= dpp->bound_ctrl << 19;
 485          encoding |= dpp->dpp_ctrl << 8;
 486          encoding |= (0xFF) & dpp_op.physReg().reg;
 487          out.push_back(encoding);
 488          return;
 489       } else {
 490          unreachable("unimplemented instruction format");
 491       }
 492    }
 493
 494    /* append literal dword */
 495    for (const Operand& op : instr->operands) {
 496       if (op.isLiteral()) {
 497          out.push_back(op.constantValue());
 498          break;
 499       }
 500    }
 501 }
 502
 503 void emit_block(asm_context& ctx, std::vector<uint32_t>& out, Block& block)
 504 {
 505    for (aco_ptr<Instruction>& instr : block.instructions) {
 506 #if 0
 507       int start_idx = out.size();
 508       std::cerr << "Encoding:\t" << std::endl;
 509       aco_print_instr(&*instr, stderr);
 510       std::cerr << std::endl;
 511 #endif
 512       emit_instruction(ctx, out, instr.get());
 513 #if 0
 514       for (int i = start_idx; i < out.size(); i++)
 515          std::cerr << "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex << out[i] << std::endl;
 516 #endif
 517    }
 518 }
 519
 520 void fix_exports(asm_context& ctx, std::vector<uint32_t>& out, Program* program)
 521 {
 522    for (int idx = program->blocks.size() - 1; idx >= 0; idx--) {
 523       Block& block = program->blocks[idx];
 524       std::vector<aco_ptr<Instruction>>::reverse_iterator it = block.instructions.rbegin();
 525       bool endBlock = false;
 526       bool exported = false;
 527       while ( it != block.instructions.rend())
 528       {
 529          if ((*it)->format == Format::EXP && endBlock) {
 530             Export_instruction* exp = static_cast<Export_instruction*>((*it).get());
 531             if (program->stage & hw_vs) {
 532                if (exp->dest >= V_008DFC_SQ_EXP_POS && exp->dest <= (V_008DFC_SQ_EXP_POS + 3)) {
 533                   exp->done = true;
 534                   exported = true;
 535                   break;
 536                }
 537             } else {
 538                exp->done = true;
 539                exp->valid_mask = true;
 540                exported = true;
 541                break;
 542             }
 543          } else if ((*it)->definitions.size() && (*it)->definitions[0].physReg() == exec)
 544             break;
 545          else if ((*it)->opcode == aco_opcode::s_endpgm) {
 546             if (endBlock)
 547                break;
 548             endBlock = true;
 549          }
 550          ++it;
 551       }
 552       if (!endBlock || exported)
 553          continue;
 554       /* we didn't find an Export instruction and have to insert a null export */
 555       aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
 556       for (unsigned i = 0; i < 4; i++)
 557          exp->operands[i] = Operand(v1);
 558       exp->enabled_mask = 0;
 559       exp->compressed = false;
 560       exp->done = true;
 561       exp->valid_mask = program->stage & hw_fs;
 562       if (program->stage & hw_fs)
 563          exp->dest = 9; /* NULL */
 564       else
 565          exp->dest = V_008DFC_SQ_EXP_POS;
 566       /* insert the null export 1 instruction before endpgm */
 567       block.instructions.insert(block.instructions.end() - 1, std::move(exp));
 568    }
 569 }
 570
 571 void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
 572 {
 573    for (std::pair<int, SOPP_instruction*> branch : ctx.branches)
 574    {
 575       int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
 576       out[branch.first] |= (uint16_t) offset;
 577    }
 578 }
 579
 580 void fix_constaddrs(asm_context& ctx, std::vector<uint32_t>& out)
 581 {
 582    for (unsigned addr : ctx.constaddrs)
 583       out[addr] += out.size() * 4u;
 584 }
 585
 586 unsigned emit_program(Program* program,
 587                       std::vector<uint32_t>& code)
 588 {
 589    asm_context ctx(program);
 590
 591    if (program->stage & (hw_vs | hw_fs))
 592       fix_exports(ctx, code, program);
 593
 594    for (Block& block : program->blocks) {
 595       block.offset = code.size();
 596       emit_block(ctx, code, block);
 597    }
 598
 599    fix_branches(ctx, code);
 600    fix_constaddrs(ctx, code);
 601
 602    unsigned constant_data_offset = code.size() * sizeof(uint32_t);
 603    while (program->constant_data.size() % 4u)
 604       program->constant_data.push_back(0);
 605    /* Copy constant data */
 606    code.insert(code.end(), (uint32_t*)program->constant_data.data(),
 607                (uint32_t*)(program->constant_data.data() + program->constant_data.size()));
 608
 609    return constant_data_offset;
 610 }
 611
 612 }