src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include <array>
  28 #include <map>
  29
  30 namespace aco {
  31
  32 #ifndef NDEBUG
  33 void perfwarn(bool cond, const char *msg, Instruction *instr)
  34 {
  35    if (cond) {
  36       fprintf(stderr, "ACO performance warning: %s\n", msg);
  37       if (instr) {
  38          fprintf(stderr, "instruction: ");
  39          aco_print_instr(instr, stderr);
  40          fprintf(stderr, "\n");
  41       }
  42
  43       if (debug_flags & DEBUG_PERFWARN)
  44          exit(1);
  45    }
  46 }
  47 #endif
  48
  49 void validate(Program* program, FILE * output)
  50 {
  51    if (!(debug_flags & DEBUG_VALIDATE))
  52       return;
  53
  54    bool is_valid = true;
  55    auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
  56       if (!check) {
  57          fprintf(output, "%s: ", msg);
  58          aco_print_instr(instr, output);
  59          fprintf(output, "\n");
  60          is_valid = false;
  61       }
  62    };
  63
  64    for (Block& block : program->blocks) {
  65       for (aco_ptr<Instruction>& instr : block.instructions) {
  66
  67          /* check base format */
  68          Format base_format = instr->format;
  69          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
  70          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
  71          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
  72             base_format = Format::VOP1;
  73          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
  74             base_format = Format::VOP2;
  75          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
  76             base_format = Format::VOPC;
  77          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP)
  78             base_format = Format::VINTRP;
  79          check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
  80
  81          /* check VOP3 modifiers */
  82          if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
  83             check(base_format == Format::VOP2 ||
  84                   base_format == Format::VOP1 ||
  85                   base_format == Format::VOPC ||
  86                   base_format == Format::VINTRP,
  87                   "Format cannot have VOP3A/VOP3B applied", instr.get());
  88          }
  89
  90          /* check for undefs */
  91          for (unsigned i = 0; i < instr->operands.size(); i++) {
  92             if (instr->operands[i].isUndefined()) {
  93                bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
  94                bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
  95                                    instr->format == Format::PSEUDO_REDUCTION ||
  96                                    (flat && i == 1) || (instr->format == Format::MIMG && i == 2) ||
  97                                    ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 0);
  98                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
  99             }
 100          }
 101
 102          /* check num literals */
 103          if (instr->isSALU() || instr->isVALU()) {
 104             unsigned num_literals = 0;
 105             for (unsigned i = 0; i < instr->operands.size(); i++)
 106             {
 107                if (instr->operands[i].isLiteral()) {
 108                   check(instr->format == Format::SOP1 ||
 109                         instr->format == Format::SOP2 ||
 110                         instr->format == Format::SOPC ||
 111                         instr->format == Format::VOP1 ||
 112                         instr->format == Format::VOP2 ||
 113                         instr->format == Format::VOPC,
 114                         "Literal applied on wrong instruction format", instr.get());
 115
 116                   num_literals++;
 117                   check(!instr->isVALU() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
 118                }
 119             }
 120             check(num_literals <= 1, "Only 1 Literal allowed", instr.get());
 121
 122             /* check num sgprs for VALU */
 123             if (instr->isVALU()) {
 124                check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
 125                      (int) instr->format & (int) Format::VOPC ||
 126                      instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 127                      instr->opcode == aco_opcode::v_readlane_b32,
 128                      "Wrong Definition type for VALU instruction", instr.get());
 129                unsigned num_sgpr = 0;
 130                unsigned sgpr_idx = instr->operands.size();
 131                for (unsigned i = 0; i < instr->operands.size(); i++)
 132                {
 133                   if (instr->operands[i].isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
 134                      check(i != 1 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for SGPR argument", instr.get());
 135
 136                      if (sgpr_idx == instr->operands.size() || instr->operands[sgpr_idx].tempId() != instr->operands[i].tempId())
 137                         num_sgpr++;
 138                      sgpr_idx = i;
 139                   }
 140
 141                   if (instr->operands[i].isConstant() && !instr->operands[i].isLiteral())
 142                      check(i == 0 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for constant argument", instr.get());
 143                }
 144                check(num_sgpr + num_literals <= 1, "Only 1 Literal OR 1 SGPR allowed", instr.get());
 145             }
 146
 147             if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
 148                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
 149                for (const Operand& op : instr->operands) {
 150                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
 151                        "Wrong Operand type for SALU instruction", instr.get());
 152             }
 153          }
 154          }
 155
 156          switch (instr->format) {
 157          case Format::PSEUDO: {
 158             if (instr->opcode == aco_opcode::p_create_vector) {
 159                unsigned size = 0;
 160                for (const Operand& op : instr->operands) {
 161                   size += op.size();
 162                }
 163                check(size == instr->definitions[0].size(), "Definition size does not match operand sizes", instr.get());
 164                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
 165                   for (const Operand& op : instr->operands) {
 166                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 167                            "Wrong Operand type for scalar vector", instr.get());
 168                   }
 169                }
 170             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 171                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
 172                check(instr->operands[1].constantValue() < instr->operands[0].size(), "Index out of range", instr.get());
 173                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
 174                      "Cannot extract SGPR value from VGPR vector", instr.get());
 175             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 176                check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
 177                for (unsigned i = 0; i < instr->operands.size(); i++) {
 178                   if (instr->operands[i].isTemp())
 179                      check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
 180                            (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
 181                            "Operand and Definition types do not match", instr.get());
 182                }
 183             } else if (instr->opcode == aco_opcode::p_phi) {
 184                check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 185                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == s2, "Logical Phi Definition must be vgpr or divergent boolean", instr.get());
 186             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 187                for (const Operand& op : instr->operands)
 188                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
 189                check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 190             }
 191             break;
 192          }
 193          case Format::SMEM: {
 194             if (instr->operands.size() >= 1)
 195                check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
 196             if (instr->operands.size() >= 2)
 197                check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
 198                      "SMEM offset must be constant or sgpr", instr.get());
 199             if (!instr->definitions.empty())
 200                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
 201             break;
 202          }
 203          case Format::MTBUF:
 204          case Format::MUBUF:
 205          case Format::MIMG: {
 206             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
 207             check(instr->operands[0].hasRegClass() && instr->operands[0].regClass().type() == RegType::vgpr,
 208                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 209             check(instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
 210             check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
 211             break;
 212          }
 213          case Format::DS: {
 214             for (const Operand& op : instr->operands) {
 215                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
 216                      "Only VGPRs are valid DS instruction operands", instr.get());
 217             }
 218             if (!instr->definitions.empty())
 219                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
 220             break;
 221          }
 222          case Format::EXP: {
 223             for (unsigned i = 0; i < 4; i++)
 224                check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
 225                      "Only VGPRs are valid Export arguments", instr.get());
 226             break;
 227          }
 228          case Format::FLAT:
 229             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
 230             /* fallthrough */
 231          case Format::GLOBAL:
 232          case Format::SCRATCH: {
 233             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
 234             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
 235                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 236             if (!instr->definitions.empty())
 237                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 238             else
 239                check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 240             break;
 241          }
 242          default:
 243             break;
 244          }
 245       }
 246    }
 247    assert(is_valid);
 248 }
 249
 250 /* RA validation */
 251 namespace {
 252
 253 struct Location {
 254    Location() : block(NULL), instr(NULL) {}
 255
 256    Block *block;
 257    Instruction *instr; //NULL if it's the block's live-in
 258 };
 259
 260 struct Assignment {
 261    Location defloc;
 262    Location firstloc;
 263    PhysReg reg;
 264 };
 265
 266 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
 267    va_list args;
 268    va_start(args, fmt);
 269    char msg[1024];
 270    vsprintf(msg, fmt, args);
 271    va_end(args);
 272
 273    fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
 274    if (loc.instr) {
 275       aco_print_instr(loc.instr, stderr);
 276       fprintf(stderr, "\n%s", msg);
 277    } else {
 278       fprintf(stderr, "%s", msg);
 279    }
 280    if (loc2.block) {
 281       fprintf(stderr, " in BB%d:\n", loc2.block->index);
 282       aco_print_instr(loc2.instr, stderr);
 283    }
 284    fprintf(stderr, "\n\n");
 285
 286    return true;
 287 }
 288
 289 } /* end namespace */
 290
 291 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
 292    if (!(debug_flags & DEBUG_VALIDATE_RA))
 293       return false;
 294
 295    bool err = false;
 296    aco::live live_vars = aco::live_var_analysis(program, options);
 297    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
 298
 299    std::map<unsigned, Assignment> assignments;
 300    for (Block& block : program->blocks) {
 301       Location loc;
 302       loc.block = &block;
 303       for (aco_ptr<Instruction>& instr : block.instructions) {
 304          if (instr->opcode == aco_opcode::p_phi) {
 305             for (unsigned i = 0; i < instr->operands.size(); i++) {
 306                if (instr->operands[i].isTemp() &&
 307                    instr->operands[i].getTemp().type() == RegType::sgpr &&
 308                    instr->operands[i].isFirstKill())
 309                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
 310             }
 311          }
 312
 313          loc.instr = instr.get();
 314          for (unsigned i = 0; i < instr->operands.size(); i++) {
 315             Operand& op = instr->operands[i];
 316             if (!op.isTemp())
 317                continue;
 318             if (!op.isFixed())
 319                err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
 320             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
 321                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
 322             if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) ||
 323                 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
 324                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
 325             if (!assignments[op.tempId()].firstloc.block)
 326                assignments[op.tempId()].firstloc = loc;
 327             if (!assignments[op.tempId()].defloc.block)
 328                assignments[op.tempId()].reg = op.physReg();
 329          }
 330
 331          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 332             Definition& def = instr->definitions[i];
 333             if (!def.isTemp())
 334                continue;
 335             if (!def.isFixed())
 336                err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
 337             if (assignments[def.tempId()].defloc.block)
 338                err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
 339             if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) ||
 340                 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
 341                err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
 342             if (!assignments[def.tempId()].firstloc.block)
 343                assignments[def.tempId()].firstloc = loc;
 344             assignments[def.tempId()].defloc = loc;
 345             assignments[def.tempId()].reg = def.physReg();
 346          }
 347       }
 348    }
 349
 350    for (Block& block : program->blocks) {
 351       Location loc;
 352       loc.block = &block;
 353
 354       std::array<unsigned, 512> regs;
 355       regs.fill(0);
 356
 357       std::set<Temp> live;
 358       live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
 359       /* remove killed p_phi sgpr operands */
 360       for (Temp tmp : phi_sgpr_ops[block.index])
 361          live.erase(tmp);
 362
 363       /* check live out */
 364       for (Temp tmp : live) {
 365          PhysReg reg = assignments.at(tmp.id()).reg;
 366          for (unsigned i = 0; i < tmp.size(); i++) {
 367             if (regs[reg + i]) {
 368                err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 369             }
 370             regs[reg + i] = tmp.id();
 371          }
 372       }
 373       regs.fill(0);
 374
 375       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
 376          aco_ptr<Instruction>& instr = *it;
 377
 378          /* check killed p_phi sgpr operands */
 379          if (instr->opcode == aco_opcode::p_logical_end) {
 380             for (Temp tmp : phi_sgpr_ops[block.index]) {
 381                PhysReg reg = assignments.at(tmp.id()).reg;
 382                for (unsigned i = 0; i < tmp.size(); i++) {
 383                   if (regs[reg + i])
 384                      err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 385                }
 386                live.emplace(tmp);
 387             }
 388          }
 389
 390          for (const Definition& def : instr->definitions) {
 391             if (!def.isTemp())
 392                continue;
 393             live.erase(def.getTemp());
 394          }
 395
 396          /* don't count phi operands as live-in, since they are actually
 397           * killed when they are copied at the predecessor */
 398          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 399             for (const Operand& op : instr->operands) {
 400                if (!op.isTemp())
 401                   continue;
 402                live.insert(op.getTemp());
 403             }
 404          }
 405       }
 406
 407       for (Temp tmp : live) {
 408          PhysReg reg = assignments.at(tmp.id()).reg;
 409          for (unsigned i = 0; i < tmp.size(); i++)
 410             regs[reg + i] = tmp.id();
 411       }
 412
 413       for (aco_ptr<Instruction>& instr : block.instructions) {
 414          loc.instr = instr.get();
 415
 416          /* remove killed p_phi operands from regs */
 417          if (instr->opcode == aco_opcode::p_logical_end) {
 418             for (Temp tmp : phi_sgpr_ops[block.index]) {
 419                PhysReg reg = assignments.at(tmp.id()).reg;
 420                regs[reg] = 0;
 421             }
 422          }
 423
 424          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 425             for (const Operand& op : instr->operands) {
 426                if (!op.isTemp())
 427                   continue;
 428                if (op.isFirstKill()) {
 429                   for (unsigned j = 0; j < op.getTemp().size(); j++)
 430                      regs[op.physReg() + j] = 0;
 431                }
 432             }
 433          }
 434
 435          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 436             Definition& def = instr->definitions[i];
 437             if (!def.isTemp())
 438                continue;
 439             Temp tmp = def.getTemp();
 440             PhysReg reg = assignments.at(tmp.id()).reg;
 441             for (unsigned j = 0; j < tmp.size(); j++) {
 442                if (regs[reg + j])
 443                   err |= ra_fail(output, loc, assignments.at(regs[reg + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg + j]);
 444                regs[reg + j] = tmp.id();
 445             }
 446          }
 447
 448          for (const Definition& def : instr->definitions) {
 449             if (!def.isTemp())
 450                continue;
 451             if (def.isKill()) {
 452                for (unsigned j = 0; j < def.getTemp().size(); j++)
 453                   regs[def.physReg() + j] = 0;
 454             }
 455          }
 456       }
 457    }
 458
 459    return err;
 460 }
 461 }