src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include <array>
  28 #include <map>
  29
  30 namespace aco {
  31
  32 #ifndef NDEBUG
  33 void perfwarn(bool cond, const char *msg, Instruction *instr)
  34 {
  35    if (cond) {
  36       fprintf(stderr, "ACO performance warning: %s\n", msg);
  37       if (instr) {
  38          fprintf(stderr, "instruction: ");
  39          aco_print_instr(instr, stderr);
  40          fprintf(stderr, "\n");
  41       }
  42
  43       if (debug_flags & DEBUG_PERFWARN)
  44          exit(1);
  45    }
  46 }
  47 #endif
  48
  49 void validate(Program* program, FILE * output)
  50 {
  51    if (!(debug_flags & DEBUG_VALIDATE))
  52       return;
  53
  54    bool is_valid = true;
  55    auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
  56       if (!check) {
  57          fprintf(output, "%s: ", msg);
  58          aco_print_instr(instr, output);
  59          fprintf(output, "\n");
  60          is_valid = false;
  61       }
  62    };
  63    auto check_block = [&output, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
  64       if (!check) {
  65          fprintf(output, "%s: BB%u\n", msg, block->index);
  66          is_valid = false;
  67       }
  68    };
  69
  70    for (Block& block : program->blocks) {
  71       for (aco_ptr<Instruction>& instr : block.instructions) {
  72
  73          /* check base format */
  74          Format base_format = instr->format;
  75          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
  76          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
  77          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
  78             base_format = Format::VOP1;
  79          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
  80             base_format = Format::VOP2;
  81          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
  82             base_format = Format::VOPC;
  83          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP)
  84             base_format = Format::VINTRP;
  85          check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
  86
  87          /* check VOP3 modifiers */
  88          if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
  89             check(base_format == Format::VOP2 ||
  90                   base_format == Format::VOP1 ||
  91                   base_format == Format::VOPC ||
  92                   base_format == Format::VINTRP,
  93                   "Format cannot have VOP3A/VOP3B applied", instr.get());
  94          }
  95
  96          /* check for undefs */
  97          for (unsigned i = 0; i < instr->operands.size(); i++) {
  98             if (instr->operands[i].isUndefined()) {
  99                bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
 100                bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
 101                                    instr->format == Format::PSEUDO_REDUCTION ||
 102                                    (flat && i == 1) || (instr->format == Format::MIMG && i == 2) ||
 103                                    ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 0);
 104                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
 105             }
 106          }
 107
 108          /* check num literals */
 109          if (instr->isSALU() || instr->isVALU()) {
 110             unsigned num_literals = 0;
 111             for (unsigned i = 0; i < instr->operands.size(); i++)
 112             {
 113                if (instr->operands[i].isLiteral() && instr->isVOP3() && program->chip_class >= GFX10) {
 114                   num_literals++;
 115                } else if (instr->operands[i].isLiteral()) {
 116                   check(instr->format == Format::SOP1 ||
 117                         instr->format == Format::SOP2 ||
 118                         instr->format == Format::SOPC ||
 119                         instr->format == Format::VOP1 ||
 120                         instr->format == Format::VOP2 ||
 121                         instr->format == Format::VOPC,
 122                         "Literal applied on wrong instruction format", instr.get());
 123
 124                   num_literals++;
 125                   check(!instr->isVALU() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
 126                }
 127             }
 128             check(num_literals <= 1, "Only 1 Literal allowed", instr.get());
 129
 130             /* check num sgprs for VALU */
 131             if (instr->isVALU()) {
 132                check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
 133                      (int) instr->format & (int) Format::VOPC ||
 134                      instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 135                      instr->opcode == aco_opcode::v_readlane_b32,
 136                      "Wrong Definition type for VALU instruction", instr.get());
 137                unsigned num_sgpr = 0;
 138                unsigned sgpr_idx = instr->operands.size();
 139                for (unsigned i = 0; i < instr->operands.size(); i++)
 140                {
 141                   if (instr->operands[i].isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
 142                      check(i != 1 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for SGPR argument", instr.get());
 143
 144                      if (sgpr_idx == instr->operands.size() || instr->operands[sgpr_idx].tempId() != instr->operands[i].tempId())
 145                         num_sgpr++;
 146                      sgpr_idx = i;
 147                   }
 148
 149                   if (instr->operands[i].isConstant() && !instr->operands[i].isLiteral())
 150                      check(i == 0 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for constant argument", instr.get());
 151                }
 152                check(num_sgpr + num_literals <= 1, "Only 1 Literal OR 1 SGPR allowed", instr.get());
 153             }
 154
 155             if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
 156                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
 157                for (const Operand& op : instr->operands) {
 158                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
 159                        "Wrong Operand type for SALU instruction", instr.get());
 160             }
 161          }
 162          }
 163
 164          switch (instr->format) {
 165          case Format::PSEUDO: {
 166             if (instr->opcode == aco_opcode::p_create_vector) {
 167                unsigned size = 0;
 168                for (const Operand& op : instr->operands) {
 169                   size += op.size();
 170                }
 171                check(size == instr->definitions[0].size(), "Definition size does not match operand sizes", instr.get());
 172                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
 173                   for (const Operand& op : instr->operands) {
 174                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 175                            "Wrong Operand type for scalar vector", instr.get());
 176                   }
 177                }
 178             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 179                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
 180                check(instr->operands[1].constantValue() < instr->operands[0].size(), "Index out of range", instr.get());
 181                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
 182                      "Cannot extract SGPR value from VGPR vector", instr.get());
 183             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 184                check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
 185                for (unsigned i = 0; i < instr->operands.size(); i++) {
 186                   if (instr->operands[i].isTemp())
 187                      check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
 188                            (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
 189                            "Operand and Definition types do not match", instr.get());
 190                }
 191             } else if (instr->opcode == aco_opcode::p_phi) {
 192                check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 193                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == s2, "Logical Phi Definition must be vgpr or divergent boolean", instr.get());
 194             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 195                for (const Operand& op : instr->operands)
 196                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
 197                check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 198             }
 199             break;
 200          }
 201          case Format::SMEM: {
 202             if (instr->operands.size() >= 1)
 203                check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
 204             if (instr->operands.size() >= 2)
 205                check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
 206                      "SMEM offset must be constant or sgpr", instr.get());
 207             if (!instr->definitions.empty())
 208                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
 209             break;
 210          }
 211          case Format::MTBUF:
 212          case Format::MUBUF:
 213          case Format::MIMG: {
 214             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
 215             check(instr->operands[0].hasRegClass() && instr->operands[0].regClass().type() == RegType::vgpr,
 216                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 217             check(instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
 218             check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
 219             break;
 220          }
 221          case Format::DS: {
 222             for (const Operand& op : instr->operands) {
 223                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
 224                      "Only VGPRs are valid DS instruction operands", instr.get());
 225             }
 226             if (!instr->definitions.empty())
 227                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
 228             break;
 229          }
 230          case Format::EXP: {
 231             for (unsigned i = 0; i < 4; i++)
 232                check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
 233                      "Only VGPRs are valid Export arguments", instr.get());
 234             break;
 235          }
 236          case Format::FLAT:
 237             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
 238             /* fallthrough */
 239          case Format::GLOBAL:
 240          case Format::SCRATCH: {
 241             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
 242             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
 243                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 244             if (!instr->definitions.empty())
 245                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 246             else
 247                check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 248             break;
 249          }
 250          default:
 251             break;
 252          }
 253       }
 254    }
 255
 256    /* validate CFG */
 257    for (unsigned i = 0; i < program->blocks.size(); i++) {
 258       Block& block = program->blocks[i];
 259       check_block(block.index == i, "block.index must match actual index", &block);
 260
 261       /* predecessors/successors should be sorted */
 262       for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
 263          check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
 264       for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
 265          check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
 266       for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
 267          check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
 268       for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
 269          check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
 270
 271       /* critical edges are not allowed */
 272       if (block.linear_preds.size() > 1) {
 273          for (unsigned pred : block.linear_preds)
 274             check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
 275          for (unsigned pred : block.logical_preds)
 276             check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
 277       }
 278    }
 279
 280    assert(is_valid);
 281 }
 282
 283 /* RA validation */
 284 namespace {
 285
 286 struct Location {
 287    Location() : block(NULL), instr(NULL) {}
 288
 289    Block *block;
 290    Instruction *instr; //NULL if it's the block's live-in
 291 };
 292
 293 struct Assignment {
 294    Location defloc;
 295    Location firstloc;
 296    PhysReg reg;
 297 };
 298
 299 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
 300    va_list args;
 301    va_start(args, fmt);
 302    char msg[1024];
 303    vsprintf(msg, fmt, args);
 304    va_end(args);
 305
 306    fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
 307    if (loc.instr) {
 308       aco_print_instr(loc.instr, stderr);
 309       fprintf(stderr, "\n%s", msg);
 310    } else {
 311       fprintf(stderr, "%s", msg);
 312    }
 313    if (loc2.block) {
 314       fprintf(stderr, " in BB%d:\n", loc2.block->index);
 315       aco_print_instr(loc2.instr, stderr);
 316    }
 317    fprintf(stderr, "\n\n");
 318
 319    return true;
 320 }
 321
 322 } /* end namespace */
 323
 324 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
 325    if (!(debug_flags & DEBUG_VALIDATE_RA))
 326       return false;
 327
 328    bool err = false;
 329    aco::live live_vars = aco::live_var_analysis(program, options);
 330    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
 331
 332    std::map<unsigned, Assignment> assignments;
 333    for (Block& block : program->blocks) {
 334       Location loc;
 335       loc.block = &block;
 336       for (aco_ptr<Instruction>& instr : block.instructions) {
 337          if (instr->opcode == aco_opcode::p_phi) {
 338             for (unsigned i = 0; i < instr->operands.size(); i++) {
 339                if (instr->operands[i].isTemp() &&
 340                    instr->operands[i].getTemp().type() == RegType::sgpr &&
 341                    instr->operands[i].isFirstKill())
 342                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
 343             }
 344          }
 345
 346          loc.instr = instr.get();
 347          for (unsigned i = 0; i < instr->operands.size(); i++) {
 348             Operand& op = instr->operands[i];
 349             if (!op.isTemp())
 350                continue;
 351             if (!op.isFixed())
 352                err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
 353             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
 354                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
 355             if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) ||
 356                 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
 357                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
 358             if (!assignments[op.tempId()].firstloc.block)
 359                assignments[op.tempId()].firstloc = loc;
 360             if (!assignments[op.tempId()].defloc.block)
 361                assignments[op.tempId()].reg = op.physReg();
 362          }
 363
 364          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 365             Definition& def = instr->definitions[i];
 366             if (!def.isTemp())
 367                continue;
 368             if (!def.isFixed())
 369                err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
 370             if (assignments[def.tempId()].defloc.block)
 371                err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
 372             if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) ||
 373                 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
 374                err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
 375             if (!assignments[def.tempId()].firstloc.block)
 376                assignments[def.tempId()].firstloc = loc;
 377             assignments[def.tempId()].defloc = loc;
 378             assignments[def.tempId()].reg = def.physReg();
 379          }
 380       }
 381    }
 382
 383    for (Block& block : program->blocks) {
 384       Location loc;
 385       loc.block = &block;
 386
 387       std::array<unsigned, 512> regs;
 388       regs.fill(0);
 389
 390       std::set<Temp> live;
 391       live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
 392       /* remove killed p_phi sgpr operands */
 393       for (Temp tmp : phi_sgpr_ops[block.index])
 394          live.erase(tmp);
 395
 396       /* check live out */
 397       for (Temp tmp : live) {
 398          PhysReg reg = assignments.at(tmp.id()).reg;
 399          for (unsigned i = 0; i < tmp.size(); i++) {
 400             if (regs[reg + i]) {
 401                err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 402             }
 403             regs[reg + i] = tmp.id();
 404          }
 405       }
 406       regs.fill(0);
 407
 408       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
 409          aco_ptr<Instruction>& instr = *it;
 410
 411          /* check killed p_phi sgpr operands */
 412          if (instr->opcode == aco_opcode::p_logical_end) {
 413             for (Temp tmp : phi_sgpr_ops[block.index]) {
 414                PhysReg reg = assignments.at(tmp.id()).reg;
 415                for (unsigned i = 0; i < tmp.size(); i++) {
 416                   if (regs[reg + i])
 417                      err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 418                }
 419                live.emplace(tmp);
 420             }
 421          }
 422
 423          for (const Definition& def : instr->definitions) {
 424             if (!def.isTemp())
 425                continue;
 426             live.erase(def.getTemp());
 427          }
 428
 429          /* don't count phi operands as live-in, since they are actually
 430           * killed when they are copied at the predecessor */
 431          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 432             for (const Operand& op : instr->operands) {
 433                if (!op.isTemp())
 434                   continue;
 435                live.insert(op.getTemp());
 436             }
 437          }
 438       }
 439
 440       for (Temp tmp : live) {
 441          PhysReg reg = assignments.at(tmp.id()).reg;
 442          for (unsigned i = 0; i < tmp.size(); i++)
 443             regs[reg + i] = tmp.id();
 444       }
 445
 446       for (aco_ptr<Instruction>& instr : block.instructions) {
 447          loc.instr = instr.get();
 448
 449          /* remove killed p_phi operands from regs */
 450          if (instr->opcode == aco_opcode::p_logical_end) {
 451             for (Temp tmp : phi_sgpr_ops[block.index]) {
 452                PhysReg reg = assignments.at(tmp.id()).reg;
 453                regs[reg] = 0;
 454             }
 455          }
 456
 457          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 458             for (const Operand& op : instr->operands) {
 459                if (!op.isTemp())
 460                   continue;
 461                if (op.isFirstKill()) {
 462                   for (unsigned j = 0; j < op.getTemp().size(); j++)
 463                      regs[op.physReg() + j] = 0;
 464                }
 465             }
 466          }
 467
 468          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 469             Definition& def = instr->definitions[i];
 470             if (!def.isTemp())
 471                continue;
 472             Temp tmp = def.getTemp();
 473             PhysReg reg = assignments.at(tmp.id()).reg;
 474             for (unsigned j = 0; j < tmp.size(); j++) {
 475                if (regs[reg + j])
 476                   err |= ra_fail(output, loc, assignments.at(regs[reg + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg + j]);
 477                regs[reg + j] = tmp.id();
 478             }
 479          }
 480
 481          for (const Definition& def : instr->definitions) {
 482             if (!def.isTemp())
 483                continue;
 484             if (def.isKill()) {
 485                for (unsigned j = 0; j < def.getTemp().size(); j++)
 486                   regs[def.physReg() + j] = 0;
 487             }
 488          }
 489       }
 490    }
 491
 492    return err;
 493 }
 494 }