src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include <map>
  28
  29 namespace aco {
  30
  31 #ifndef NDEBUG
  32 void perfwarn(bool cond, const char *msg, Instruction *instr)
  33 {
  34    if (cond) {
  35       fprintf(stderr, "ACO performance warning: %s\n", msg);
  36       if (instr) {
  37          fprintf(stderr, "instruction: ");
  38          aco_print_instr(instr, stderr);
  39          fprintf(stderr, "\n");
  40       }
  41
  42       if (debug_flags & DEBUG_PERFWARN)
  43          exit(1);
  44    }
  45 }
  46 #endif
  47
  48 void validate(Program* program, FILE * output)
  49 {
  50    if (!(debug_flags & DEBUG_VALIDATE))
  51       return;
  52
  53    bool is_valid = true;
  54    auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
  55       if (!check) {
  56          fprintf(output, "%s: ", msg);
  57          aco_print_instr(instr, output);
  58          fprintf(output, "\n");
  59          is_valid = false;
  60       }
  61    };
  62
  63    for (Block& block : program->blocks) {
  64       for (aco_ptr<Instruction>& instr : block.instructions) {
  65
  66          /* check base format */
  67          Format base_format = instr->format;
  68          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
  69          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
  70          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
  71             base_format = Format::VOP1;
  72          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
  73             base_format = Format::VOP2;
  74          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
  75             base_format = Format::VOPC;
  76          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP)
  77             base_format = Format::VINTRP;
  78          check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
  79
  80          /* check VOP3 modifiers */
  81          if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
  82             check(base_format == Format::VOP2 ||
  83                   base_format == Format::VOP1 ||
  84                   base_format == Format::VOPC ||
  85                   base_format == Format::VINTRP,
  86                   "Format cannot have VOP3A/VOP3B applied", instr.get());
  87          }
  88
  89          /* check for undefs */
  90          for (unsigned i = 0; i < instr->operands.size(); i++) {
  91             if (instr->operands[i].isUndefined()) {
  92                bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
  93                bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
  94                                    instr->format == Format::PSEUDO_REDUCTION ||
  95                                    (flat && i == 1) || (instr->format == Format::MIMG && i == 2) ||
  96                                    ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 0);
  97                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
  98             }
  99          }
 100
 101          /* check num literals */
 102          if (instr->isSALU() || instr->isVALU()) {
 103             unsigned num_literals = 0;
 104             for (unsigned i = 0; i < instr->operands.size(); i++)
 105             {
 106                if (instr->operands[i].isLiteral()) {
 107                   check(instr->format == Format::SOP1 ||
 108                         instr->format == Format::SOP2 ||
 109                         instr->format == Format::SOPC ||
 110                         instr->format == Format::VOP1 ||
 111                         instr->format == Format::VOP2 ||
 112                         instr->format == Format::VOPC,
 113                         "Literal applied on wrong instruction format", instr.get());
 114
 115                   num_literals++;
 116                   check(!instr->isVALU() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
 117                }
 118             }
 119             check(num_literals <= 1, "Only 1 Literal allowed", instr.get());
 120
 121             /* check num sgprs for VALU */
 122             if (instr->isVALU()) {
 123                check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
 124                      (int) instr->format & (int) Format::VOPC ||
 125                      instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 126                      instr->opcode == aco_opcode::v_readlane_b32,
 127                      "Wrong Definition type for VALU instruction", instr.get());
 128                unsigned num_sgpr = 0;
 129                unsigned sgpr_idx = instr->operands.size();
 130                for (unsigned i = 0; i < instr->operands.size(); i++)
 131                {
 132                   if (instr->operands[i].isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
 133                      check(i != 1 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for SGPR argument", instr.get());
 134
 135                      if (sgpr_idx == instr->operands.size() || instr->operands[sgpr_idx].tempId() != instr->operands[i].tempId())
 136                         num_sgpr++;
 137                      sgpr_idx = i;
 138                   }
 139
 140                   if (instr->operands[i].isConstant() && !instr->operands[i].isLiteral())
 141                      check(i == 0 || (int) instr->format & (int) Format::VOP3A, "Wrong source position for constant argument", instr.get());
 142                }
 143                check(num_sgpr + num_literals <= 1, "Only 1 Literal OR 1 SGPR allowed", instr.get());
 144             }
 145
 146             if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
 147                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
 148                for (const Operand& op : instr->operands) {
 149                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
 150                        "Wrong Operand type for SALU instruction", instr.get());
 151             }
 152          }
 153          }
 154
 155          switch (instr->format) {
 156          case Format::PSEUDO: {
 157             if (instr->opcode == aco_opcode::p_create_vector) {
 158                unsigned size = 0;
 159                for (const Operand& op : instr->operands) {
 160                   size += op.size();
 161                }
 162                check(size == instr->definitions[0].size(), "Definition size does not match operand sizes", instr.get());
 163                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
 164                   for (const Operand& op : instr->operands) {
 165                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 166                            "Wrong Operand type for scalar vector", instr.get());
 167                   }
 168                }
 169             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 170                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
 171                check(instr->operands[1].constantValue() < instr->operands[0].size(), "Index out of range", instr.get());
 172                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
 173                      "Cannot extract SGPR value from VGPR vector", instr.get());
 174             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 175                check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
 176                for (unsigned i = 0; i < instr->operands.size(); i++) {
 177                   if (instr->operands[i].isTemp())
 178                      check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
 179                            (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
 180                            "Operand and Definition types do not match", instr.get());
 181                }
 182             } else if (instr->opcode == aco_opcode::p_phi) {
 183                check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 184                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == s2, "Logical Phi Definition must be vgpr or divergent boolean", instr.get());
 185             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 186                for (const Operand& op : instr->operands)
 187                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
 188                check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 189             }
 190             break;
 191          }
 192          case Format::SMEM: {
 193             if (instr->operands.size() >= 1)
 194                check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
 195             if (instr->operands.size() >= 2)
 196                check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
 197                      "SMEM offset must be constant or sgpr", instr.get());
 198             if (!instr->definitions.empty())
 199                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
 200             break;
 201          }
 202          case Format::MTBUF:
 203          case Format::MUBUF:
 204          case Format::MIMG: {
 205             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
 206             check(instr->operands[0].hasRegClass() && instr->operands[0].regClass().type() == RegType::vgpr,
 207                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 208             check(instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
 209             check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
 210             break;
 211          }
 212          case Format::DS: {
 213             for (const Operand& op : instr->operands) {
 214                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
 215                      "Only VGPRs are valid DS instruction operands", instr.get());
 216             }
 217             if (!instr->definitions.empty())
 218                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
 219             break;
 220          }
 221          case Format::EXP: {
 222             for (unsigned i = 0; i < 4; i++)
 223                check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
 224                      "Only VGPRs are valid Export arguments", instr.get());
 225             break;
 226          }
 227          case Format::FLAT:
 228             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
 229             /* fallthrough */
 230          case Format::GLOBAL:
 231          case Format::SCRATCH: {
 232             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
 233             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
 234                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 235             if (!instr->definitions.empty())
 236                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 237             else
 238                check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 239             break;
 240          }
 241          default:
 242             break;
 243          }
 244       }
 245    }
 246    assert(is_valid);
 247 }
 248
 249 /* RA validation */
 250 namespace {
 251
 252 struct Location {
 253    Location() : block(NULL), instr(NULL) {}
 254
 255    Block *block;
 256    Instruction *instr; //NULL if it's the block's live-in
 257 };
 258
 259 struct Assignment {
 260    Location defloc;
 261    Location firstloc;
 262    PhysReg reg;
 263 };
 264
 265 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
 266    va_list args;
 267    va_start(args, fmt);
 268    char msg[1024];
 269    vsprintf(msg, fmt, args);
 270    va_end(args);
 271
 272    fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
 273    if (loc.instr) {
 274       aco_print_instr(loc.instr, stderr);
 275       fprintf(stderr, "\n%s", msg);
 276    } else {
 277       fprintf(stderr, "%s", msg);
 278    }
 279    if (loc2.block) {
 280       fprintf(stderr, " in BB%d:\n", loc2.block->index);
 281       aco_print_instr(loc2.instr, stderr);
 282    }
 283    fprintf(stderr, "\n\n");
 284
 285    return true;
 286 }
 287
 288 } /* end namespace */
 289
 290 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
 291    if (!(debug_flags & DEBUG_VALIDATE_RA))
 292       return false;
 293
 294    bool err = false;
 295    aco::live live_vars = aco::live_var_analysis(program, options);
 296    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
 297
 298    std::map<unsigned, Assignment> assignments;
 299    for (Block& block : program->blocks) {
 300       Location loc;
 301       loc.block = &block;
 302       for (aco_ptr<Instruction>& instr : block.instructions) {
 303          if (instr->opcode == aco_opcode::p_phi) {
 304             for (unsigned i = 0; i < instr->operands.size(); i++) {
 305                if (instr->operands[i].isTemp() &&
 306                    instr->operands[i].getTemp().type() == RegType::sgpr &&
 307                    instr->operands[i].isFirstKill())
 308                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
 309             }
 310          }
 311
 312          loc.instr = instr.get();
 313          for (unsigned i = 0; i < instr->operands.size(); i++) {
 314             Operand& op = instr->operands[i];
 315             if (!op.isTemp())
 316                continue;
 317             if (!op.isFixed())
 318                err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
 319             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
 320                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
 321             if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) ||
 322                 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
 323                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
 324             if (!assignments[op.tempId()].firstloc.block)
 325                assignments[op.tempId()].firstloc = loc;
 326             if (!assignments[op.tempId()].defloc.block)
 327                assignments[op.tempId()].reg = op.physReg();
 328          }
 329
 330          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 331             Definition& def = instr->definitions[i];
 332             if (!def.isTemp())
 333                continue;
 334             if (!def.isFixed())
 335                err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
 336             if (assignments[def.tempId()].defloc.block)
 337                err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
 338             if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) ||
 339                 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
 340                err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
 341             if (!assignments[def.tempId()].firstloc.block)
 342                assignments[def.tempId()].firstloc = loc;
 343             assignments[def.tempId()].defloc = loc;
 344             assignments[def.tempId()].reg = def.physReg();
 345          }
 346       }
 347    }
 348
 349    for (Block& block : program->blocks) {
 350       Location loc;
 351       loc.block = &block;
 352
 353       std::array<unsigned, 512> regs;
 354       regs.fill(0);
 355
 356       std::set<Temp> live;
 357       live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
 358       /* remove killed p_phi sgpr operands */
 359       for (Temp tmp : phi_sgpr_ops[block.index])
 360          live.erase(tmp);
 361
 362       /* check live out */
 363       for (Temp tmp : live) {
 364          PhysReg reg = assignments.at(tmp.id()).reg;
 365          for (unsigned i = 0; i < tmp.size(); i++) {
 366             if (regs[reg + i]) {
 367                err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 368             }
 369             regs[reg + i] = tmp.id();
 370          }
 371       }
 372       regs.fill(0);
 373
 374       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
 375          aco_ptr<Instruction>& instr = *it;
 376
 377          /* check killed p_phi sgpr operands */
 378          if (instr->opcode == aco_opcode::p_logical_end) {
 379             for (Temp tmp : phi_sgpr_ops[block.index]) {
 380                PhysReg reg = assignments.at(tmp.id()).reg;
 381                for (unsigned i = 0; i < tmp.size(); i++) {
 382                   if (regs[reg + i])
 383                      err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 384                }
 385                live.emplace(tmp);
 386             }
 387          }
 388
 389          for (const Definition& def : instr->definitions) {
 390             if (!def.isTemp())
 391                continue;
 392             live.erase(def.getTemp());
 393          }
 394
 395          /* don't count phi operands as live-in, since they are actually
 396           * killed when they are copied at the predecessor */
 397          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 398             for (const Operand& op : instr->operands) {
 399                if (!op.isTemp())
 400                   continue;
 401                live.insert(op.getTemp());
 402             }
 403          }
 404       }
 405
 406       for (Temp tmp : live) {
 407          PhysReg reg = assignments.at(tmp.id()).reg;
 408          for (unsigned i = 0; i < tmp.size(); i++)
 409             regs[reg + i] = tmp.id();
 410       }
 411
 412       for (aco_ptr<Instruction>& instr : block.instructions) {
 413          loc.instr = instr.get();
 414
 415          /* remove killed p_phi operands from regs */
 416          if (instr->opcode == aco_opcode::p_logical_end) {
 417             for (Temp tmp : phi_sgpr_ops[block.index]) {
 418                PhysReg reg = assignments.at(tmp.id()).reg;
 419                regs[reg] = 0;
 420             }
 421          }
 422
 423          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 424             for (const Operand& op : instr->operands) {
 425                if (!op.isTemp())
 426                   continue;
 427                if (op.isFirstKill()) {
 428                   for (unsigned j = 0; j < op.getTemp().size(); j++)
 429                      regs[op.physReg() + j] = 0;
 430                }
 431             }
 432          }
 433
 434          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 435             Definition& def = instr->definitions[i];
 436             if (!def.isTemp())
 437                continue;
 438             Temp tmp = def.getTemp();
 439             PhysReg reg = assignments.at(tmp.id()).reg;
 440             for (unsigned j = 0; j < tmp.size(); j++) {
 441                if (regs[reg + j])
 442                   err |= ra_fail(output, loc, assignments.at(regs[reg + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg + j]);
 443                regs[reg + j] = tmp.id();
 444             }
 445          }
 446
 447          for (const Definition& def : instr->definitions) {
 448             if (!def.isTemp())
 449                continue;
 450             if (def.isKill()) {
 451                for (unsigned j = 0; j < def.getTemp().size(); j++)
 452                   regs[def.physReg() + j] = 0;
 453             }
 454          }
 455       }
 456    }
 457
 458    return err;
 459 }
 460 }