src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include <array>
  28 #include <map>
  29
  30 namespace aco {
  31
  32 #ifndef NDEBUG
  33 void perfwarn(bool cond, const char *msg, Instruction *instr)
  34 {
  35    if (cond) {
  36       fprintf(stderr, "ACO performance warning: %s\n", msg);
  37       if (instr) {
  38          fprintf(stderr, "instruction: ");
  39          aco_print_instr(instr, stderr);
  40          fprintf(stderr, "\n");
  41       }
  42
  43       if (debug_flags & DEBUG_PERFWARN)
  44          exit(1);
  45    }
  46 }
  47 #endif
  48
  49 void validate(Program* program, FILE * output)
  50 {
  51    if (!(debug_flags & DEBUG_VALIDATE))
  52       return;
  53
  54    bool is_valid = true;
  55    auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
  56       if (!check) {
  57          fprintf(output, "%s: ", msg);
  58          aco_print_instr(instr, output);
  59          fprintf(output, "\n");
  60          is_valid = false;
  61       }
  62    };
  63    auto check_block = [&output, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
  64       if (!check) {
  65          fprintf(output, "%s: BB%u\n", msg, block->index);
  66          is_valid = false;
  67       }
  68    };
  69
  70    for (Block& block : program->blocks) {
  71       for (aco_ptr<Instruction>& instr : block.instructions) {
  72
  73          /* check base format */
  74          Format base_format = instr->format;
  75          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
  76          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
  77          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
  78             base_format = Format::VOP1;
  79          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
  80             base_format = Format::VOP2;
  81          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
  82             base_format = Format::VOPC;
  83          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP)
  84             base_format = Format::VINTRP;
  85          check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
  86
  87          /* check VOP3 modifiers */
  88          if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
  89             check(base_format == Format::VOP2 ||
  90                   base_format == Format::VOP1 ||
  91                   base_format == Format::VOPC ||
  92                   base_format == Format::VINTRP,
  93                   "Format cannot have VOP3A/VOP3B applied", instr.get());
  94          }
  95
  96          /* check SDWA */
  97          if (instr->isSDWA()) {
  98             check(base_format == Format::VOP2 ||
  99                   base_format == Format::VOP1 ||
 100                   base_format == Format::VOPC,
 101                   "Format cannot have SDWA applied", instr.get());
 102
 103             check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
 104
 105             SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get());
 106             check(sdwa->omod == 0 || program->chip_class >= GFX9, "SDWA omod only supported on GFX9+", instr.get());
 107             if (base_format == Format::VOPC) {
 108                check(sdwa->clamp == false || program->chip_class == GFX8, "SDWA VOPC clamp only supported on GFX8", instr.get());
 109                check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
 110                      program->chip_class >= GFX9,
 111                      "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
 112             }
 113
 114             if (instr->operands.size() >= 3) {
 115                check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
 116                      "3rd operand must be fixed to vcc with SDWA", instr.get());
 117             }
 118             if (instr->definitions.size() >= 2) {
 119                check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
 120                      "2nd definition must be fixed to vcc with SDWA", instr.get());
 121             }
 122
 123             check(instr->opcode != aco_opcode::v_madmk_f32 &&
 124                   instr->opcode != aco_opcode::v_madak_f32 &&
 125                   instr->opcode != aco_opcode::v_madmk_f16 &&
 126                   instr->opcode != aco_opcode::v_madak_f16 &&
 127                   instr->opcode != aco_opcode::v_readfirstlane_b32 &&
 128                   instr->opcode != aco_opcode::v_clrexcp &&
 129                   instr->opcode != aco_opcode::v_swap_b32,
 130                   "SDWA can't be used with this opcode", instr.get());
 131             if (program->chip_class != GFX8) {
 132                check(instr->opcode != aco_opcode::v_mac_f32 &&
 133                      instr->opcode != aco_opcode::v_mac_f16 &&
 134                      instr->opcode != aco_opcode::v_fmac_f32 &&
 135                      instr->opcode != aco_opcode::v_fmac_f16,
 136                      "SDWA can't be used with this opcode", instr.get());
 137             }
 138          }
 139
 140          /* check for undefs */
 141          for (unsigned i = 0; i < instr->operands.size(); i++) {
 142             if (instr->operands[i].isUndefined()) {
 143                bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
 144                bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
 145                                    instr->format == Format::PSEUDO_REDUCTION ||
 146                                    (flat && i == 1) || (instr->format == Format::MIMG && i == 1) ||
 147                                    ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 1);
 148                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
 149             }
 150          }
 151
 152          if (instr->isSALU() || instr->isVALU()) {
 153             /* check literals */
 154             Operand literal(s1);
 155             for (unsigned i = 0; i < instr->operands.size(); i++)
 156             {
 157                Operand op = instr->operands[i];
 158                if (!op.isLiteral())
 159                   continue;
 160
 161                check(instr->format == Format::SOP1 ||
 162                      instr->format == Format::SOP2 ||
 163                      instr->format == Format::SOPC ||
 164                      instr->format == Format::VOP1 ||
 165                      instr->format == Format::VOP2 ||
 166                      instr->format == Format::VOPC ||
 167                      (instr->isVOP3() && program->chip_class >= GFX10),
 168                      "Literal applied on wrong instruction format", instr.get());
 169
 170                check(literal.isUndefined() || (literal.size() == op.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get());
 171                literal = op;
 172                check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
 173             }
 174
 175             /* check num sgprs for VALU */
 176             if (instr->isVALU()) {
 177                bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
 178                                  instr->opcode == aco_opcode::v_lshrrev_b64 ||
 179                                  instr->opcode == aco_opcode::v_ashrrev_i64;
 180                unsigned const_bus_limit = 1;
 181                if (program->chip_class >= GFX10 && !is_shift64)
 182                   const_bus_limit = 2;
 183
 184                uint32_t scalar_mask = instr->isVOP3() ? 0x7 : 0x5;
 185                if (instr->isSDWA())
 186                   scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
 187
 188                check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
 189                      (int) instr->format & (int) Format::VOPC ||
 190                      instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 191                      instr->opcode == aco_opcode::v_readlane_b32 ||
 192                      instr->opcode == aco_opcode::v_readlane_b32_e64,
 193                      "Wrong Definition type for VALU instruction", instr.get());
 194                unsigned num_sgprs = 0;
 195                unsigned sgpr[] = {0, 0};
 196                for (unsigned i = 0; i < instr->operands.size(); i++)
 197                {
 198                   Operand op = instr->operands[i];
 199                   if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 200                       instr->opcode == aco_opcode::v_readlane_b32 ||
 201                       instr->opcode == aco_opcode::v_readlane_b32_e64 ||
 202                       instr->opcode == aco_opcode::v_writelane_b32 ||
 203                       instr->opcode == aco_opcode::v_writelane_b32_e64) {
 204                      check(!op.isLiteral(), "No literal allowed on VALU instruction", instr.get());
 205                      check(i == 1 || (op.isTemp() && op.regClass() == v1), "Wrong Operand type for VALU instruction", instr.get());
 206                      continue;
 207                   }
 208                   if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
 209                      check(scalar_mask & (1 << i), "Wrong source position for SGPR argument", instr.get());
 210
 211                      if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
 212                         if (num_sgprs < 2)
 213                            sgpr[num_sgprs++] = op.tempId();
 214                      }
 215                   }
 216
 217                   if (op.isConstant() && !op.isLiteral())
 218                      check(scalar_mask & (1 << i), "Wrong source position for constant argument", instr.get());
 219                }
 220                check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get());
 221             }
 222
 223             if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
 224                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
 225                for (const Operand& op : instr->operands) {
 226                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
 227                        "Wrong Operand type for SALU instruction", instr.get());
 228             }
 229          }
 230          }
 231
 232          switch (instr->format) {
 233          case Format::PSEUDO: {
 234             if (instr->opcode == aco_opcode::p_create_vector) {
 235                unsigned size = 0;
 236                for (const Operand& op : instr->operands) {
 237                   size += op.size();
 238                }
 239                check(size == instr->definitions[0].size(), "Definition size does not match operand sizes", instr.get());
 240                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
 241                   for (const Operand& op : instr->operands) {
 242                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 243                            "Wrong Operand type for scalar vector", instr.get());
 244                   }
 245                }
 246             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 247                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
 248                check(instr->operands[1].constantValue() < instr->operands[0].size(), "Index out of range", instr.get());
 249                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
 250                      "Cannot extract SGPR value from VGPR vector", instr.get());
 251             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 252                check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
 253                for (unsigned i = 0; i < instr->operands.size(); i++) {
 254                   if (instr->operands[i].isTemp())
 255                      check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
 256                            (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
 257                            "Operand and Definition types do not match", instr.get());
 258                }
 259             } else if (instr->opcode == aco_opcode::p_phi) {
 260                check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 261                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == program->lane_mask, "Logical Phi Definition must be vgpr or divergent boolean", instr.get());
 262             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 263                for (const Operand& op : instr->operands)
 264                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
 265                check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 266             }
 267             break;
 268          }
 269          case Format::SMEM: {
 270             if (instr->operands.size() >= 1)
 271                check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
 272             if (instr->operands.size() >= 2)
 273                check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
 274                      "SMEM offset must be constant or sgpr", instr.get());
 275             if (!instr->definitions.empty())
 276                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
 277             break;
 278          }
 279          case Format::MTBUF:
 280          case Format::MUBUF: {
 281             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
 282             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr,
 283                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 284             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
 285             check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
 286             break;
 287          }
 288          case Format::MIMG: {
 289             check(instr->operands.size() == 3, "MIMG instructions must have exactly 3 operands", instr.get());
 290             check(instr->operands[0].hasRegClass() && (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
 291                   "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
 292             if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr)
 293                check(instr->operands[1].regClass() == s4, "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
 294             else if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr)
 295                check((instr->definitions.empty() || instr->definitions[0].regClass() == instr->operands[1].regClass() ||
 296                      instr->opcode == aco_opcode::image_atomic_cmpswap || instr->opcode == aco_opcode::image_atomic_fcmpswap),
 297                      "MIMG operands[1] (VDATA) must be the same as definitions[0] for atomics", instr.get());
 298             check(instr->operands[2].hasRegClass() && instr->operands[2].regClass().type() == RegType::vgpr,
 299                   "MIMG operands[2] (VADDR) must be VGPR", instr.get());
 300             check(instr->definitions.empty() || (instr->definitions[0].isTemp() && instr->definitions[0].regClass().type() == RegType::vgpr),
 301                   "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
 302             break;
 303          }
 304          case Format::DS: {
 305             for (const Operand& op : instr->operands) {
 306                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
 307                      "Only VGPRs are valid DS instruction operands", instr.get());
 308             }
 309             if (!instr->definitions.empty())
 310                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
 311             break;
 312          }
 313          case Format::EXP: {
 314             for (unsigned i = 0; i < 4; i++)
 315                check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
 316                      "Only VGPRs are valid Export arguments", instr.get());
 317             break;
 318          }
 319          case Format::FLAT:
 320             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
 321             /* fallthrough */
 322          case Format::GLOBAL:
 323          case Format::SCRATCH: {
 324             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
 325             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
 326                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 327             if (!instr->definitions.empty())
 328                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 329             else
 330                check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 331             break;
 332          }
 333          default:
 334             break;
 335          }
 336       }
 337    }
 338
 339    /* validate CFG */
 340    for (unsigned i = 0; i < program->blocks.size(); i++) {
 341       Block& block = program->blocks[i];
 342       check_block(block.index == i, "block.index must match actual index", &block);
 343
 344       /* predecessors/successors should be sorted */
 345       for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
 346          check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
 347       for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
 348          check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
 349       for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
 350          check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
 351       for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
 352          check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
 353
 354       /* critical edges are not allowed */
 355       if (block.linear_preds.size() > 1) {
 356          for (unsigned pred : block.linear_preds)
 357             check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
 358          for (unsigned pred : block.logical_preds)
 359             check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
 360       }
 361    }
 362
 363    assert(is_valid);
 364 }
 365
 366 /* RA validation */
 367 namespace {
 368
 369 struct Location {
 370    Location() : block(NULL), instr(NULL) {}
 371
 372    Block *block;
 373    Instruction *instr; //NULL if it's the block's live-in
 374 };
 375
 376 struct Assignment {
 377    Location defloc;
 378    Location firstloc;
 379    PhysReg reg;
 380 };
 381
 382 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
 383    va_list args;
 384    va_start(args, fmt);
 385    char msg[1024];
 386    vsprintf(msg, fmt, args);
 387    va_end(args);
 388
 389    fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
 390    if (loc.instr) {
 391       aco_print_instr(loc.instr, stderr);
 392       fprintf(stderr, "\n%s", msg);
 393    } else {
 394       fprintf(stderr, "%s", msg);
 395    }
 396    if (loc2.block) {
 397       fprintf(stderr, " in BB%d:\n", loc2.block->index);
 398       aco_print_instr(loc2.instr, stderr);
 399    }
 400    fprintf(stderr, "\n\n");
 401
 402    return true;
 403 }
 404
 405 } /* end namespace */
 406
 407 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
 408    if (!(debug_flags & DEBUG_VALIDATE_RA))
 409       return false;
 410
 411    bool err = false;
 412    aco::live live_vars = aco::live_var_analysis(program, options);
 413    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
 414
 415    std::map<unsigned, Assignment> assignments;
 416    for (Block& block : program->blocks) {
 417       Location loc;
 418       loc.block = &block;
 419       for (aco_ptr<Instruction>& instr : block.instructions) {
 420          if (instr->opcode == aco_opcode::p_phi) {
 421             for (unsigned i = 0; i < instr->operands.size(); i++) {
 422                if (instr->operands[i].isTemp() &&
 423                    instr->operands[i].getTemp().type() == RegType::sgpr &&
 424                    instr->operands[i].isFirstKill())
 425                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
 426             }
 427          }
 428
 429          loc.instr = instr.get();
 430          for (unsigned i = 0; i < instr->operands.size(); i++) {
 431             Operand& op = instr->operands[i];
 432             if (!op.isTemp())
 433                continue;
 434             if (!op.isFixed())
 435                err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
 436             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
 437                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
 438             if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) ||
 439                 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
 440                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
 441             if (op.physReg() == vcc && !program->needs_vcc)
 442                err |= ra_fail(output, loc, Location(), "Operand %d fixed to vcc but needs_vcc=false", i);
 443             if (!assignments[op.tempId()].firstloc.block)
 444                assignments[op.tempId()].firstloc = loc;
 445             if (!assignments[op.tempId()].defloc.block)
 446                assignments[op.tempId()].reg = op.physReg();
 447          }
 448
 449          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 450             Definition& def = instr->definitions[i];
 451             if (!def.isTemp())
 452                continue;
 453             if (!def.isFixed())
 454                err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
 455             if (assignments[def.tempId()].defloc.block)
 456                err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
 457             if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) ||
 458                 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
 459                err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
 460             if (def.physReg() == vcc && !program->needs_vcc)
 461                err |= ra_fail(output, loc, Location(), "Definition %d fixed to vcc but needs_vcc=false", i);
 462             if (!assignments[def.tempId()].firstloc.block)
 463                assignments[def.tempId()].firstloc = loc;
 464             assignments[def.tempId()].defloc = loc;
 465             assignments[def.tempId()].reg = def.physReg();
 466          }
 467       }
 468    }
 469
 470    for (Block& block : program->blocks) {
 471       Location loc;
 472       loc.block = &block;
 473
 474       std::array<unsigned, 512> regs;
 475       regs.fill(0);
 476
 477       std::set<Temp> live;
 478       live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
 479       /* remove killed p_phi sgpr operands */
 480       for (Temp tmp : phi_sgpr_ops[block.index])
 481          live.erase(tmp);
 482
 483       /* check live out */
 484       for (Temp tmp : live) {
 485          PhysReg reg = assignments.at(tmp.id()).reg;
 486          for (unsigned i = 0; i < tmp.size(); i++) {
 487             if (regs[reg + i]) {
 488                err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 489             }
 490             regs[reg + i] = tmp.id();
 491          }
 492       }
 493       regs.fill(0);
 494
 495       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
 496          aco_ptr<Instruction>& instr = *it;
 497
 498          /* check killed p_phi sgpr operands */
 499          if (instr->opcode == aco_opcode::p_logical_end) {
 500             for (Temp tmp : phi_sgpr_ops[block.index]) {
 501                PhysReg reg = assignments.at(tmp.id()).reg;
 502                for (unsigned i = 0; i < tmp.size(); i++) {
 503                   if (regs[reg + i])
 504                      err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
 505                }
 506                live.emplace(tmp);
 507             }
 508          }
 509
 510          for (const Definition& def : instr->definitions) {
 511             if (!def.isTemp())
 512                continue;
 513             live.erase(def.getTemp());
 514          }
 515
 516          /* don't count phi operands as live-in, since they are actually
 517           * killed when they are copied at the predecessor */
 518          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 519             for (const Operand& op : instr->operands) {
 520                if (!op.isTemp())
 521                   continue;
 522                live.insert(op.getTemp());
 523             }
 524          }
 525       }
 526
 527       for (Temp tmp : live) {
 528          PhysReg reg = assignments.at(tmp.id()).reg;
 529          for (unsigned i = 0; i < tmp.size(); i++)
 530             regs[reg + i] = tmp.id();
 531       }
 532
 533       for (aco_ptr<Instruction>& instr : block.instructions) {
 534          loc.instr = instr.get();
 535
 536          /* remove killed p_phi operands from regs */
 537          if (instr->opcode == aco_opcode::p_logical_end) {
 538             for (Temp tmp : phi_sgpr_ops[block.index]) {
 539                PhysReg reg = assignments.at(tmp.id()).reg;
 540                regs[reg] = 0;
 541             }
 542          }
 543
 544          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 545             for (const Operand& op : instr->operands) {
 546                if (!op.isTemp())
 547                   continue;
 548                if (op.isFirstKillBeforeDef()) {
 549                   for (unsigned j = 0; j < op.getTemp().size(); j++)
 550                      regs[op.physReg() + j] = 0;
 551                }
 552             }
 553          }
 554
 555          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 556             Definition& def = instr->definitions[i];
 557             if (!def.isTemp())
 558                continue;
 559             Temp tmp = def.getTemp();
 560             PhysReg reg = assignments.at(tmp.id()).reg;
 561             for (unsigned j = 0; j < tmp.size(); j++) {
 562                if (regs[reg + j])
 563                   err |= ra_fail(output, loc, assignments.at(regs[reg + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg + j]);
 564                regs[reg + j] = tmp.id();
 565             }
 566          }
 567
 568          for (const Definition& def : instr->definitions) {
 569             if (!def.isTemp())
 570                continue;
 571             if (def.isKill()) {
 572                for (unsigned j = 0; j < def.getTemp().size(); j++)
 573                   regs[def.physReg() + j] = 0;
 574             }
 575          }
 576
 577          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 578             for (const Operand& op : instr->operands) {
 579                if (!op.isTemp())
 580                   continue;
 581                if (op.isLateKill() && op.isFirstKill()) {
 582                   for (unsigned j = 0; j < op.getTemp().size(); j++)
 583                      regs[op.physReg() + j] = 0;
 584                }
 585             }
 586          }
 587       }
 588    }
 589
 590    return err;
 591 }
 592 }