src/amd/compiler/aco_validate.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26
  27 #include <array>
  28 #include <map>
  29
  30 namespace aco {
  31
  32 #ifndef NDEBUG
  33 void perfwarn(bool cond, const char *msg, Instruction *instr)
  34 {
  35    if (cond) {
  36       fprintf(stderr, "ACO performance warning: %s\n", msg);
  37       if (instr) {
  38          fprintf(stderr, "instruction: ");
  39          aco_print_instr(instr, stderr);
  40          fprintf(stderr, "\n");
  41       }
  42
  43       if (debug_flags & DEBUG_PERFWARN)
  44          exit(1);
  45    }
  46 }
  47 #endif
  48
  49 bool validate(Program* program, FILE *output)
  50 {
  51    bool is_valid = true;
  52    auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
  53       if (!check) {
  54          fprintf(output, "%s: ", msg);
  55          aco_print_instr(instr, output);
  56          fprintf(output, "\n");
  57          is_valid = false;
  58       }
  59    };
  60    auto check_block = [&output, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
  61       if (!check) {
  62          fprintf(output, "%s: BB%u\n", msg, block->index);
  63          is_valid = false;
  64       }
  65    };
  66
  67    for (Block& block : program->blocks) {
  68       for (aco_ptr<Instruction>& instr : block.instructions) {
  69
  70          /* check base format */
  71          Format base_format = instr->format;
  72          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
  73          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
  74          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
  75             base_format = Format::VOP1;
  76          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
  77             base_format = Format::VOP2;
  78          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
  79             base_format = Format::VOPC;
  80          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
  81             if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
  82                 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
  83                 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
  84                 instr->opcode == aco_opcode::v_interp_p2_f16) {
  85                /* v_interp_*_fp16 are considered VINTRP by the compiler but
  86                 * they are emitted as VOP3.
  87                 */
  88                base_format = Format::VOP3;
  89             } else {
  90                base_format = Format::VINTRP;
  91             }
  92          }
  93          check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
  94
  95          /* check VOP3 modifiers */
  96          if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
  97             check(base_format == Format::VOP2 ||
  98                   base_format == Format::VOP1 ||
  99                   base_format == Format::VOPC ||
 100                   base_format == Format::VINTRP,
 101                   "Format cannot have VOP3A/VOP3B applied", instr.get());
 102          }
 103
 104          /* check SDWA */
 105          if (instr->isSDWA()) {
 106             check(base_format == Format::VOP2 ||
 107                   base_format == Format::VOP1 ||
 108                   base_format == Format::VOPC,
 109                   "Format cannot have SDWA applied", instr.get());
 110
 111             check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
 112
 113             SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get());
 114             check(sdwa->omod == 0 || program->chip_class >= GFX9, "SDWA omod only supported on GFX9+", instr.get());
 115             if (base_format == Format::VOPC) {
 116                check(sdwa->clamp == false || program->chip_class == GFX8, "SDWA VOPC clamp only supported on GFX8", instr.get());
 117                check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
 118                      program->chip_class >= GFX9,
 119                      "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
 120             }
 121
 122             if (instr->operands.size() >= 3) {
 123                check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
 124                      "3rd operand must be fixed to vcc with SDWA", instr.get());
 125             }
 126             if (instr->definitions.size() >= 2) {
 127                check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
 128                      "2nd definition must be fixed to vcc with SDWA", instr.get());
 129             }
 130
 131             check(instr->opcode != aco_opcode::v_madmk_f32 &&
 132                   instr->opcode != aco_opcode::v_madak_f32 &&
 133                   instr->opcode != aco_opcode::v_madmk_f16 &&
 134                   instr->opcode != aco_opcode::v_madak_f16 &&
 135                   instr->opcode != aco_opcode::v_readfirstlane_b32 &&
 136                   instr->opcode != aco_opcode::v_clrexcp &&
 137                   instr->opcode != aco_opcode::v_swap_b32,
 138                   "SDWA can't be used with this opcode", instr.get());
 139             if (program->chip_class != GFX8) {
 140                check(instr->opcode != aco_opcode::v_mac_f32 &&
 141                      instr->opcode != aco_opcode::v_mac_f16 &&
 142                      instr->opcode != aco_opcode::v_fmac_f32 &&
 143                      instr->opcode != aco_opcode::v_fmac_f16,
 144                      "SDWA can't be used with this opcode", instr.get());
 145             }
 146
 147             for (unsigned i = 0; i < MIN2(instr->operands.size(), 2); i++) {
 148                if (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword())
 149                   check((sdwa->sel[i] & sdwa_asuint) == (sdwa_isra | instr->operands[i].bytes()), "Unexpected SDWA sel for sub-dword operand", instr.get());
 150             }
 151             if (instr->definitions[0].regClass().is_subdword())
 152                check((sdwa->dst_sel & sdwa_asuint) == (sdwa_isra | instr->definitions[0].bytes()), "Unexpected SDWA sel for sub-dword definition", instr.get());
 153          }
 154
 155          /* check opsel */
 156          if (instr->isVOP3()) {
 157             VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get());
 158             check(vop3->opsel == 0 || program->chip_class >= GFX9, "Opsel is only supported on GFX9+", instr.get());
 159
 160             for (unsigned i = 0; i < 3; i++) {
 161                if (i >= instr->operands.size() ||
 162                    (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
 163                   check((vop3->opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
 164             }
 165             if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
 166                check((vop3->opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition", instr.get());
 167          }
 168
 169          /* check for undefs */
 170          for (unsigned i = 0; i < instr->operands.size(); i++) {
 171             if (instr->operands[i].isUndefined()) {
 172                bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
 173                bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
 174                                    instr->format == Format::PSEUDO_REDUCTION ||
 175                                    instr->opcode == aco_opcode::p_create_vector ||
 176                                    (flat && i == 1) || (instr->format == Format::MIMG && i == 1) ||
 177                                    ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 1);
 178                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
 179             } else {
 180                check(instr->operands[i].isFixed() || instr->operands[i].isTemp() || instr->operands[i].isConstant(), "Uninitialized Operand", instr.get());
 181             }
 182          }
 183
 184          /* check subdword definitions */
 185          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 186             if (instr->definitions[i].regClass().is_subdword())
 187                check(instr->format == Format::PSEUDO || instr->definitions[i].bytes() <= 4, "Only Pseudo instructions can write subdword registers larger than 4 bytes", instr.get());
 188          }
 189
 190          if (instr->isSALU() || instr->isVALU()) {
 191             /* check literals */
 192             Operand literal(s1);
 193             for (unsigned i = 0; i < instr->operands.size(); i++)
 194             {
 195                Operand op = instr->operands[i];
 196                if (!op.isLiteral())
 197                   continue;
 198
 199                check(instr->format == Format::SOP1 ||
 200                      instr->format == Format::SOP2 ||
 201                      instr->format == Format::SOPC ||
 202                      instr->format == Format::VOP1 ||
 203                      instr->format == Format::VOP2 ||
 204                      instr->format == Format::VOPC ||
 205                      (instr->isVOP3() && program->chip_class >= GFX10),
 206                      "Literal applied on wrong instruction format", instr.get());
 207
 208                check(literal.isUndefined() || (literal.size() == op.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get());
 209                literal = op;
 210                check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
 211             }
 212
 213             /* check num sgprs for VALU */
 214             if (instr->isVALU()) {
 215                bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
 216                                  instr->opcode == aco_opcode::v_lshrrev_b64 ||
 217                                  instr->opcode == aco_opcode::v_ashrrev_i64;
 218                unsigned const_bus_limit = 1;
 219                if (program->chip_class >= GFX10 && !is_shift64)
 220                   const_bus_limit = 2;
 221
 222                uint32_t scalar_mask = instr->isVOP3() ? 0x7 : 0x5;
 223                if (instr->isSDWA())
 224                   scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
 225
 226                if ((int) instr->format & (int) Format::VOPC ||
 227                    instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 228                    instr->opcode == aco_opcode::v_readlane_b32 ||
 229                    instr->opcode == aco_opcode::v_readlane_b32_e64) {
 230                   check(instr->definitions[0].getTemp().type() == RegType::sgpr,
 231                         "Wrong Definition type for VALU instruction", instr.get());
 232                } else {
 233                   check(instr->definitions[0].getTemp().type() == RegType::vgpr,
 234                         "Wrong Definition type for VALU instruction", instr.get());
 235                }
 236
 237                unsigned num_sgprs = 0;
 238                unsigned sgpr[] = {0, 0};
 239                for (unsigned i = 0; i < instr->operands.size(); i++)
 240                {
 241                   Operand op = instr->operands[i];
 242                   if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
 243                       instr->opcode == aco_opcode::v_readlane_b32 ||
 244                       instr->opcode == aco_opcode::v_readlane_b32_e64) {
 245                      check(i != 1 ||
 246                            (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
 247                            op.isConstant(),
 248                            "Must be a SGPR or a constant", instr.get());
 249                      check(i == 1 ||
 250                            (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
 251                            "Wrong Operand type for VALU instruction", instr.get());
 252                      continue;
 253                   }
 254
 255                   if (instr->opcode == aco_opcode::v_writelane_b32 ||
 256                       instr->opcode == aco_opcode::v_writelane_b32_e64) {
 257                      check(i != 2 ||
 258                            (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
 259                            "Wrong Operand type for VALU instruction", instr.get());
 260                      check(i == 2 ||
 261                            (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
 262                            op.isConstant(),
 263                            "Must be a SGPR or a constant", instr.get());
 264                      continue;
 265                   }
 266                   if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
 267                      check(scalar_mask & (1 << i), "Wrong source position for SGPR argument", instr.get());
 268
 269                      if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
 270                         if (num_sgprs < 2)
 271                            sgpr[num_sgprs++] = op.tempId();
 272                      }
 273                   }
 274
 275                   if (op.isConstant() && !op.isLiteral())
 276                      check(scalar_mask & (1 << i), "Wrong source position for constant argument", instr.get());
 277                }
 278                check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get());
 279             }
 280
 281             if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
 282                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
 283                for (const Operand& op : instr->operands) {
 284                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
 285                        "Wrong Operand type for SALU instruction", instr.get());
 286             }
 287          }
 288          }
 289
 290          switch (instr->format) {
 291          case Format::PSEUDO: {
 292             bool is_subdword = false;
 293             bool has_const_sgpr = false;
 294             bool has_literal = false;
 295             for (Definition def : instr->definitions)
 296                is_subdword |= def.regClass().is_subdword();
 297             for (unsigned i = 0; i < instr->operands.size(); i++) {
 298                if (instr->opcode == aco_opcode::p_extract_vector && i == 1)
 299                   continue;
 300                Operand op = instr->operands[i];
 301                is_subdword |= op.hasRegClass() && op.regClass().is_subdword();
 302                has_const_sgpr |= op.isConstant() || (op.hasRegClass() && op.regClass().type() == RegType::sgpr);
 303                has_literal |= op.isLiteral();
 304             }
 305
 306             check(!is_subdword || !has_const_sgpr || program->chip_class >= GFX9,
 307                   "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get());
 308             check(!is_subdword || !has_literal, "Sub-dword pseudo instructions cannot take literals", instr.get());
 309
 310             if (instr->opcode == aco_opcode::p_create_vector) {
 311                unsigned size = 0;
 312                for (const Operand& op : instr->operands) {
 313                   size += op.bytes();
 314                }
 315                check(size == instr->definitions[0].bytes(), "Definition size does not match operand sizes", instr.get());
 316                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
 317                   for (const Operand& op : instr->operands) {
 318                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
 319                            "Wrong Operand type for scalar vector", instr.get());
 320                   }
 321                }
 322             } else if (instr->opcode == aco_opcode::p_extract_vector) {
 323                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
 324                check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <= instr->operands[0].bytes(), "Index out of range", instr.get());
 325                check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
 326                      "Cannot extract SGPR value from VGPR vector", instr.get());
 327             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
 328                check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
 329                for (unsigned i = 0; i < instr->operands.size(); i++) {
 330                   if (instr->operands[i].isTemp())
 331                      check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
 332                            (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
 333                            "Operand and Definition types do not match", instr.get());
 334                }
 335             } else if (instr->opcode == aco_opcode::p_phi) {
 336                check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 337                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "Logical Phi Definition must be vgpr", instr.get());
 338             } else if (instr->opcode == aco_opcode::p_linear_phi) {
 339                for (const Operand& op : instr->operands)
 340                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
 341                check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
 342             }
 343             break;
 344          }
 345          case Format::SMEM: {
 346             if (instr->operands.size() >= 1)
 347                check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
 348             if (instr->operands.size() >= 2)
 349                check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
 350                      "SMEM offset must be constant or sgpr", instr.get());
 351             if (!instr->definitions.empty())
 352                check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
 353             break;
 354          }
 355          case Format::MTBUF:
 356          case Format::MUBUF: {
 357             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
 358             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr,
 359                   "VADDR must be in vgpr for VMEM instructions", instr.get());
 360             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
 361             check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
 362             break;
 363          }
 364          case Format::MIMG: {
 365             check(instr->operands.size() == 3, "MIMG instructions must have exactly 3 operands", instr.get());
 366             check(instr->operands[0].hasRegClass() && (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
 367                   "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
 368             if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr)
 369                check(instr->operands[1].regClass() == s4, "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
 370             else if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr)
 371                check((instr->definitions.empty() || instr->definitions[0].regClass() == instr->operands[1].regClass() ||
 372                      instr->opcode == aco_opcode::image_atomic_cmpswap || instr->opcode == aco_opcode::image_atomic_fcmpswap),
 373                      "MIMG operands[1] (VDATA) must be the same as definitions[0] for atomics", instr.get());
 374             check(instr->operands[2].hasRegClass() && instr->operands[2].regClass().type() == RegType::vgpr,
 375                   "MIMG operands[2] (VADDR) must be VGPR", instr.get());
 376             check(instr->definitions.empty() || (instr->definitions[0].isTemp() && instr->definitions[0].regClass().type() == RegType::vgpr),
 377                   "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
 378             break;
 379          }
 380          case Format::DS: {
 381             for (const Operand& op : instr->operands) {
 382                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
 383                      "Only VGPRs are valid DS instruction operands", instr.get());
 384             }
 385             if (!instr->definitions.empty())
 386                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
 387             break;
 388          }
 389          case Format::EXP: {
 390             for (unsigned i = 0; i < 4; i++)
 391                check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
 392                      "Only VGPRs are valid Export arguments", instr.get());
 393             break;
 394          }
 395          case Format::FLAT:
 396             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
 397             /* fallthrough */
 398          case Format::GLOBAL:
 399          case Format::SCRATCH: {
 400             check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
 401             check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
 402                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
 403             if (!instr->definitions.empty())
 404                check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
 405             else
 406                check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
 407             break;
 408          }
 409          default:
 410             break;
 411          }
 412       }
 413    }
 414
 415    /* validate CFG */
 416    for (unsigned i = 0; i < program->blocks.size(); i++) {
 417       Block& block = program->blocks[i];
 418       check_block(block.index == i, "block.index must match actual index", &block);
 419
 420       /* predecessors/successors should be sorted */
 421       for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
 422          check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
 423       for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
 424          check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
 425       for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
 426          check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
 427       for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
 428          check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
 429
 430       /* critical edges are not allowed */
 431       if (block.linear_preds.size() > 1) {
 432          for (unsigned pred : block.linear_preds)
 433             check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
 434          for (unsigned pred : block.logical_preds)
 435             check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
 436       }
 437    }
 438
 439    return is_valid;
 440 }
 441
 442 /* RA validation */
 443 namespace {
 444
 445 struct Location {
 446    Location() : block(NULL), instr(NULL) {}
 447
 448    Block *block;
 449    Instruction *instr; //NULL if it's the block's live-in
 450 };
 451
 452 struct Assignment {
 453    Location defloc;
 454    Location firstloc;
 455    PhysReg reg;
 456 };
 457
 458 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
 459    va_list args;
 460    va_start(args, fmt);
 461    char msg[1024];
 462    vsprintf(msg, fmt, args);
 463    va_end(args);
 464
 465    fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
 466    if (loc.instr) {
 467       aco_print_instr(loc.instr, stderr);
 468       fprintf(stderr, "\n%s", msg);
 469    } else {
 470       fprintf(stderr, "%s", msg);
 471    }
 472    if (loc2.block) {
 473       fprintf(stderr, " in BB%d:\n", loc2.block->index);
 474       aco_print_instr(loc2.instr, stderr);
 475    }
 476    fprintf(stderr, "\n\n");
 477
 478    return true;
 479 }
 480
 481 bool validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
 482 {
 483    Operand op = instr->operands[index];
 484    unsigned byte = op.physReg().byte();
 485
 486    if (instr->opcode == aco_opcode::p_as_uniform)
 487       return byte == 0;
 488    if (instr->format == Format::PSEUDO && chip >= GFX8)
 489       return true;
 490    if (instr->isSDWA() && (static_cast<SDWA_instruction *>(instr.get())->sel[index] & sdwa_asuint) == (sdwa_isra | op.bytes()))
 491       return true;
 492    if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
 493       return true;
 494
 495    switch (instr->opcode) {
 496    case aco_opcode::v_cvt_f32_ubyte1:
 497       if (byte == 1)
 498          return true;
 499       break;
 500    case aco_opcode::v_cvt_f32_ubyte2:
 501       if (byte == 2)
 502          return true;
 503       break;
 504    case aco_opcode::v_cvt_f32_ubyte3:
 505       if (byte == 3)
 506          return true;
 507       break;
 508    case aco_opcode::ds_write_b8_d16_hi:
 509    case aco_opcode::ds_write_b16_d16_hi:
 510       if (byte == 2 && index == 1)
 511          return true;
 512       break;
 513    case aco_opcode::buffer_store_byte_d16_hi:
 514    case aco_opcode::buffer_store_short_d16_hi:
 515       if (byte == 2 && index == 3)
 516          return true;
 517       break;
 518    case aco_opcode::flat_store_byte_d16_hi:
 519    case aco_opcode::flat_store_short_d16_hi:
 520    case aco_opcode::scratch_store_byte_d16_hi:
 521    case aco_opcode::scratch_store_short_d16_hi:
 522    case aco_opcode::global_store_byte_d16_hi:
 523    case aco_opcode::global_store_short_d16_hi:
 524       if (byte == 2 && index == 2)
 525          return true;
 526    default:
 527       break;
 528    }
 529
 530    return byte == 0;
 531 }
 532
 533 bool validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
 534 {
 535    Definition def = instr->definitions[0];
 536    unsigned byte = def.physReg().byte();
 537
 538    if (instr->format == Format::PSEUDO && chip >= GFX8)
 539       return true;
 540    if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
 541       return true;
 542    if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
 543       return true;
 544
 545    switch (instr->opcode) {
 546    case aco_opcode::buffer_load_ubyte_d16_hi:
 547    case aco_opcode::buffer_load_short_d16_hi:
 548    case aco_opcode::flat_load_ubyte_d16_hi:
 549    case aco_opcode::flat_load_short_d16_hi:
 550    case aco_opcode::scratch_load_ubyte_d16_hi:
 551    case aco_opcode::scratch_load_short_d16_hi:
 552    case aco_opcode::global_load_ubyte_d16_hi:
 553    case aco_opcode::global_load_short_d16_hi:
 554    case aco_opcode::ds_read_u8_d16_hi:
 555    case aco_opcode::ds_read_u16_d16_hi:
 556       return byte == 2;
 557    default:
 558       break;
 559    }
 560
 561    return byte == 0;
 562 }
 563
 564 unsigned get_subdword_bytes_written(Program *program, const aco_ptr<Instruction>& instr, unsigned index)
 565 {
 566    chip_class chip = program->chip_class;
 567    Definition def = instr->definitions[index];
 568
 569    if (instr->format == Format::PSEUDO)
 570       return chip >= GFX8 ? def.bytes() : def.size() * 4u;
 571    if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
 572       return def.bytes();
 573
 574    switch (instr->opcode) {
 575    case aco_opcode::buffer_load_ubyte_d16:
 576    case aco_opcode::buffer_load_short_d16:
 577    case aco_opcode::flat_load_ubyte_d16:
 578    case aco_opcode::flat_load_short_d16:
 579    case aco_opcode::scratch_load_ubyte_d16:
 580    case aco_opcode::scratch_load_short_d16:
 581    case aco_opcode::global_load_ubyte_d16:
 582    case aco_opcode::global_load_short_d16:
 583    case aco_opcode::ds_read_u8_d16:
 584    case aco_opcode::ds_read_u16_d16:
 585    case aco_opcode::buffer_load_ubyte_d16_hi:
 586    case aco_opcode::buffer_load_short_d16_hi:
 587    case aco_opcode::flat_load_ubyte_d16_hi:
 588    case aco_opcode::flat_load_short_d16_hi:
 589    case aco_opcode::scratch_load_ubyte_d16_hi:
 590    case aco_opcode::scratch_load_short_d16_hi:
 591    case aco_opcode::global_load_ubyte_d16_hi:
 592    case aco_opcode::global_load_short_d16_hi:
 593    case aco_opcode::ds_read_u8_d16_hi:
 594    case aco_opcode::ds_read_u16_d16_hi:
 595       return program->sram_ecc_enabled ? 4 : 2;
 596    case aco_opcode::v_mad_f16:
 597    case aco_opcode::v_mad_u16:
 598    case aco_opcode::v_mad_i16:
 599    case aco_opcode::v_fma_f16:
 600    case aco_opcode::v_div_fixup_f16:
 601    case aco_opcode::v_interp_p2_f16:
 602       if (chip >= GFX9)
 603          return 2;
 604    default:
 605       break;
 606    }
 607
 608    return MAX2(chip >= GFX10 ? def.bytes() : 4, instr_info.definition_size[(int)instr->opcode] / 8u);
 609 }
 610
 611 } /* end namespace */
 612
 613 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
 614    if (!(debug_flags & DEBUG_VALIDATE_RA))
 615       return false;
 616
 617    bool err = false;
 618    aco::live live_vars = aco::live_var_analysis(program, options);
 619    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
 620
 621    std::map<unsigned, Assignment> assignments;
 622    for (Block& block : program->blocks) {
 623       Location loc;
 624       loc.block = &block;
 625       for (aco_ptr<Instruction>& instr : block.instructions) {
 626          if (instr->opcode == aco_opcode::p_phi) {
 627             for (unsigned i = 0; i < instr->operands.size(); i++) {
 628                if (instr->operands[i].isTemp() &&
 629                    instr->operands[i].getTemp().type() == RegType::sgpr &&
 630                    instr->operands[i].isFirstKill())
 631                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
 632             }
 633          }
 634
 635          loc.instr = instr.get();
 636          for (unsigned i = 0; i < instr->operands.size(); i++) {
 637             Operand& op = instr->operands[i];
 638             if (!op.isTemp())
 639                continue;
 640             if (!op.isFixed())
 641                err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
 642             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
 643                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
 644             if ((op.getTemp().type() == RegType::vgpr && op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
 645                 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
 646                err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
 647             if (op.physReg() == vcc && !program->needs_vcc)
 648                err |= ra_fail(output, loc, Location(), "Operand %d fixed to vcc but needs_vcc=false", i);
 649             if (op.regClass().is_subdword() && !validate_subdword_operand(program->chip_class, instr, i))
 650                err |= ra_fail(output, loc, Location(), "Operand %d not aligned correctly", i);
 651             if (!assignments[op.tempId()].firstloc.block)
 652                assignments[op.tempId()].firstloc = loc;
 653             if (!assignments[op.tempId()].defloc.block)
 654                assignments[op.tempId()].reg = op.physReg();
 655          }
 656
 657          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 658             Definition& def = instr->definitions[i];
 659             if (!def.isTemp())
 660                continue;
 661             if (!def.isFixed())
 662                err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
 663             if (assignments[def.tempId()].defloc.block)
 664                err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
 665             if ((def.getTemp().type() == RegType::vgpr && def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
 666                 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
 667                err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
 668             if (def.physReg() == vcc && !program->needs_vcc)
 669                err |= ra_fail(output, loc, Location(), "Definition %d fixed to vcc but needs_vcc=false", i);
 670             if (def.regClass().is_subdword() && !validate_subdword_definition(program->chip_class, instr))
 671                err |= ra_fail(output, loc, Location(), "Definition %d not aligned correctly", i);
 672             if (!assignments[def.tempId()].firstloc.block)
 673                assignments[def.tempId()].firstloc = loc;
 674             assignments[def.tempId()].defloc = loc;
 675             assignments[def.tempId()].reg = def.physReg();
 676          }
 677       }
 678    }
 679
 680    for (Block& block : program->blocks) {
 681       Location loc;
 682       loc.block = &block;
 683
 684       std::array<unsigned, 2048> regs; /* register file in bytes */
 685       regs.fill(0);
 686
 687       std::set<Temp> live;
 688       live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
 689       /* remove killed p_phi sgpr operands */
 690       for (Temp tmp : phi_sgpr_ops[block.index])
 691          live.erase(tmp);
 692
 693       /* check live out */
 694       for (Temp tmp : live) {
 695          PhysReg reg = assignments.at(tmp.id()).reg;
 696          for (unsigned i = 0; i < tmp.bytes(); i++) {
 697             if (regs[reg.reg_b + i]) {
 698                err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
 699             }
 700             regs[reg.reg_b + i] = tmp.id();
 701          }
 702       }
 703       regs.fill(0);
 704
 705       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
 706          aco_ptr<Instruction>& instr = *it;
 707
 708          /* check killed p_phi sgpr operands */
 709          if (instr->opcode == aco_opcode::p_logical_end) {
 710             for (Temp tmp : phi_sgpr_ops[block.index]) {
 711                PhysReg reg = assignments.at(tmp.id()).reg;
 712                for (unsigned i = 0; i < tmp.bytes(); i++) {
 713                   if (regs[reg.reg_b + i])
 714                      err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
 715                }
 716                live.emplace(tmp);
 717             }
 718          }
 719
 720          for (const Definition& def : instr->definitions) {
 721             if (!def.isTemp())
 722                continue;
 723             live.erase(def.getTemp());
 724          }
 725
 726          /* don't count phi operands as live-in, since they are actually
 727           * killed when they are copied at the predecessor */
 728          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 729             for (const Operand& op : instr->operands) {
 730                if (!op.isTemp())
 731                   continue;
 732                live.insert(op.getTemp());
 733             }
 734          }
 735       }
 736
 737       for (Temp tmp : live) {
 738          PhysReg reg = assignments.at(tmp.id()).reg;
 739          for (unsigned i = 0; i < tmp.bytes(); i++)
 740             regs[reg.reg_b + i] = tmp.id();
 741       }
 742
 743       for (aco_ptr<Instruction>& instr : block.instructions) {
 744          loc.instr = instr.get();
 745
 746          /* remove killed p_phi operands from regs */
 747          if (instr->opcode == aco_opcode::p_logical_end) {
 748             for (Temp tmp : phi_sgpr_ops[block.index]) {
 749                PhysReg reg = assignments.at(tmp.id()).reg;
 750                for (unsigned i = 0; i < tmp.bytes(); i++)
 751                   regs[reg.reg_b + i] = 0;
 752             }
 753          }
 754
 755          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 756             for (const Operand& op : instr->operands) {
 757                if (!op.isTemp())
 758                   continue;
 759                if (op.isFirstKillBeforeDef()) {
 760                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
 761                      regs[op.physReg().reg_b + j] = 0;
 762                }
 763             }
 764          }
 765
 766          for (unsigned i = 0; i < instr->definitions.size(); i++) {
 767             Definition& def = instr->definitions[i];
 768             if (!def.isTemp())
 769                continue;
 770             Temp tmp = def.getTemp();
 771             PhysReg reg = assignments.at(tmp.id()).reg;
 772             for (unsigned j = 0; j < tmp.bytes(); j++) {
 773                if (regs[reg.reg_b + j])
 774                   err |= ra_fail(output, loc, assignments.at(regs[reg.reg_b + j]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg.reg_b + j]);
 775                regs[reg.reg_b + j] = tmp.id();
 776             }
 777             if (def.regClass().is_subdword() && def.bytes() < 4) {
 778                unsigned written = get_subdword_bytes_written(program, instr, i);
 779                /* If written=4, the instruction still might write the upper half. In that case, it's the lower half that isn't preserved */
 780                for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
 781                   unsigned written_reg = reg.reg() * 4u + j;
 782                   if (regs[written_reg] && regs[written_reg] != def.tempId())
 783                      err |= ra_fail(output, loc, assignments.at(regs[written_reg]).defloc, "Assignment of element %d of %%%d overwrites the full register taken by %%%d from instruction", i, tmp.id(), regs[written_reg]);
 784                }
 785             }
 786          }
 787
 788          for (const Definition& def : instr->definitions) {
 789             if (!def.isTemp())
 790                continue;
 791             if (def.isKill()) {
 792                for (unsigned j = 0; j < def.getTemp().bytes(); j++)
 793                   regs[def.physReg().reg_b + j] = 0;
 794             }
 795          }
 796
 797          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
 798             for (const Operand& op : instr->operands) {
 799                if (!op.isTemp())
 800                   continue;
 801                if (op.isLateKill() && op.isFirstKill()) {
 802                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
 803                      regs[op.physReg().reg_b + j] = 0;
 804                }
 805             }
 806          }
 807       }
 808    }
 809
 810    return err;
 811 }
 812 }