src/amd/compiler/aco_insert_exec_mask.cpp

   1 /*
   2  * Copyright © 2019 Valve Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "aco_ir.h"
  26 #include "aco_builder.h"
  27 #include "util/u_math.h"
  28
  29 namespace aco {
  30
  31 namespace {
  32
  33 enum WQMState : uint8_t {
  34    Unspecified = 0,
  35    Exact = 1 << 0,
  36    WQM = 1 << 1, /* with control flow applied */
  37    Preserve_WQM = 1 << 2,
  38    Exact_Branch = 1 << 3,
  39 };
  40
  41 enum mask_type : uint8_t {
  42    mask_type_global = 1 << 0,
  43    mask_type_exact = 1 << 1,
  44    mask_type_wqm = 1 << 2,
  45    mask_type_loop = 1 << 3, /* active lanes of a loop */
  46    mask_type_initial = 1 << 4, /* initially active lanes */
  47 };
  48
  49 struct wqm_ctx {
  50    Program* program;
  51    /* state for WQM propagation */
  52    std::set<unsigned> worklist;
  53    std::vector<uint16_t> defined_in;
  54    std::vector<bool> needs_wqm;
  55    std::vector<bool> branch_wqm; /* true if the branch condition in this block should be in wqm */
  56    bool loop;
  57    bool wqm;
  58    wqm_ctx(Program* program) : program(program),
  59                                defined_in(program->peekAllocationId(), 0xFFFF),
  60                                needs_wqm(program->peekAllocationId()),
  61                                branch_wqm(program->blocks.size()),
  62                                loop(false),
  63                                wqm(false)
  64    {
  65       for (unsigned i = 0; i < program->blocks.size(); i++)
  66          worklist.insert(i);
  67    }
  68 };
  69
  70 struct loop_info {
  71    Block* loop_header;
  72    uint16_t num_exec_masks;
  73    uint8_t needs;
  74    bool has_divergent_break;
  75    bool has_divergent_continue;
  76    bool has_discard; /* has a discard or demote */
  77    loop_info(Block* b, uint16_t num, uint8_t needs, bool breaks, bool cont, bool discard) :
  78              loop_header(b), num_exec_masks(num), needs(needs), has_divergent_break(breaks),
  79              has_divergent_continue(cont), has_discard(discard) {}
  80 };
  81
  82 struct block_info {
  83    std::vector<std::pair<Temp, uint8_t>> exec;
  84    std::vector<WQMState> instr_needs;
  85    uint8_t block_needs;
  86    uint8_t ever_again_needs;
  87    /* more... */
  88 };
  89
  90 struct exec_ctx {
  91    Program *program;
  92    std::vector<block_info> info;
  93    std::vector<loop_info> loop;
  94    bool handle_wqm = false;
  95    exec_ctx(Program *program) : program(program), info(program->blocks.size()) {}
  96 };
  97
  98 bool pred_by_exec_mask(aco_ptr<Instruction>& instr) {
  99    if (instr->isSALU())
 100       return instr->reads_exec();
 101    if (instr->format == Format::SMEM || instr->isSALU())
 102       return false;
 103    if (instr->format == Format::PSEUDO_BARRIER)
 104       return false;
 105
 106    if (instr->format == Format::PSEUDO) {
 107       switch (instr->opcode) {
 108       case aco_opcode::p_create_vector:
 109          return instr->definitions[0].getTemp().type() == RegType::vgpr;
 110       case aco_opcode::p_extract_vector:
 111       case aco_opcode::p_split_vector:
 112          return instr->operands[0].getTemp().type() == RegType::vgpr;
 113       case aco_opcode::p_spill:
 114       case aco_opcode::p_reload:
 115          return false;
 116       default:
 117          break;
 118       }
 119    }
 120
 121    if (instr->opcode == aco_opcode::v_readlane_b32 ||
 122        instr->opcode == aco_opcode::v_readlane_b32_e64 ||
 123        instr->opcode == aco_opcode::v_writelane_b32 ||
 124        instr->opcode == aco_opcode::v_writelane_b32_e64)
 125       return false;
 126
 127    return true;
 128 }
 129
 130 bool needs_exact(aco_ptr<Instruction>& instr) {
 131    if (instr->format == Format::MUBUF) {
 132       MUBUF_instruction *mubuf = static_cast<MUBUF_instruction *>(instr.get());
 133       return mubuf->disable_wqm;
 134    } else if (instr->format == Format::MTBUF) {
 135       MTBUF_instruction *mtbuf = static_cast<MTBUF_instruction *>(instr.get());
 136       return mtbuf->disable_wqm;
 137    } else if (instr->format == Format::MIMG) {
 138       MIMG_instruction *mimg = static_cast<MIMG_instruction *>(instr.get());
 139       return mimg->disable_wqm;
 140    } else if (instr->format == Format::FLAT || instr->format == Format::GLOBAL) {
 141       FLAT_instruction *flat = static_cast<FLAT_instruction *>(instr.get());
 142       return flat->disable_wqm;
 143    } else {
 144       return instr->format == Format::EXP || instr->opcode == aco_opcode::p_fs_buffer_store_smem;
 145    }
 146 }
 147
 148 void set_needs_wqm(wqm_ctx &ctx, Temp tmp)
 149 {
 150    if (!ctx.needs_wqm[tmp.id()]) {
 151       ctx.needs_wqm[tmp.id()] = true;
 152       if (ctx.defined_in[tmp.id()] != 0xFFFF)
 153          ctx.worklist.insert(ctx.defined_in[tmp.id()]);
 154    }
 155 }
 156
 157 void mark_block_wqm(wqm_ctx &ctx, unsigned block_idx)
 158 {
 159    if (ctx.branch_wqm[block_idx])
 160       return;
 161
 162    ctx.branch_wqm[block_idx] = true;
 163    Block& block = ctx.program->blocks[block_idx];
 164    aco_ptr<Instruction>& branch = block.instructions.back();
 165
 166    if (branch->opcode != aco_opcode::p_branch) {
 167       assert(!branch->operands.empty() && branch->operands[0].isTemp());
 168       set_needs_wqm(ctx, branch->operands[0].getTemp());
 169    }
 170
 171    /* TODO: this sets more branch conditions to WQM than it needs to
 172     * it should be enough to stop at the "exec mask top level" */
 173    if (block.kind & block_kind_top_level)
 174       return;
 175
 176    for (unsigned pred_idx : block.logical_preds)
 177       mark_block_wqm(ctx, pred_idx);
 178 }
 179
 180 void get_block_needs(wqm_ctx &ctx, exec_ctx &exec_ctx, Block* block)
 181 {
 182    block_info& info = exec_ctx.info[block->index];
 183
 184    std::vector<WQMState> instr_needs(block->instructions.size());
 185
 186    if (block->kind & block_kind_top_level) {
 187       if (ctx.loop && ctx.wqm) {
 188          /* mark all break conditions as WQM */
 189          unsigned block_idx = block->index + 1;
 190          while (!(ctx.program->blocks[block_idx].kind & block_kind_top_level)) {
 191             if (ctx.program->blocks[block_idx].kind & block_kind_break)
 192                mark_block_wqm(ctx, block_idx);
 193             block_idx++;
 194          }
 195       } else if (ctx.loop && !ctx.wqm) {
 196          /* Ensure a branch never results in an exec mask with only helper
 197           * invocations (which can cause a loop to repeat infinitively if it's
 198           * break branches are done in exact). */
 199          unsigned block_idx = block->index;
 200          do {
 201             if ((ctx.program->blocks[block_idx].kind & block_kind_branch))
 202                exec_ctx.info[block_idx].block_needs |= Exact_Branch;
 203             block_idx++;
 204          } while (!(ctx.program->blocks[block_idx].kind & block_kind_top_level));
 205       }
 206
 207       ctx.loop = false;
 208       ctx.wqm = false;
 209    }
 210
 211    for (int i = block->instructions.size() - 1; i >= 0; --i) {
 212       aco_ptr<Instruction>& instr = block->instructions[i];
 213
 214       WQMState needs = needs_exact(instr) ? Exact : Unspecified;
 215       bool propagate_wqm = instr->opcode == aco_opcode::p_wqm;
 216       bool preserve_wqm = instr->opcode == aco_opcode::p_discard_if;
 217       bool pred_by_exec = pred_by_exec_mask(instr);
 218       for (const Definition& definition : instr->definitions) {
 219          if (!definition.isTemp())
 220             continue;
 221          const unsigned def = definition.tempId();
 222          ctx.defined_in[def] = block->index;
 223          if (needs == Unspecified && ctx.needs_wqm[def]) {
 224             needs = pred_by_exec ? WQM : Unspecified;
 225             propagate_wqm = true;
 226          }
 227       }
 228
 229       if (propagate_wqm) {
 230          for (const Operand& op : instr->operands) {
 231             if (op.isTemp()) {
 232                set_needs_wqm(ctx, op.getTemp());
 233             }
 234          }
 235       } else if (preserve_wqm && info.block_needs & WQM) {
 236          needs = Preserve_WQM;
 237       }
 238
 239       /* ensure the condition controlling the control flow for this phi is in WQM */
 240       if (needs == WQM && instr->opcode == aco_opcode::p_phi) {
 241          for (unsigned pred_idx : block->logical_preds)
 242             mark_block_wqm(ctx, pred_idx);
 243       }
 244
 245       instr_needs[i] = needs;
 246       info.block_needs |= needs;
 247    }
 248
 249    info.instr_needs = instr_needs;
 250
 251    /* for "if (<cond>) <wqm code>" or "while (<cond>) <wqm code>",
 252     * <cond> should be computed in WQM */
 253    if (info.block_needs & WQM && !(block->kind & block_kind_top_level)) {
 254       for (unsigned pred_idx : block->logical_preds)
 255          mark_block_wqm(ctx, pred_idx);
 256       ctx.wqm = true;
 257    }
 258    if (block->kind & block_kind_loop_header)
 259       ctx.loop = true;
 260 }
 261
 262 void calculate_wqm_needs(exec_ctx& exec_ctx)
 263 {
 264    wqm_ctx ctx(exec_ctx.program);
 265
 266    while (!ctx.worklist.empty()) {
 267       unsigned block_index = *std::prev(ctx.worklist.end());
 268       ctx.worklist.erase(std::prev(ctx.worklist.end()));
 269
 270       get_block_needs(ctx, exec_ctx, &exec_ctx.program->blocks[block_index]);
 271    }
 272
 273    uint8_t ever_again_needs = 0;
 274    for (int i = exec_ctx.program->blocks.size() - 1; i >= 0; i--) {
 275       exec_ctx.info[i].ever_again_needs = ever_again_needs;
 276       Block& block = exec_ctx.program->blocks[i];
 277
 278       if (block.kind & block_kind_needs_lowering)
 279          exec_ctx.info[i].block_needs |= Exact;
 280
 281       /* if discard is used somewhere in nested CF, we need to preserve the WQM mask */
 282       if ((block.kind & block_kind_discard ||
 283            block.kind & block_kind_uses_discard_if) &&
 284           ever_again_needs & WQM)
 285          exec_ctx.info[i].block_needs |= Preserve_WQM;
 286
 287       ever_again_needs |= exec_ctx.info[i].block_needs & ~Exact_Branch;
 288       if (block.kind & block_kind_discard ||
 289           block.kind & block_kind_uses_discard_if ||
 290           block.kind & block_kind_uses_demote)
 291          ever_again_needs |= Exact;
 292
 293       /* don't propagate WQM preservation further than the next top_level block */
 294       if (block.kind & block_kind_top_level)
 295          ever_again_needs &= ~Preserve_WQM;
 296       else
 297          exec_ctx.info[i].block_needs &= ~Preserve_WQM;
 298    }
 299    exec_ctx.handle_wqm = true;
 300 }
 301
 302 void transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
 303 {
 304    if (ctx.info[idx].exec.back().second & mask_type_wqm)
 305       return;
 306    if (ctx.info[idx].exec.back().second & mask_type_global) {
 307       Temp exec_mask = ctx.info[idx].exec.back().first;
 308       exec_mask = bld.sop1(Builder::s_wqm, bld.def(bld.lm, exec), bld.def(s1, scc), exec_mask);
 309       ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
 310       return;
 311    }
 312    /* otherwise, the WQM mask should be one below the current mask */
 313    ctx.info[idx].exec.pop_back();
 314    assert(ctx.info[idx].exec.back().second & mask_type_wqm);
 315    assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
 316    ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec),
 317                                                 ctx.info[idx].exec.back().first);
 318 }
 319
 320 void transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
 321 {
 322    if (ctx.info[idx].exec.back().second & mask_type_exact)
 323       return;
 324    /* We can't remove the loop exec mask, because that can cause exec.size() to
 325     * be less than num_exec_masks. The loop exec mask also needs to be kept
 326     * around for various uses. */
 327    if ((ctx.info[idx].exec.back().second & mask_type_global) &&
 328        !(ctx.info[idx].exec.back().second & mask_type_loop)) {
 329       ctx.info[idx].exec.pop_back();
 330       assert(ctx.info[idx].exec.back().second & mask_type_exact);
 331       assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
 332       ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec),
 333                                                    ctx.info[idx].exec.back().first);
 334       return;
 335    }
 336    /* otherwise, we create an exact mask and push to the stack */
 337    Temp wqm = ctx.info[idx].exec.back().first;
 338    Temp exact = bld.tmp(bld.lm);
 339    wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
 340                   bld.exec(Definition(exact)), ctx.info[idx].exec[0].first, bld.exec(wqm));
 341    ctx.info[idx].exec.back().first = wqm;
 342    ctx.info[idx].exec.emplace_back(exact, mask_type_exact);
 343 }
 344
 345 unsigned add_coupling_code(exec_ctx& ctx, Block* block,
 346                            std::vector<aco_ptr<Instruction>>& instructions)
 347 {
 348    unsigned idx = block->index;
 349    Builder bld(ctx.program, &instructions);
 350    std::vector<unsigned>& preds = block->linear_preds;
 351
 352    /* start block */
 353    if (idx == 0) {
 354       aco_ptr<Instruction>& startpgm = block->instructions[0];
 355       assert(startpgm->opcode == aco_opcode::p_startpgm);
 356       Temp exec_mask = startpgm->definitions.back().getTemp();
 357       bld.insert(std::move(startpgm));
 358
 359       /* exec seems to need to be manually initialized with combined shaders */
 360       if (util_bitcount(ctx.program->stage & sw_mask) > 1) {
 361          bld.sop1(Builder::s_mov, bld.exec(Definition(exec_mask)), bld.lm == s2 ? Operand(UINT64_MAX) : Operand(UINT32_MAX));
 362          instructions[0]->definitions.pop_back();
 363       }
 364
 365       if (ctx.handle_wqm) {
 366          ctx.info[0].exec.emplace_back(exec_mask, mask_type_global | mask_type_exact | mask_type_initial);
 367          /* if this block only needs WQM, initialize already */
 368          if (ctx.info[0].block_needs == WQM)
 369             transition_to_WQM(ctx, bld, 0);
 370       } else {
 371          uint8_t mask = mask_type_global;
 372          if (ctx.program->needs_wqm) {
 373             exec_mask = bld.sop1(Builder::s_wqm, bld.def(bld.lm, exec), bld.def(s1, scc), bld.exec(exec_mask));
 374             mask |= mask_type_wqm;
 375          } else {
 376             mask |= mask_type_exact;
 377          }
 378          ctx.info[0].exec.emplace_back(exec_mask, mask);
 379       }
 380
 381       return 1;
 382    }
 383
 384    /* loop entry block */
 385    if (block->kind & block_kind_loop_header) {
 386       assert(preds[0] == idx - 1);
 387       ctx.info[idx].exec = ctx.info[idx - 1].exec;
 388       loop_info& info = ctx.loop.back();
 389       while (ctx.info[idx].exec.size() > info.num_exec_masks)
 390          ctx.info[idx].exec.pop_back();
 391
 392       /* create ssa names for outer exec masks */
 393       if (info.has_discard) {
 394          aco_ptr<Pseudo_instruction> phi;
 395          for (int i = 0; i < info.num_exec_masks - 1; i++) {
 396             phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1));
 397             phi->definitions[0] = bld.def(bld.lm);
 398             phi->operands[0] = Operand(ctx.info[preds[0]].exec[i].first);
 399             ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
 400          }
 401       }
 402
 403       /* create ssa name for restore mask */
 404       if (info.has_divergent_break) {
 405          /* this phi might be trivial but ensures a parallelcopy on the loop header */
 406          aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
 407          phi->definitions[0] = bld.def(bld.lm);
 408          phi->operands[0] = Operand(ctx.info[preds[0]].exec[info.num_exec_masks - 1].first);
 409          ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
 410       }
 411
 412       /* create ssa name for loop active mask */
 413       aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
 414       if (info.has_divergent_continue)
 415          phi->definitions[0] = bld.def(bld.lm);
 416       else
 417          phi->definitions[0] = bld.def(bld.lm, exec);
 418       phi->operands[0] = Operand(ctx.info[preds[0]].exec.back().first);
 419       Temp loop_active = bld.insert(std::move(phi));
 420
 421       if (info.has_divergent_break) {
 422          uint8_t mask_type = (ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact)) | mask_type_loop;
 423          ctx.info[idx].exec.emplace_back(loop_active, mask_type);
 424       } else {
 425          ctx.info[idx].exec.back().first = loop_active;
 426          ctx.info[idx].exec.back().second |= mask_type_loop;
 427       }
 428
 429       /* create a parallelcopy to move the active mask to exec */
 430       unsigned i = 0;
 431       if (info.has_divergent_continue) {
 432          while (block->instructions[i]->opcode != aco_opcode::p_logical_start) {
 433             bld.insert(std::move(block->instructions[i]));
 434             i++;
 435          }
 436          uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
 437          assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
 438          ctx.info[idx].exec.emplace_back(bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec),
 439                                                     ctx.info[idx].exec.back().first), mask_type);
 440       }
 441
 442       return i;
 443    }
 444
 445    /* loop exit block */
 446    if (block->kind & block_kind_loop_exit) {
 447       Block* header = ctx.loop.back().loop_header;
 448       loop_info& info = ctx.loop.back();
 449
 450       for (ASSERTED unsigned pred : preds)
 451          assert(ctx.info[pred].exec.size() >= info.num_exec_masks);
 452
 453       /* fill the loop header phis */
 454       std::vector<unsigned>& header_preds = header->linear_preds;
 455       int k = 0;
 456       if (info.has_discard) {
 457          while (k < info.num_exec_masks - 1) {
 458             aco_ptr<Instruction>& phi = header->instructions[k];
 459             assert(phi->opcode == aco_opcode::p_linear_phi);
 460             for (unsigned i = 1; i < phi->operands.size(); i++)
 461                phi->operands[i] = Operand(ctx.info[header_preds[i]].exec[k].first);
 462             k++;
 463          }
 464       }
 465       aco_ptr<Instruction>& phi = header->instructions[k++];
 466       assert(phi->opcode == aco_opcode::p_linear_phi);
 467       for (unsigned i = 1; i < phi->operands.size(); i++)
 468          phi->operands[i] = Operand(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
 469
 470       if (info.has_divergent_break) {
 471          aco_ptr<Instruction>& phi = header->instructions[k];
 472          assert(phi->opcode == aco_opcode::p_linear_phi);
 473          for (unsigned i = 1; i < phi->operands.size(); i++)
 474             phi->operands[i] = Operand(ctx.info[header_preds[i]].exec[info.num_exec_masks].first);
 475       }
 476
 477       assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
 478
 479       /* create the loop exit phis if not trivial */
 480       for (unsigned k = 0; k < info.num_exec_masks; k++) {
 481          Temp same = ctx.info[preds[0]].exec[k].first;
 482          uint8_t type = ctx.info[header_preds[0]].exec[k].second;
 483          bool trivial = true;
 484
 485          for (unsigned i = 1; i < preds.size() && trivial; i++) {
 486             if (ctx.info[preds[i]].exec[k].first != same)
 487                trivial = false;
 488          }
 489
 490          if (trivial) {
 491             ctx.info[idx].exec.emplace_back(same, type);
 492          } else {
 493             /* create phi for loop footer */
 494             aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
 495             phi->definitions[0] = bld.def(bld.lm);
 496             for (unsigned i = 0; i < phi->operands.size(); i++)
 497                phi->operands[i] = Operand(ctx.info[preds[i]].exec[k].first);
 498             ctx.info[idx].exec.emplace_back(bld.insert(std::move(phi)), type);
 499          }
 500       }
 501       assert(ctx.info[idx].exec.size() == info.num_exec_masks);
 502
 503       /* create a parallelcopy to move the live mask to exec */
 504       unsigned i = 0;
 505       while (block->instructions[i]->opcode != aco_opcode::p_logical_start) {
 506          bld.insert(std::move(block->instructions[i]));
 507          i++;
 508       }
 509
 510       if (ctx.handle_wqm) {
 511          if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
 512             if ((ctx.info[idx].block_needs | ctx.info[idx].ever_again_needs) == 0 ||
 513                 (ctx.info[idx].block_needs | ctx.info[idx].ever_again_needs) == Exact) {
 514                ctx.info[idx].exec.back().second |= mask_type_global;
 515                transition_to_Exact(ctx, bld, idx);
 516                ctx.handle_wqm = false;
 517             }
 518          }
 519          if (ctx.info[idx].block_needs == WQM)
 520             transition_to_WQM(ctx, bld, idx);
 521          else if (ctx.info[idx].block_needs == Exact)
 522             transition_to_Exact(ctx, bld, idx);
 523       }
 524
 525       assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
 526       ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec),
 527                                                    ctx.info[idx].exec.back().first);
 528
 529       ctx.loop.pop_back();
 530       return i;
 531    }
 532
 533    if (preds.size() == 1) {
 534       ctx.info[idx].exec = ctx.info[preds[0]].exec;
 535    } else {
 536       assert(preds.size() == 2);
 537       /* if one of the predecessors ends in exact mask, we pop it from stack */
 538       unsigned num_exec_masks = std::min(ctx.info[preds[0]].exec.size(),
 539                                          ctx.info[preds[1]].exec.size());
 540       if (block->kind & block_kind_top_level && !(block->kind & block_kind_merge))
 541          num_exec_masks = std::min(num_exec_masks, 2u);
 542
 543       /* create phis for diverged exec masks */
 544       for (unsigned i = 0; i < num_exec_masks; i++) {
 545          bool in_exec = i == num_exec_masks - 1 && !(block->kind & block_kind_merge);
 546          if (!in_exec && ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
 547             assert(ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
 548             ctx.info[idx].exec.emplace_back(ctx.info[preds[0]].exec[i]);
 549             continue;
 550          }
 551
 552          Temp phi = bld.pseudo(aco_opcode::p_linear_phi, in_exec ? bld.def(bld.lm, exec) : bld.def(bld.lm),
 553                                ctx.info[preds[0]].exec[i].first,
 554                                ctx.info[preds[1]].exec[i].first);
 555          uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
 556          ctx.info[idx].exec.emplace_back(phi, mask_type);
 557       }
 558    }
 559
 560    unsigned i = 0;
 561    while (block->instructions[i]->opcode == aco_opcode::p_phi ||
 562           block->instructions[i]->opcode == aco_opcode::p_linear_phi) {
 563       bld.insert(std::move(block->instructions[i]));
 564       i++;
 565    }
 566
 567    if (block->kind & block_kind_merge)
 568       ctx.info[idx].exec.pop_back();
 569
 570    if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 3) {
 571       assert(ctx.info[idx].exec.back().second == mask_type_exact);
 572       assert(block->kind & block_kind_merge);
 573       ctx.info[idx].exec.pop_back();
 574    }
 575
 576    /* try to satisfy the block's needs */
 577    if (ctx.handle_wqm) {
 578       if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
 579          if ((ctx.info[idx].block_needs | ctx.info[idx].ever_again_needs) == 0 ||
 580              (ctx.info[idx].block_needs | ctx.info[idx].ever_again_needs) == Exact) {
 581             ctx.info[idx].exec.back().second |= mask_type_global;
 582             transition_to_Exact(ctx, bld, idx);
 583             ctx.handle_wqm = false;
 584          }
 585       }
 586       if (ctx.info[idx].block_needs == WQM)
 587          transition_to_WQM(ctx, bld, idx);
 588       else if (ctx.info[idx].block_needs == Exact)
 589          transition_to_Exact(ctx, bld, idx);
 590    }
 591
 592    if (block->kind & block_kind_merge) {
 593       Temp restore = ctx.info[idx].exec.back().first;
 594       assert(restore.size() == bld.lm.size());
 595       ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec), restore);
 596    }
 597
 598    return i;
 599 }
 600
 601 void lower_fs_buffer_store_smem(Builder& bld, bool need_check, aco_ptr<Instruction>& instr, Temp cur_exec)
 602 {
 603    Operand offset = instr->operands[1];
 604    if (need_check) {
 605       /* if exec is zero, then use UINT32_MAX as an offset and make this store a no-op */
 606       Temp nonempty = bld.sopc(Builder::s_cmp_lg, bld.def(s1, scc), cur_exec, Operand(0u));
 607
 608       if (offset.isLiteral())
 609          offset = bld.sop1(aco_opcode::s_mov_b32, bld.def(s1), offset);
 610
 611       offset = bld.sop2(aco_opcode::s_cselect_b32, bld.hint_m0(bld.def(s1)),
 612                         offset, Operand(UINT32_MAX), bld.scc(nonempty));
 613    } else if (offset.isConstant() && offset.constantValue() > 0xFFFFF) {
 614       offset = bld.sop1(aco_opcode::s_mov_b32, bld.hint_m0(bld.def(s1)), offset);
 615    }
 616    if (!offset.isConstant())
 617       offset.setFixed(m0);
 618
 619    switch (instr->operands[2].size()) {
 620    case 1:
 621       instr->opcode = aco_opcode::s_buffer_store_dword;
 622       break;
 623    case 2:
 624       instr->opcode = aco_opcode::s_buffer_store_dwordx2;
 625       break;
 626    case 4:
 627       instr->opcode = aco_opcode::s_buffer_store_dwordx4;
 628       break;
 629    default:
 630       unreachable("Invalid SMEM buffer store size");
 631    }
 632    instr->operands[1] = offset;
 633    /* as_uniform() needs to be done here so it's done in exact mode and helper
 634     * lanes don't contribute. */
 635    instr->operands[2] = Operand(bld.as_uniform(instr->operands[2]));
 636 }
 637
 638 void process_instructions(exec_ctx& ctx, Block* block,
 639                           std::vector<aco_ptr<Instruction>>& instructions,
 640                           unsigned idx)
 641 {
 642    WQMState state;
 643    if (ctx.info[block->index].exec.back().second & mask_type_wqm)
 644       state = WQM;
 645    else {
 646       assert(!ctx.handle_wqm || ctx.info[block->index].exec.back().second & mask_type_exact);
 647       state = Exact;
 648    }
 649
 650    /* if the block doesn't need both, WQM and Exact, we can skip processing the instructions */
 651    bool process = (ctx.handle_wqm &&
 652                    (ctx.info[block->index].block_needs & state) !=
 653                    (ctx.info[block->index].block_needs & (WQM | Exact))) ||
 654                   block->kind & block_kind_uses_discard_if ||
 655                   block->kind & block_kind_uses_demote ||
 656                   block->kind & block_kind_needs_lowering;
 657    if (!process) {
 658       std::vector<aco_ptr<Instruction>>::iterator it = std::next(block->instructions.begin(), idx);
 659       instructions.insert(instructions.end(),
 660                           std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(it),
 661                           std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(block->instructions.end()));
 662       return;
 663    }
 664
 665    Builder bld(ctx.program, &instructions);
 666
 667    for (; idx < block->instructions.size(); idx++) {
 668       aco_ptr<Instruction> instr = std::move(block->instructions[idx]);
 669
 670       WQMState needs = ctx.handle_wqm ? ctx.info[block->index].instr_needs[idx] : Unspecified;
 671
 672       if (instr->opcode == aco_opcode::p_discard_if) {
 673          if (ctx.info[block->index].block_needs & Preserve_WQM) {
 674             assert(block->kind & block_kind_top_level);
 675             transition_to_WQM(ctx, bld, block->index);
 676             ctx.info[block->index].exec.back().second &= ~mask_type_global;
 677          }
 678          int num = ctx.info[block->index].exec.size();
 679          assert(num);
 680          Operand cond = instr->operands[0];
 681          for (int i = num - 1; i >= 0; i--) {
 682             Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
 683                                           ctx.info[block->index].exec[i].first, cond);
 684             if (i == num - 1) {
 685                andn2->operands[0].setFixed(exec);
 686                andn2->definitions[0].setFixed(exec);
 687             }
 688             if (i == 0) {
 689                instr->opcode = aco_opcode::p_exit_early_if;
 690                instr->operands[0] = bld.scc(andn2->definitions[1].getTemp());
 691             }
 692             ctx.info[block->index].exec[i].first = andn2->definitions[0].getTemp();
 693          }
 694          assert(!ctx.handle_wqm || (ctx.info[block->index].exec[0].second & mask_type_wqm) == 0);
 695
 696       } else if (needs == WQM && state != WQM) {
 697          transition_to_WQM(ctx, bld, block->index);
 698          state = WQM;
 699       } else if (needs == Exact && state != Exact) {
 700          transition_to_Exact(ctx, bld, block->index);
 701          state = Exact;
 702       }
 703
 704       if (instr->opcode == aco_opcode::p_is_helper || instr->opcode == aco_opcode::p_load_helper) {
 705          Definition dst = instr->definitions[0];
 706          assert(dst.size() == bld.lm.size());
 707          if (state == Exact) {
 708             instr.reset(create_instruction<SOP1_instruction>(bld.w64or32(Builder::s_mov), Format::SOP1, 1, 1));
 709             instr->operands[0] = Operand(0u);
 710             instr->definitions[0] = dst;
 711          } else {
 712             std::pair<Temp, uint8_t>& exact_mask = ctx.info[block->index].exec[0];
 713             if (instr->opcode == aco_opcode::p_load_helper &&
 714                 !(ctx.info[block->index].exec[0].second & mask_type_initial)) {
 715                /* find last initial exact mask */
 716                for (int i = block->index; i >= 0; i--) {
 717                   if (ctx.program->blocks[i].kind & block_kind_top_level &&
 718                       ctx.info[i].exec[0].second & mask_type_initial) {
 719                      exact_mask = ctx.info[i].exec[0];
 720                      break;
 721                   }
 722                }
 723             }
 724
 725             assert(instr->opcode == aco_opcode::p_is_helper || exact_mask.second & mask_type_initial);
 726             assert(exact_mask.second & mask_type_exact);
 727
 728             instr.reset(create_instruction<SOP2_instruction>(bld.w64or32(Builder::s_andn2), Format::SOP2, 2, 2));
 729             instr->operands[0] = Operand(ctx.info[block->index].exec.back().first); /* current exec */
 730             instr->operands[1] = Operand(exact_mask.first);
 731             instr->definitions[0] = dst;
 732             instr->definitions[1] = bld.def(s1, scc);
 733          }
 734       } else if (instr->opcode == aco_opcode::p_demote_to_helper) {
 735          /* turn demote into discard_if with only exact masks */
 736          assert((ctx.info[block->index].exec[0].second & (mask_type_exact | mask_type_global)) == (mask_type_exact | mask_type_global));
 737          ctx.info[block->index].exec[0].second &= ~mask_type_initial;
 738
 739          int num;
 740          Temp cond, exit_cond;
 741          if (instr->operands[0].isConstant()) {
 742             assert(instr->operands[0].constantValue() == -1u);
 743             /* transition to exact and set exec to zero */
 744             Temp old_exec = ctx.info[block->index].exec.back().first;
 745             Temp new_exec = bld.tmp(bld.lm);
 746             exit_cond = bld.tmp(s1);
 747             cond = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
 748                             bld.exec(Definition(new_exec)), Operand(0u), bld.exec(old_exec));
 749
 750             num = ctx.info[block->index].exec.size() - 2;
 751             if (ctx.info[block->index].exec.back().second & mask_type_exact) {
 752                ctx.info[block->index].exec.back().first = new_exec;
 753             } else {
 754                ctx.info[block->index].exec.back().first = cond;
 755                ctx.info[block->index].exec.emplace_back(new_exec, mask_type_exact);
 756             }
 757          } else {
 758             /* demote_if: transition to exact */
 759             transition_to_Exact(ctx, bld, block->index);
 760             assert(instr->operands[0].isTemp());
 761             cond = instr->operands[0].getTemp();
 762             num = ctx.info[block->index].exec.size() - 1;
 763          }
 764
 765          for (int i = num; i >= 0; i--) {
 766             if (ctx.info[block->index].exec[i].second & mask_type_exact) {
 767                Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
 768                                              ctx.info[block->index].exec[i].first, cond);
 769                if (i == (int)ctx.info[block->index].exec.size() - 1) {
 770                   andn2->operands[0].setFixed(exec);
 771                   andn2->definitions[0].setFixed(exec);
 772                }
 773
 774                ctx.info[block->index].exec[i].first = andn2->definitions[0].getTemp();
 775                exit_cond = andn2->definitions[1].getTemp();
 776             } else {
 777                assert(i != 0);
 778             }
 779          }
 780          instr->opcode = aco_opcode::p_exit_early_if;
 781          instr->operands[0] = bld.scc(exit_cond);
 782          state = Exact;
 783
 784       } else if (instr->opcode == aco_opcode::p_fs_buffer_store_smem) {
 785          bool need_check = ctx.info[block->index].exec.size() != 1 &&
 786                            !(ctx.info[block->index].exec[ctx.info[block->index].exec.size() - 2].second & Exact);
 787          lower_fs_buffer_store_smem(bld, need_check, instr, ctx.info[block->index].exec.back().first);
 788       }
 789
 790       bld.insert(std::move(instr));
 791    }
 792 }
 793
 794 void add_branch_code(exec_ctx& ctx, Block* block)
 795 {
 796    unsigned idx = block->index;
 797    Builder bld(ctx.program, block);
 798
 799    if (idx == ctx.program->blocks.size() - 1)
 800       return;
 801
 802    /* try to disable wqm handling */
 803    if (ctx.handle_wqm && block->kind & block_kind_top_level) {
 804       if (ctx.info[idx].exec.size() == 3) {
 805          assert(ctx.info[idx].exec[1].second == mask_type_wqm);
 806          ctx.info[idx].exec.pop_back();
 807       }
 808       assert(ctx.info[idx].exec.size() <= 2);
 809
 810       if (ctx.info[idx].ever_again_needs == 0 ||
 811           ctx.info[idx].ever_again_needs == Exact) {
 812          /* transition to Exact */
 813          aco_ptr<Instruction> branch = std::move(block->instructions.back());
 814          block->instructions.pop_back();
 815          ctx.info[idx].exec.back().second |= mask_type_global;
 816          transition_to_Exact(ctx, bld, idx);
 817          bld.insert(std::move(branch));
 818          ctx.handle_wqm = false;
 819
 820       } else if (ctx.info[idx].block_needs & Preserve_WQM) {
 821          /* transition to WQM and remove global flag */
 822          aco_ptr<Instruction> branch = std::move(block->instructions.back());
 823          block->instructions.pop_back();
 824          transition_to_WQM(ctx, bld, idx);
 825          ctx.info[idx].exec.back().second &= ~mask_type_global;
 826          bld.insert(std::move(branch));
 827       }
 828    }
 829
 830    if (block->kind & block_kind_loop_preheader) {
 831       /* collect information about the succeeding loop */
 832       bool has_divergent_break = false;
 833       bool has_divergent_continue = false;
 834       bool has_discard = false;
 835       uint8_t needs = 0;
 836       unsigned loop_nest_depth = ctx.program->blocks[idx + 1].loop_nest_depth;
 837
 838       for (unsigned i = idx + 1; ctx.program->blocks[i].loop_nest_depth >= loop_nest_depth; i++) {
 839          Block& loop_block = ctx.program->blocks[i];
 840          needs |= ctx.info[i].block_needs;
 841
 842          if (loop_block.kind & block_kind_uses_discard_if ||
 843              loop_block.kind & block_kind_discard ||
 844              loop_block.kind & block_kind_uses_demote)
 845             has_discard = true;
 846          if (loop_block.loop_nest_depth != loop_nest_depth)
 847             continue;
 848
 849          if (loop_block.kind & block_kind_uniform)
 850             continue;
 851          else if (loop_block.kind & block_kind_break)
 852             has_divergent_break = true;
 853          else if (loop_block.kind & block_kind_continue)
 854             has_divergent_continue = true;
 855       }
 856
 857       if (ctx.handle_wqm) {
 858          if (needs & WQM) {
 859             aco_ptr<Instruction> branch = std::move(block->instructions.back());
 860             block->instructions.pop_back();
 861             transition_to_WQM(ctx, bld, idx);
 862             bld.insert(std::move(branch));
 863          } else {
 864             aco_ptr<Instruction> branch = std::move(block->instructions.back());
 865             block->instructions.pop_back();
 866             transition_to_Exact(ctx, bld, idx);
 867             bld.insert(std::move(branch));
 868          }
 869       }
 870
 871       unsigned num_exec_masks = ctx.info[idx].exec.size();
 872       if (block->kind & block_kind_top_level)
 873          num_exec_masks = std::min(num_exec_masks, 2u);
 874
 875       ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]],
 876                             num_exec_masks,
 877                             needs,
 878                             has_divergent_break,
 879                             has_divergent_continue,
 880                             has_discard);
 881    }
 882
 883    if (block->kind & block_kind_discard) {
 884
 885       assert(block->instructions.back()->format == Format::PSEUDO_BRANCH);
 886       aco_ptr<Instruction> branch = std::move(block->instructions.back());
 887       block->instructions.pop_back();
 888
 889       /* create a discard_if() instruction with the exec mask as condition */
 890       unsigned num = 0;
 891       if (ctx.loop.size()) {
 892          /* if we're in a loop, only discard from the outer exec masks */
 893          num = ctx.loop.back().num_exec_masks;
 894       } else {
 895          num = ctx.info[idx].exec.size() - 1;
 896       }
 897
 898       Temp old_exec = ctx.info[idx].exec.back().first;
 899       Temp new_exec = bld.tmp(bld.lm);
 900       Temp cond = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
 901                            bld.exec(Definition(new_exec)), Operand(0u), bld.exec(old_exec));
 902       ctx.info[idx].exec.back().first = new_exec;
 903
 904       for (int i = num - 1; i >= 0; i--) {
 905          Instruction *andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
 906                                        ctx.info[block->index].exec[i].first, cond);
 907          if (i == (int)ctx.info[idx].exec.size() - 1)
 908             andn2->definitions[0].setFixed(exec);
 909          if (i == 0)
 910             bld.pseudo(aco_opcode::p_exit_early_if, bld.scc(andn2->definitions[1].getTemp()));
 911          ctx.info[block->index].exec[i].first = andn2->definitions[0].getTemp();
 912       }
 913       assert(!ctx.handle_wqm || (ctx.info[block->index].exec[0].second & mask_type_wqm) == 0);
 914
 915       if ((block->kind & (block_kind_break | block_kind_uniform)) == block_kind_break)
 916          ctx.info[idx].exec.back().first = cond;
 917       bld.insert(std::move(branch));
 918       /* no return here as it can be followed by a divergent break */
 919    }
 920
 921    if (block->kind & block_kind_continue_or_break) {
 922       assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind & block_kind_loop_header);
 923       assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind & block_kind_loop_exit);
 924       assert(block->instructions.back()->opcode == aco_opcode::p_branch);
 925       block->instructions.pop_back();
 926
 927       while (!(ctx.info[idx].exec.back().second & mask_type_loop))
 928          ctx.info[idx].exec.pop_back();
 929
 930       ctx.info[idx].exec.back().first = bld.pseudo(aco_opcode::p_parallelcopy, bld.def(bld.lm, exec), ctx.info[idx].exec.back().first);
 931       bld.branch(aco_opcode::p_cbranch_nz, bld.exec(ctx.info[idx].exec.back().first), block->linear_succs[1], block->linear_succs[0]);
 932       return;
 933    }
 934
 935    if (block->kind & block_kind_uniform) {
 936       Pseudo_branch_instruction* branch = static_cast<Pseudo_branch_instruction*>(block->instructions.back().get());
 937       if (branch->opcode == aco_opcode::p_branch) {
 938          branch->target[0] = block->linear_succs[0];
 939       } else {
 940          branch->target[0] = block->linear_succs[1];
 941          branch->target[1] = block->linear_succs[0];
 942       }
 943       return;
 944    }
 945
 946    if (block->kind & block_kind_branch) {
 947
 948       if (ctx.handle_wqm &&
 949           ctx.info[idx].exec.size() >= 2 &&
 950           ctx.info[idx].exec.back().second == mask_type_exact &&
 951           !(ctx.info[idx].block_needs & Exact_Branch) &&
 952           ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].second & mask_type_wqm) {
 953          /* return to wqm before branching */
 954          ctx.info[idx].exec.pop_back();
 955       }
 956
 957       // orig = s_and_saveexec_b64
 958       assert(block->linear_succs.size() == 2);
 959       assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
 960       Temp cond = block->instructions.back()->operands[0].getTemp();
 961       block->instructions.pop_back();
 962
 963       if (ctx.info[idx].block_needs & Exact_Branch)
 964          transition_to_Exact(ctx, bld, idx);
 965
 966       Temp current_exec = ctx.info[idx].exec.back().first;
 967       uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
 968
 969       Temp then_mask = bld.tmp(bld.lm);
 970       Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
 971                                bld.exec(Definition(then_mask)), cond, bld.exec(current_exec));
 972
 973       ctx.info[idx].exec.back().first = old_exec;
 974
 975       /* add next current exec to the stack */
 976       ctx.info[idx].exec.emplace_back(then_mask, mask_type);
 977
 978       bld.branch(aco_opcode::p_cbranch_z, bld.exec(then_mask), block->linear_succs[1], block->linear_succs[0]);
 979       return;
 980    }
 981
 982    if (block->kind & block_kind_invert) {
 983       // exec = s_andn2_b64 (original_exec, exec)
 984       assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_nz);
 985       block->instructions.pop_back();
 986       Temp then_mask = ctx.info[idx].exec.back().first;
 987       uint8_t mask_type = ctx.info[idx].exec.back().second;
 988       ctx.info[idx].exec.pop_back();
 989       Temp orig_exec = ctx.info[idx].exec.back().first;
 990       Temp else_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm, exec),
 991                                 bld.def(s1, scc), orig_exec, bld.exec(then_mask));
 992
 993       /* add next current exec to the stack */
 994       ctx.info[idx].exec.emplace_back(else_mask, mask_type);
 995
 996       bld.branch(aco_opcode::p_cbranch_z, bld.exec(else_mask), block->linear_succs[1], block->linear_succs[0]);
 997       return;
 998    }
 999
1000    if (block->kind & block_kind_break) {
1001       // loop_mask = s_andn2_b64 (loop_mask, exec)
1002       assert(block->instructions.back()->opcode == aco_opcode::p_branch);
1003       block->instructions.pop_back();
1004
1005       Temp current_exec = ctx.info[idx].exec.back().first;
1006       Temp cond = Temp();
1007       for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
1008          cond = bld.tmp(s1);
1009          Temp exec_mask = ctx.info[idx].exec[exec_idx].first;
1010          exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
1011                               exec_mask, current_exec);
1012          ctx.info[idx].exec[exec_idx].first = exec_mask;
1013          if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
1014             break;
1015       }
1016
1017       /* check if the successor is the merge block, otherwise set exec to 0 */
1018       // TODO: this could be done better by directly branching to the merge block
1019       unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
1020       Block& succ = ctx.program->blocks[succ_idx];
1021       if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
1022          ctx.info[idx].exec.back().first = bld.sop1(Builder::s_mov, bld.def(bld.lm, exec), Operand(0u));
1023       }
1024
1025       bld.branch(aco_opcode::p_cbranch_nz, bld.scc(cond), block->linear_succs[1], block->linear_succs[0]);
1026       return;
1027    }
1028
1029    if (block->kind & block_kind_continue) {
1030       assert(block->instructions.back()->opcode == aco_opcode::p_branch);
1031       block->instructions.pop_back();
1032
1033       Temp current_exec = ctx.info[idx].exec.back().first;
1034       Temp cond = Temp();
1035       for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
1036          if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
1037             break;
1038          cond = bld.tmp(s1);
1039          Temp exec_mask = ctx.info[idx].exec[exec_idx].first;
1040          exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
1041                               exec_mask, bld.exec(current_exec));
1042          ctx.info[idx].exec[exec_idx].first = exec_mask;
1043       }
1044       assert(cond != Temp());
1045
1046       /* check if the successor is the merge block, otherwise set exec to 0 */
1047       // TODO: this could be done better by directly branching to the merge block
1048       unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
1049       Block& succ = ctx.program->blocks[succ_idx];
1050       if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
1051          ctx.info[idx].exec.back().first = bld.sop1(Builder::s_mov, bld.def(bld.lm, exec), Operand(0u));
1052       }
1053
1054       bld.branch(aco_opcode::p_cbranch_nz, bld.scc(cond), block->linear_succs[1], block->linear_succs[0]);
1055       return;
1056    }
1057 }
1058
1059 void process_block(exec_ctx& ctx, Block* block)
1060 {
1061    std::vector<aco_ptr<Instruction>> instructions;
1062    instructions.reserve(block->instructions.size());
1063
1064    unsigned idx = add_coupling_code(ctx, block, instructions);
1065
1066    assert(block->index != ctx.program->blocks.size() - 1 ||
1067           ctx.info[block->index].exec.size() <= 2);
1068
1069    process_instructions(ctx, block, instructions, idx);
1070
1071    block->instructions = std::move(instructions);
1072
1073    add_branch_code(ctx, block);
1074
1075    block->live_out_exec = ctx.info[block->index].exec.back().first;
1076 }
1077
1078 } /* end namespace */
1079
1080
1081 void insert_exec_mask(Program *program)
1082 {
1083    exec_ctx ctx(program);
1084
1085    if (program->needs_wqm && program->needs_exact)
1086       calculate_wqm_needs(ctx);
1087
1088    for (Block& block : program->blocks)
1089       process_block(ctx, &block);
1090
1091 }
1092
1093 }
1094