src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool uniform_has_then_branch;
  89    bool then_branch_divergent;
  90    Block BB_invert;
  91    Block BB_endif;
  92 };
  93
  94 static bool visit_cf_list(struct isel_context *ctx,
  95                           struct exec_list *list);
  96
  97 static void add_logical_edge(unsigned pred_idx, Block *succ)
  98 {
  99    succ->logical_preds.emplace_back(pred_idx);
 100 }
 101
 102
 103 static void add_linear_edge(unsigned pred_idx, Block *succ)
 104 {
 105    succ->linear_preds.emplace_back(pred_idx);
 106 }
 107
 108 static void add_edge(unsigned pred_idx, Block *succ)
 109 {
 110    add_logical_edge(pred_idx, succ);
 111    add_linear_edge(pred_idx, succ);
 112 }
 113
 114 static void append_logical_start(Block *b)
 115 {
 116    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 117 }
 118
 119 static void append_logical_end(Block *b)
 120 {
 121    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 122 }
 123
 124 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 125 {
 126    assert(ctx->allocated[def->index].id());
 127    return ctx->allocated[def->index];
 128 }
 129
 130 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 131                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 132 {
 133    Builder bld(ctx->program, ctx->block);
 134    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 135    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 136
 137    if (ctx->program->wave_size == 32) {
 138       return thread_id_lo;
 139    } else {
 140       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 141       return thread_id_hi;
 142    }
 143 }
 144
 145 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 146 {
 147    Builder bld(ctx->program, ctx->block);
 148
 149    if (!dst.id())
 150       dst = bld.tmp(src.regClass());
 151
 152    assert(src.size() == dst.size());
 153
 154    if (ctx->stage != fragment_fs) {
 155       if (!dst.id())
 156          return src;
 157
 158       bld.copy(Definition(dst), src);
 159       return dst;
 160    }
 161
 162    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 163    ctx->program->needs_wqm |= program_needs_wqm;
 164    return dst;
 165 }
 166
 167 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 168 {
 169    if (index.regClass() == s1)
 170       return bld.readlane(bld.def(s1), data, index);
 171
 172    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 173
 174    /* Currently not implemented on GFX6-7 */
 175    assert(ctx->options->chip_class >= GFX8);
 176
 177    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 178       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 179    }
 180
 181    /* GFX10, wave64 mode:
 182     * The bpermute instruction is limited to half-wave operation, which means that it can't
 183     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 184     * emulate it here.
 185     */
 186    if (!ctx->has_gfx10_wave64_bpermute) {
 187       ctx->has_gfx10_wave64_bpermute = true;
 188       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 189       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 190    }
 191
 192    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 193    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 194    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 195    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 196
 197    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 198                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 199 }
 200
 201 Temp as_vgpr(isel_context *ctx, Temp val)
 202 {
 203    if (val.type() == RegType::sgpr) {
 204       Builder bld(ctx->program, ctx->block);
 205       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 206    }
 207    assert(val.type() == RegType::vgpr);
 208    return val;
 209 }
 210
 211 //assumes a != 0xffffffff
 212 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 213 {
 214    assert(b != 0);
 215    Builder bld(ctx->program, ctx->block);
 216
 217    if (util_is_power_of_two_or_zero(b)) {
 218       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 219       return;
 220    }
 221
 222    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 223
 224    assert(info.multiplier <= 0xffffffff);
 225
 226    bool pre_shift = info.pre_shift != 0;
 227    bool increment = info.increment != 0;
 228    bool multiply = true;
 229    bool post_shift = info.post_shift != 0;
 230
 231    if (!pre_shift && !increment && !multiply && !post_shift) {
 232       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 233       return;
 234    }
 235
 236    Temp pre_shift_dst = a;
 237    if (pre_shift) {
 238       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 239       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 240    }
 241
 242    Temp increment_dst = pre_shift_dst;
 243    if (increment) {
 244       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 245       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 246    }
 247
 248    Temp multiply_dst = increment_dst;
 249    if (multiply) {
 250       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 251       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 252                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 253    }
 254
 255    if (post_shift) {
 256       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 257    }
 258 }
 259
 260 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 261 {
 262    Builder bld(ctx->program, ctx->block);
 263    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 264 }
 265
 266
 267 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 268 {
 269    /* no need to extract the whole vector */
 270    if (src.regClass() == dst_rc) {
 271       assert(idx == 0);
 272       return src;
 273    }
 274
 275    assert(src.bytes() > (idx * dst_rc.bytes()));
 276    Builder bld(ctx->program, ctx->block);
 277    auto it = ctx->allocated_vec.find(src.id());
 278    if (it != ctx->allocated_vec.end() && dst_rc.bytes() == it->second[idx].regClass().bytes()) {
 279       if (it->second[idx].regClass() == dst_rc) {
 280          return it->second[idx];
 281       } else {
 282          assert(!dst_rc.is_subdword());
 283          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 284          return bld.copy(bld.def(dst_rc), it->second[idx]);
 285       }
 286    }
 287
 288    if (dst_rc.is_subdword())
 289       src = as_vgpr(ctx, src);
 290
 291    if (src.bytes() == dst_rc.bytes()) {
 292       assert(idx == 0);
 293       return bld.copy(bld.def(dst_rc), src);
 294    } else {
 295       Temp dst = bld.tmp(dst_rc);
 296       emit_extract_vector(ctx, src, idx, dst);
 297       return dst;
 298    }
 299 }
 300
 301 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 302 {
 303    if (num_components == 1)
 304       return;
 305    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 306       return;
 307    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 308    split->operands[0] = Operand(vec_src);
 309    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 310    RegClass rc;
 311    if (num_components > vec_src.size()) {
 312       if (vec_src.type() == RegType::sgpr)
 313          return;
 314
 315       /* sub-dword split */
 316       assert(vec_src.type() == RegType::vgpr);
 317       rc = RegClass(RegType::vgpr, vec_src.bytes() / num_components).as_subdword();
 318    } else {
 319       rc = RegClass(vec_src.type(), vec_src.size() / num_components);
 320    }
 321    for (unsigned i = 0; i < num_components; i++) {
 322       elems[i] = {ctx->program->allocateId(), rc};
 323       split->definitions[i] = Definition(elems[i]);
 324    }
 325    ctx->block->instructions.emplace_back(std::move(split));
 326    ctx->allocated_vec.emplace(vec_src.id(), elems);
 327 }
 328
 329 /* This vector expansion uses a mask to determine which elements in the new vector
 330  * come from the original vector. The other elements are undefined. */
 331 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 332 {
 333    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 334
 335    if (vec_src == dst)
 336       return;
 337
 338    Builder bld(ctx->program, ctx->block);
 339    if (num_components == 1) {
 340       if (dst.type() == RegType::sgpr)
 341          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 342       else
 343          bld.copy(Definition(dst), vec_src);
 344       return;
 345    }
 346
 347    unsigned component_size = dst.size() / num_components;
 348    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 349
 350    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 351    vec->definitions[0] = Definition(dst);
 352    unsigned k = 0;
 353    for (unsigned i = 0; i < num_components; i++) {
 354       if (mask & (1 << i)) {
 355          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 356          if (dst.type() == RegType::sgpr)
 357             src = bld.as_uniform(src);
 358          vec->operands[i] = Operand(src);
 359       } else {
 360          vec->operands[i] = Operand(0u);
 361       }
 362       elems[i] = vec->operands[i].getTemp();
 363    }
 364    ctx->block->instructions.emplace_back(std::move(vec));
 365    ctx->allocated_vec.emplace(dst.id(), elems);
 366 }
 367
 368 /* adjust misaligned small bit size loads */
 369 void byte_align_scalar(isel_context *ctx, Temp vec, Operand offset, Temp dst)
 370 {
 371    Builder bld(ctx->program, ctx->block);
 372    Operand shift;
 373    Temp select = Temp();
 374    if (offset.isConstant()) {
 375       assert(offset.constantValue() && offset.constantValue() < 4);
 376       shift = Operand(offset.constantValue() * 8);
 377    } else {
 378       /* bit_offset = 8 * (offset & 0x3) */
 379       Temp tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), offset, Operand(3u));
 380       select = bld.tmp(s1);
 381       shift = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.scc(Definition(select)), tmp, Operand(3u));
 382    }
 383
 384    if (vec.size() == 1) {
 385       bld.sop2(aco_opcode::s_lshr_b32, Definition(dst), bld.def(s1, scc), vec, shift);
 386    } else if (vec.size() == 2) {
 387       Temp tmp = dst.size() == 2 ? dst : bld.tmp(s2);
 388       bld.sop2(aco_opcode::s_lshr_b64, Definition(tmp), bld.def(s1, scc), vec, shift);
 389       if (tmp == dst)
 390          emit_split_vector(ctx, dst, 2);
 391       else
 392          emit_extract_vector(ctx, tmp, 0, dst);
 393    } else if (vec.size() == 4) {
 394       Temp lo = bld.tmp(s2), hi = bld.tmp(s2);
 395       bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), vec);
 396       hi = bld.pseudo(aco_opcode::p_extract_vector, bld.def(s1), hi, Operand(0u));
 397       if (select != Temp())
 398          hi = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), hi, Operand(0u), select);
 399       lo = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), lo, shift);
 400       Temp mid = bld.tmp(s1);
 401       lo = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), Definition(mid), lo);
 402       hi = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), hi, shift);
 403       mid = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), hi, mid);
 404       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, mid);
 405       emit_split_vector(ctx, dst, 2);
 406    }
 407 }
 408
 409 /* this function trims subdword vectors:
 410  * if dst is vgpr - split the src and create a shrunk version according to the mask.
 411  * if dst is sgpr - split the src, but move the original to sgpr. */
 412 void trim_subdword_vector(isel_context *ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 413 {
 414    assert(vec_src.type() == RegType::vgpr);
 415    emit_split_vector(ctx, vec_src, num_components);
 416
 417    Builder bld(ctx->program, ctx->block);
 418    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 419    unsigned component_size = vec_src.bytes() / num_components;
 420    RegClass rc = RegClass(RegType::vgpr, component_size).as_subdword();
 421
 422    unsigned k = 0;
 423    for (unsigned i = 0; i < num_components; i++) {
 424       if (mask & (1 << i))
 425          elems[k++] = emit_extract_vector(ctx, vec_src, i, rc);
 426    }
 427
 428    if (dst.type() == RegType::vgpr) {
 429       assert(dst.bytes() == k * component_size);
 430       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, k, 1)};
 431       for (unsigned i = 0; i < k; i++)
 432          vec->operands[i] = Operand(elems[i]);
 433       vec->definitions[0] = Definition(dst);
 434       bld.insert(std::move(vec));
 435    } else {
 436       // TODO: alignbyte if mask doesn't start with 1?
 437       assert(mask & 1);
 438       assert(dst.size() == vec_src.size());
 439       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 440    }
 441    ctx->allocated_vec.emplace(dst.id(), elems);
 442 }
 443
 444 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 445 {
 446    Builder bld(ctx->program, ctx->block);
 447    if (!dst.id())
 448       dst = bld.tmp(bld.lm);
 449
 450    assert(val.regClass() == s1);
 451    assert(dst.regClass() == bld.lm);
 452
 453    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 454 }
 455
 456 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 457 {
 458    Builder bld(ctx->program, ctx->block);
 459    if (!dst.id())
 460       dst = bld.tmp(s1);
 461
 462    assert(val.regClass() == bld.lm);
 463    assert(dst.regClass() == s1);
 464
 465    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 466    Temp tmp = bld.tmp(s1);
 467    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 468    return emit_wqm(ctx, tmp, dst);
 469 }
 470
 471 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 472 {
 473    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 474       return get_ssa_temp(ctx, src.src.ssa);
 475
 476    if (src.src.ssa->num_components == size) {
 477       bool identity_swizzle = true;
 478       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 479          if (src.swizzle[i] != i)
 480             identity_swizzle = false;
 481       }
 482       if (identity_swizzle)
 483          return get_ssa_temp(ctx, src.src.ssa);
 484    }
 485
 486    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 487    unsigned elem_size = vec.bytes() / src.src.ssa->num_components;
 488    assert(elem_size > 0);
 489    assert(vec.bytes() % elem_size == 0);
 490
 491    if (elem_size < 4 && vec.type() == RegType::sgpr) {
 492       assert(src.src.ssa->bit_size == 8 || src.src.ssa->bit_size == 16);
 493       assert(size == 1);
 494       unsigned swizzle = src.swizzle[0];
 495       if (vec.size() > 1) {
 496          assert(src.src.ssa->bit_size == 16);
 497          vec = emit_extract_vector(ctx, vec, swizzle / 2, s1);
 498          swizzle = swizzle & 1;
 499       }
 500       if (swizzle == 0)
 501          return vec;
 502
 503       Temp dst{ctx->program->allocateId(), s1};
 504       aco_ptr<SOP2_instruction> bfe{create_instruction<SOP2_instruction>(aco_opcode::s_bfe_u32, Format::SOP2, 2, 1)};
 505       bfe->operands[0] = Operand(vec);
 506       bfe->operands[1] = Operand(uint32_t((src.src.ssa->bit_size << 16) | (src.src.ssa->bit_size * swizzle)));
 507       bfe->definitions[0] = Definition(dst);
 508       ctx->block->instructions.emplace_back(std::move(bfe));
 509       return dst;
 510    }
 511
 512    RegClass elem_rc = elem_size < 4 ? RegClass(vec.type(), elem_size).as_subdword() : RegClass(vec.type(), elem_size / 4);
 513    if (size == 1) {
 514       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 515    } else {
 516       assert(size <= 4);
 517       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 518       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 519       for (unsigned i = 0; i < size; ++i) {
 520          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 521          vec_instr->operands[i] = Operand{elems[i]};
 522       }
 523       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size / 4)};
 524       vec_instr->definitions[0] = Definition(dst);
 525       ctx->block->instructions.emplace_back(std::move(vec_instr));
 526       ctx->allocated_vec.emplace(dst.id(), elems);
 527       return dst;
 528    }
 529 }
 530
 531 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 532 {
 533    if (ptr.size() == 2)
 534       return ptr;
 535    Builder bld(ctx->program, ctx->block);
 536    if (ptr.type() == RegType::vgpr)
 537       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 538    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 539                      ptr, Operand((unsigned)ctx->options->address32_hi));
 540 }
 541
 542 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 543 {
 544    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 545    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 546    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 547    sop2->definitions[0] = Definition(dst);
 548    if (writes_scc)
 549       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 550    ctx->block->instructions.emplace_back(std::move(sop2));
 551 }
 552
 553 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 554                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 555 {
 556    Builder bld(ctx->program, ctx->block);
 557    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 558    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 559    if (src1.type() == RegType::sgpr) {
 560       if (commutative && src0.type() == RegType::vgpr) {
 561          Temp t = src0;
 562          src0 = src1;
 563          src1 = t;
 564       } else if (src0.type() == RegType::vgpr &&
 565                  op != aco_opcode::v_madmk_f32 &&
 566                  op != aco_opcode::v_madak_f32 &&
 567                  op != aco_opcode::v_madmk_f16 &&
 568                  op != aco_opcode::v_madak_f16) {
 569          /* If the instruction is not commutative, we emit a VOP3A instruction */
 570          bld.vop2_e64(op, Definition(dst), src0, src1);
 571          return;
 572       } else {
 573          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 574       }
 575    }
 576
 577    if (flush_denorms && ctx->program->chip_class < GFX9) {
 578       assert(dst.size() == 1);
 579       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 580       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 581    } else {
 582       bld.vop2(op, Definition(dst), src0, src1);
 583    }
 584 }
 585
 586 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 587                             bool flush_denorms = false)
 588 {
 589    Temp src0 = get_alu_src(ctx, instr->src[0]);
 590    Temp src1 = get_alu_src(ctx, instr->src[1]);
 591    Temp src2 = get_alu_src(ctx, instr->src[2]);
 592
 593    /* ensure that the instruction has at most 1 sgpr operand
 594     * The optimizer will inline constants for us */
 595    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 596       src0 = as_vgpr(ctx, src0);
 597    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 598       src1 = as_vgpr(ctx, src1);
 599    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 600       src2 = as_vgpr(ctx, src2);
 601
 602    Builder bld(ctx->program, ctx->block);
 603    if (flush_denorms && ctx->program->chip_class < GFX9) {
 604       assert(dst.size() == 1);
 605       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 606       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 607    } else {
 608       bld.vop3(op, Definition(dst), src0, src1, src2);
 609    }
 610 }
 611
 612 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 613 {
 614    Builder bld(ctx->program, ctx->block);
 615    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 616 }
 617
 618 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 619 {
 620    Temp src0 = get_alu_src(ctx, instr->src[0]);
 621    Temp src1 = get_alu_src(ctx, instr->src[1]);
 622    assert(src0.size() == src1.size());
 623
 624    aco_ptr<Instruction> vopc;
 625    if (src1.type() == RegType::sgpr) {
 626       if (src0.type() == RegType::vgpr) {
 627          /* to swap the operands, we might also have to change the opcode */
 628          switch (op) {
 629             case aco_opcode::v_cmp_lt_f32:
 630                op = aco_opcode::v_cmp_gt_f32;
 631                break;
 632             case aco_opcode::v_cmp_ge_f32:
 633                op = aco_opcode::v_cmp_le_f32;
 634                break;
 635             case aco_opcode::v_cmp_lt_i32:
 636                op = aco_opcode::v_cmp_gt_i32;
 637                break;
 638             case aco_opcode::v_cmp_ge_i32:
 639                op = aco_opcode::v_cmp_le_i32;
 640                break;
 641             case aco_opcode::v_cmp_lt_u32:
 642                op = aco_opcode::v_cmp_gt_u32;
 643                break;
 644             case aco_opcode::v_cmp_ge_u32:
 645                op = aco_opcode::v_cmp_le_u32;
 646                break;
 647             case aco_opcode::v_cmp_lt_f64:
 648                op = aco_opcode::v_cmp_gt_f64;
 649                break;
 650             case aco_opcode::v_cmp_ge_f64:
 651                op = aco_opcode::v_cmp_le_f64;
 652                break;
 653             case aco_opcode::v_cmp_lt_i64:
 654                op = aco_opcode::v_cmp_gt_i64;
 655                break;
 656             case aco_opcode::v_cmp_ge_i64:
 657                op = aco_opcode::v_cmp_le_i64;
 658                break;
 659             case aco_opcode::v_cmp_lt_u64:
 660                op = aco_opcode::v_cmp_gt_u64;
 661                break;
 662             case aco_opcode::v_cmp_ge_u64:
 663                op = aco_opcode::v_cmp_le_u64;
 664                break;
 665             default: /* eq and ne are commutative */
 666                break;
 667          }
 668          Temp t = src0;
 669          src0 = src1;
 670          src1 = t;
 671       } else {
 672          src1 = as_vgpr(ctx, src1);
 673       }
 674    }
 675
 676    Builder bld(ctx->program, ctx->block);
 677    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 678 }
 679
 680 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 681 {
 682    Temp src0 = get_alu_src(ctx, instr->src[0]);
 683    Temp src1 = get_alu_src(ctx, instr->src[1]);
 684    Builder bld(ctx->program, ctx->block);
 685
 686    assert(dst.regClass() == bld.lm);
 687    assert(src0.type() == RegType::sgpr);
 688    assert(src1.type() == RegType::sgpr);
 689    assert(src0.regClass() == src1.regClass());
 690
 691    /* Emit the SALU comparison instruction */
 692    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 693    /* Turn the result into a per-lane bool */
 694    bool_to_vector_condition(ctx, cmp, dst);
 695 }
 696
 697 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 698                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 699 {
 700    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 701    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 702    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 703    bool use_valu = s_op == aco_opcode::num_opcodes ||
 704                    divergent_vals ||
 705                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 706                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 707    aco_opcode op = use_valu ? v_op : s_op;
 708    assert(op != aco_opcode::num_opcodes);
 709    assert(dst.regClass() == ctx->program->lane_mask);
 710
 711    if (use_valu)
 712       emit_vopc_instruction(ctx, instr, op, dst);
 713    else
 714       emit_sopc_instruction(ctx, instr, op, dst);
 715 }
 716
 717 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 718 {
 719    Builder bld(ctx->program, ctx->block);
 720    Temp src0 = get_alu_src(ctx, instr->src[0]);
 721    Temp src1 = get_alu_src(ctx, instr->src[1]);
 722
 723    assert(dst.regClass() == bld.lm);
 724    assert(src0.regClass() == bld.lm);
 725    assert(src1.regClass() == bld.lm);
 726
 727    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 728 }
 729
 730 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 731 {
 732    Builder bld(ctx->program, ctx->block);
 733    Temp cond = get_alu_src(ctx, instr->src[0]);
 734    Temp then = get_alu_src(ctx, instr->src[1]);
 735    Temp els = get_alu_src(ctx, instr->src[2]);
 736
 737    assert(cond.regClass() == bld.lm);
 738
 739    if (dst.type() == RegType::vgpr) {
 740       aco_ptr<Instruction> bcsel;
 741       if (dst.size() == 1) {
 742          then = as_vgpr(ctx, then);
 743          els = as_vgpr(ctx, els);
 744
 745          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 746       } else if (dst.size() == 2) {
 747          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 748          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 749          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 750          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 751
 752          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 753          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 754
 755          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 756       } else {
 757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 758          nir_print_instr(&instr->instr, stderr);
 759          fprintf(stderr, "\n");
 760       }
 761       return;
 762    }
 763
 764    if (instr->dest.dest.ssa.bit_size == 1) {
 765       assert(dst.regClass() == bld.lm);
 766       assert(then.regClass() == bld.lm);
 767       assert(els.regClass() == bld.lm);
 768    }
 769
 770    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 771       if (dst.regClass() == s1 || dst.regClass() == s2) {
 772          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 773          assert(dst.size() == then.size());
 774          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 775          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 776       } else {
 777          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 778          nir_print_instr(&instr->instr, stderr);
 779          fprintf(stderr, "\n");
 780       }
 781       return;
 782    }
 783
 784    /* divergent boolean bcsel
 785     * this implements bcsel on bools: dst = s0 ? s1 : s2
 786     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 787    assert(instr->dest.dest.ssa.bit_size == 1);
 788
 789    if (cond.id() != then.id())
 790       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 791
 792    if (cond.id() == els.id())
 793       bld.sop1(Builder::s_mov, Definition(dst), then);
 794    else
 795       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 796                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 797 }
 798
 799 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 800                     aco_opcode op, uint32_t undo)
 801 {
 802    /* multiply by 16777216 to handle denormals */
 803    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 804                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 805    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 806    scaled = bld.vop1(op, bld.def(v1), scaled);
 807    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 808
 809    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 810
 811    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 812 }
 813
 814 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 815 {
 816    if (ctx->block->fp_mode.denorm32 == 0) {
 817       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 818       return;
 819    }
 820
 821    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 822 }
 823
 824 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 825 {
 826    if (ctx->block->fp_mode.denorm32 == 0) {
 827       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 828       return;
 829    }
 830
 831    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 832 }
 833
 834 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 835 {
 836    if (ctx->block->fp_mode.denorm32 == 0) {
 837       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 838       return;
 839    }
 840
 841    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 842 }
 843
 844 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 845 {
 846    if (ctx->block->fp_mode.denorm32 == 0) {
 847       bld.vop1(aco_opcode::v_log_f32, dst, val);
 848       return;
 849    }
 850
 851    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 852 }
 853
 854 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 855 {
 856    if (ctx->options->chip_class >= GFX7)
 857       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 858
 859    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 860    /* TODO: create more efficient code! */
 861    if (val.type() == RegType::sgpr)
 862       val = as_vgpr(ctx, val);
 863
 864    /* Split the input value. */
 865    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 866    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 867
 868    /* Extract the exponent and compute the unbiased value. */
 869    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 870
 871    /* Extract the fractional part. */
 872    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 873    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 874
 875    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 876    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 877
 878    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 879    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 880    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 881    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 882    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 883
 884    /* Get the sign bit. */
 885    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 886
 887    /* Decide the operation to apply depending on the unbiased exponent. */
 888    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 889    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 890    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 891    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 892    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 893    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 894
 895    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 896 }
 897
 898 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 899 {
 900    if (ctx->options->chip_class >= GFX7)
 901       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 902
 903    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 904    Temp src0 = as_vgpr(ctx, val);
 905
 906    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 907    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 908
 909    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 910    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 911    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 912
 913    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 914    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 915    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 916    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 917
 918    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 919    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 920
 921    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 922
 923    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 924    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 925
 926    return add->definitions[0].getTemp();
 927 }
 928
 929 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 930 {
 931    if (!instr->dest.dest.is_ssa) {
 932       fprintf(stderr, "nir alu dst not in ssa: ");
 933       nir_print_instr(&instr->instr, stderr);
 934       fprintf(stderr, "\n");
 935       abort();
 936    }
 937    Builder bld(ctx->program, ctx->block);
 938    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 939    switch(instr->op) {
 940    case nir_op_vec2:
 941    case nir_op_vec3:
 942    case nir_op_vec4: {
 943       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 944       unsigned num = instr->dest.dest.ssa.num_components;
 945       for (unsigned i = 0; i < num; ++i)
 946          elems[i] = get_alu_src(ctx, instr->src[i]);
 947
 948       if (instr->dest.dest.ssa.bit_size >= 32 || dst.type() == RegType::vgpr) {
 949          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 950          for (unsigned i = 0; i < num; ++i)
 951             vec->operands[i] = Operand{elems[i]};
 952          vec->definitions[0] = Definition(dst);
 953          ctx->block->instructions.emplace_back(std::move(vec));
 954          ctx->allocated_vec.emplace(dst.id(), elems);
 955       } else {
 956          // TODO: that is a bit suboptimal..
 957          Temp mask = bld.copy(bld.def(s1), Operand((1u << instr->dest.dest.ssa.bit_size) - 1));
 958          for (unsigned i = 0; i < num - 1; ++i)
 959             if (((i+1) * instr->dest.dest.ssa.bit_size) % 32)
 960                elems[i] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), elems[i], mask);
 961          for (unsigned i = 0; i < num; ++i) {
 962             unsigned bit = i * instr->dest.dest.ssa.bit_size;
 963             if (bit % 32 == 0) {
 964                elems[bit / 32] = elems[i];
 965             } else {
 966                elems[i] = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc),
 967                                    elems[i], Operand((i * instr->dest.dest.ssa.bit_size) % 32));
 968                elems[bit / 32] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), elems[bit / 32], elems[i]);
 969             }
 970          }
 971          if (dst.size() == 1)
 972             bld.copy(Definition(dst), elems[0]);
 973          else
 974             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), elems[0], elems[1]);
 975       }
 976       break;
 977    }
 978    case nir_op_mov: {
 979       Temp src = get_alu_src(ctx, instr->src[0]);
 980       aco_ptr<Instruction> mov;
 981       if (dst.type() == RegType::sgpr) {
 982          if (src.type() == RegType::vgpr)
 983             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 984          else if (src.regClass() == s1)
 985             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 986          else if (src.regClass() == s2)
 987             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 988          else
 989             unreachable("wrong src register class for nir_op_imov");
 990       } else if (dst.regClass() == v1) {
 991          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 992       } else if (dst.regClass() == v2) {
 993          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 994       } else {
 995          nir_print_instr(&instr->instr, stderr);
 996          unreachable("Should have been lowered to scalar.");
 997       }
 998       break;
 999    }
1000    case nir_op_inot: {
1001       Temp src = get_alu_src(ctx, instr->src[0]);
1002       if (instr->dest.dest.ssa.bit_size == 1) {
1003          assert(src.regClass() == bld.lm);
1004          assert(dst.regClass() == bld.lm);
1005          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
1006          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
1007          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
1008       } else if (dst.regClass() == v1) {
1009          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
1010       } else if (dst.type() == RegType::sgpr) {
1011          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
1012          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
1013       } else {
1014          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1015          nir_print_instr(&instr->instr, stderr);
1016          fprintf(stderr, "\n");
1017       }
1018       break;
1019    }
1020    case nir_op_ineg: {
1021       Temp src = get_alu_src(ctx, instr->src[0]);
1022       if (dst.regClass() == v1) {
1023          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
1024       } else if (dst.regClass() == s1) {
1025          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
1026       } else if (dst.size() == 2) {
1027          Temp src0 = bld.tmp(dst.type(), 1);
1028          Temp src1 = bld.tmp(dst.type(), 1);
1029          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
1030
1031          if (dst.regClass() == s2) {
1032             Temp carry = bld.tmp(s1);
1033             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
1034             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
1035             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1036          } else {
1037             Temp lower = bld.tmp(v1);
1038             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
1039             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
1040             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1041          }
1042       } else {
1043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1044          nir_print_instr(&instr->instr, stderr);
1045          fprintf(stderr, "\n");
1046       }
1047       break;
1048    }
1049    case nir_op_iabs: {
1050       if (dst.regClass() == s1) {
1051          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
1052       } else if (dst.regClass() == v1) {
1053          Temp src = get_alu_src(ctx, instr->src[0]);
1054          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
1055       } else {
1056          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1057          nir_print_instr(&instr->instr, stderr);
1058          fprintf(stderr, "\n");
1059       }
1060       break;
1061    }
1062    case nir_op_isign: {
1063       Temp src = get_alu_src(ctx, instr->src[0]);
1064       if (dst.regClass() == s1) {
1065          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1066          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
1067          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
1068       } else if (dst.regClass() == s2) {
1069          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
1070          Temp neqz;
1071          if (ctx->program->chip_class >= GFX8)
1072             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
1073          else
1074             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
1075          /* SCC gets zero-extended to 64 bit */
1076          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
1077       } else if (dst.regClass() == v1) {
1078          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1079          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1080          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
1081       } else if (dst.regClass() == v2) {
1082          Temp upper = emit_extract_vector(ctx, src, 1, v1);
1083          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
1084          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1085          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
1086          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
1087          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1088       } else {
1089          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1090          nir_print_instr(&instr->instr, stderr);
1091          fprintf(stderr, "\n");
1092       }
1093       break;
1094    }
1095    case nir_op_imax: {
1096       if (dst.regClass() == v1) {
1097          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
1098       } else if (dst.regClass() == s1) {
1099          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
1100       } else {
1101          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1102          nir_print_instr(&instr->instr, stderr);
1103          fprintf(stderr, "\n");
1104       }
1105       break;
1106    }
1107    case nir_op_umax: {
1108       if (dst.regClass() == v1) {
1109          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
1110       } else if (dst.regClass() == s1) {
1111          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
1112       } else {
1113          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1114          nir_print_instr(&instr->instr, stderr);
1115          fprintf(stderr, "\n");
1116       }
1117       break;
1118    }
1119    case nir_op_imin: {
1120       if (dst.regClass() == v1) {
1121          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
1122       } else if (dst.regClass() == s1) {
1123          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
1124       } else {
1125          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1126          nir_print_instr(&instr->instr, stderr);
1127          fprintf(stderr, "\n");
1128       }
1129       break;
1130    }
1131    case nir_op_umin: {
1132       if (dst.regClass() == v1) {
1133          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
1134       } else if (dst.regClass() == s1) {
1135          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1136       } else {
1137          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1138          nir_print_instr(&instr->instr, stderr);
1139          fprintf(stderr, "\n");
1140       }
1141       break;
1142    }
1143    case nir_op_ior: {
1144       if (instr->dest.dest.ssa.bit_size == 1) {
1145          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1146       } else if (dst.regClass() == v1) {
1147          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1148       } else if (dst.regClass() == s1) {
1149          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1150       } else if (dst.regClass() == s2) {
1151          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1152       } else {
1153          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1154          nir_print_instr(&instr->instr, stderr);
1155          fprintf(stderr, "\n");
1156       }
1157       break;
1158    }
1159    case nir_op_iand: {
1160       if (instr->dest.dest.ssa.bit_size == 1) {
1161          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1162       } else if (dst.regClass() == v1) {
1163          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1164       } else if (dst.regClass() == s1) {
1165          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1166       } else if (dst.regClass() == s2) {
1167          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1168       } else {
1169          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1170          nir_print_instr(&instr->instr, stderr);
1171          fprintf(stderr, "\n");
1172       }
1173       break;
1174    }
1175    case nir_op_ixor: {
1176       if (instr->dest.dest.ssa.bit_size == 1) {
1177          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1178       } else if (dst.regClass() == v1) {
1179          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1180       } else if (dst.regClass() == s1) {
1181          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1182       } else if (dst.regClass() == s2) {
1183          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1184       } else {
1185          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1186          nir_print_instr(&instr->instr, stderr);
1187          fprintf(stderr, "\n");
1188       }
1189       break;
1190    }
1191    case nir_op_ushr: {
1192       if (dst.regClass() == v1) {
1193          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1194       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1195          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1196                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1197       } else if (dst.regClass() == v2) {
1198          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1199                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1200       } else if (dst.regClass() == s2) {
1201          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1202       } else if (dst.regClass() == s1) {
1203          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1204       } else {
1205          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1206          nir_print_instr(&instr->instr, stderr);
1207          fprintf(stderr, "\n");
1208       }
1209       break;
1210    }
1211    case nir_op_ishl: {
1212       if (dst.regClass() == v1) {
1213          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1214       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1215          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1216                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1217       } else if (dst.regClass() == v2) {
1218          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1219                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1220       } else if (dst.regClass() == s1) {
1221          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1222       } else if (dst.regClass() == s2) {
1223          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1224       } else {
1225          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1226          nir_print_instr(&instr->instr, stderr);
1227          fprintf(stderr, "\n");
1228       }
1229       break;
1230    }
1231    case nir_op_ishr: {
1232       if (dst.regClass() == v1) {
1233          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1234       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1235          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1236                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1237       } else if (dst.regClass() == v2) {
1238          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1239                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1240       } else if (dst.regClass() == s1) {
1241          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1242       } else if (dst.regClass() == s2) {
1243          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1244       } else {
1245          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1246          nir_print_instr(&instr->instr, stderr);
1247          fprintf(stderr, "\n");
1248       }
1249       break;
1250    }
1251    case nir_op_find_lsb: {
1252       Temp src = get_alu_src(ctx, instr->src[0]);
1253       if (src.regClass() == s1) {
1254          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1255       } else if (src.regClass() == v1) {
1256          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1257       } else if (src.regClass() == s2) {
1258          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1259       } else {
1260          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1261          nir_print_instr(&instr->instr, stderr);
1262          fprintf(stderr, "\n");
1263       }
1264       break;
1265    }
1266    case nir_op_ufind_msb:
1267    case nir_op_ifind_msb: {
1268       Temp src = get_alu_src(ctx, instr->src[0]);
1269       if (src.regClass() == s1 || src.regClass() == s2) {
1270          aco_opcode op = src.regClass() == s2 ?
1271                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1272                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1273          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1274
1275          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1276                                         Operand(src.size() * 32u - 1u), msb_rev);
1277          Temp msb = sub.def(0).getTemp();
1278          Temp carry = sub.def(1).getTemp();
1279
1280          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1281       } else if (src.regClass() == v1) {
1282          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1283          Temp msb_rev = bld.tmp(v1);
1284          emit_vop1_instruction(ctx, instr, op, msb_rev);
1285          Temp msb = bld.tmp(v1);
1286          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1287          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1288       } else {
1289          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1290          nir_print_instr(&instr->instr, stderr);
1291          fprintf(stderr, "\n");
1292       }
1293       break;
1294    }
1295    case nir_op_bitfield_reverse: {
1296       if (dst.regClass() == s1) {
1297          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1298       } else if (dst.regClass() == v1) {
1299          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1300       } else {
1301          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1302          nir_print_instr(&instr->instr, stderr);
1303          fprintf(stderr, "\n");
1304       }
1305       break;
1306    }
1307    case nir_op_iadd: {
1308       if (dst.regClass() == s1) {
1309          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1310          break;
1311       }
1312
1313       Temp src0 = get_alu_src(ctx, instr->src[0]);
1314       Temp src1 = get_alu_src(ctx, instr->src[1]);
1315       if (dst.regClass() == v1) {
1316          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1317          break;
1318       }
1319
1320       assert(src0.size() == 2 && src1.size() == 2);
1321       Temp src00 = bld.tmp(src0.type(), 1);
1322       Temp src01 = bld.tmp(dst.type(), 1);
1323       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1324       Temp src10 = bld.tmp(src1.type(), 1);
1325       Temp src11 = bld.tmp(dst.type(), 1);
1326       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1327
1328       if (dst.regClass() == s2) {
1329          Temp carry = bld.tmp(s1);
1330          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1331          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1332          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1333       } else if (dst.regClass() == v2) {
1334          Temp dst0 = bld.tmp(v1);
1335          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1336          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1337          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_uadd_sat: {
1346       Temp src0 = get_alu_src(ctx, instr->src[0]);
1347       Temp src1 = get_alu_src(ctx, instr->src[1]);
1348       if (dst.regClass() == s1) {
1349          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1350          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1351                   src0, src1);
1352          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1353       } else if (dst.regClass() == v1) {
1354          if (ctx->options->chip_class >= GFX9) {
1355             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1356             add->operands[0] = Operand(src0);
1357             add->operands[1] = Operand(src1);
1358             add->definitions[0] = Definition(dst);
1359             add->clamp = 1;
1360             ctx->block->instructions.emplace_back(std::move(add));
1361          } else {
1362             if (src1.regClass() != v1)
1363                std::swap(src0, src1);
1364             assert(src1.regClass() == v1);
1365             Temp tmp = bld.tmp(v1);
1366             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1367             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1368          }
1369       } else {
1370          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1371          nir_print_instr(&instr->instr, stderr);
1372          fprintf(stderr, "\n");
1373       }
1374       break;
1375    }
1376    case nir_op_uadd_carry: {
1377       Temp src0 = get_alu_src(ctx, instr->src[0]);
1378       Temp src1 = get_alu_src(ctx, instr->src[1]);
1379       if (dst.regClass() == s1) {
1380          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1381          break;
1382       }
1383       if (dst.regClass() == v1) {
1384          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1385          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1386          break;
1387       }
1388
1389       Temp src00 = bld.tmp(src0.type(), 1);
1390       Temp src01 = bld.tmp(dst.type(), 1);
1391       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1392       Temp src10 = bld.tmp(src1.type(), 1);
1393       Temp src11 = bld.tmp(dst.type(), 1);
1394       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1395       if (dst.regClass() == s2) {
1396          Temp carry = bld.tmp(s1);
1397          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1398          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1399          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1400       } else if (dst.regClass() == v2) {
1401          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1402          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1403          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1404          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1405       } else {
1406          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1407          nir_print_instr(&instr->instr, stderr);
1408          fprintf(stderr, "\n");
1409       }
1410       break;
1411    }
1412    case nir_op_isub: {
1413       if (dst.regClass() == s1) {
1414          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1415          break;
1416       }
1417
1418       Temp src0 = get_alu_src(ctx, instr->src[0]);
1419       Temp src1 = get_alu_src(ctx, instr->src[1]);
1420       if (dst.regClass() == v1) {
1421          bld.vsub32(Definition(dst), src0, src1);
1422          break;
1423       }
1424
1425       Temp src00 = bld.tmp(src0.type(), 1);
1426       Temp src01 = bld.tmp(dst.type(), 1);
1427       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1428       Temp src10 = bld.tmp(src1.type(), 1);
1429       Temp src11 = bld.tmp(dst.type(), 1);
1430       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1431       if (dst.regClass() == s2) {
1432          Temp carry = bld.tmp(s1);
1433          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1434          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1435          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1436       } else if (dst.regClass() == v2) {
1437          Temp lower = bld.tmp(v1);
1438          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1439          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1440          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1441       } else {
1442          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1443          nir_print_instr(&instr->instr, stderr);
1444          fprintf(stderr, "\n");
1445       }
1446       break;
1447    }
1448    case nir_op_usub_borrow: {
1449       Temp src0 = get_alu_src(ctx, instr->src[0]);
1450       Temp src1 = get_alu_src(ctx, instr->src[1]);
1451       if (dst.regClass() == s1) {
1452          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1453          break;
1454       } else if (dst.regClass() == v1) {
1455          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1456          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1457          break;
1458       }
1459
1460       Temp src00 = bld.tmp(src0.type(), 1);
1461       Temp src01 = bld.tmp(dst.type(), 1);
1462       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1463       Temp src10 = bld.tmp(src1.type(), 1);
1464       Temp src11 = bld.tmp(dst.type(), 1);
1465       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1466       if (dst.regClass() == s2) {
1467          Temp borrow = bld.tmp(s1);
1468          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1469          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1470          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1471       } else if (dst.regClass() == v2) {
1472          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1473          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1474          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1475          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1476       } else {
1477          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1478          nir_print_instr(&instr->instr, stderr);
1479          fprintf(stderr, "\n");
1480       }
1481       break;
1482    }
1483    case nir_op_imul: {
1484       if (dst.regClass() == v1) {
1485          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1486                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1487       } else if (dst.regClass() == s1) {
1488          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1489       } else {
1490          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1491          nir_print_instr(&instr->instr, stderr);
1492          fprintf(stderr, "\n");
1493       }
1494       break;
1495    }
1496    case nir_op_umul_high: {
1497       if (dst.regClass() == v1) {
1498          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1499       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1500          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1501       } else if (dst.regClass() == s1) {
1502          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1503                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1504          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1505       } else {
1506          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1507          nir_print_instr(&instr->instr, stderr);
1508          fprintf(stderr, "\n");
1509       }
1510       break;
1511    }
1512    case nir_op_imul_high: {
1513       if (dst.regClass() == v1) {
1514          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1515       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1516          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1517       } else if (dst.regClass() == s1) {
1518          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1519                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1520          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1521       } else {
1522          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1523          nir_print_instr(&instr->instr, stderr);
1524          fprintf(stderr, "\n");
1525       }
1526       break;
1527    }
1528    case nir_op_fmul: {
1529       if (dst.size() == 1) {
1530          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1531       } else if (dst.size() == 2) {
1532          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1533                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1534       } else {
1535          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1536          nir_print_instr(&instr->instr, stderr);
1537          fprintf(stderr, "\n");
1538       }
1539       break;
1540    }
1541    case nir_op_fadd: {
1542       Temp src0 = get_alu_src(ctx, instr->src[0]);
1543       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1544       if (dst.regClass() == v2b) {
1545          Temp tmp = bld.tmp(v1);
1546          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f16, tmp, true);
1547          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1548       } else if (dst.regClass() == v1) {
1549          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1550       } else if (dst.regClass() == v2) {
1551          bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, src1);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_fsub: {
1560       Temp src0 = get_alu_src(ctx, instr->src[0]);
1561       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1562       if (dst.regClass() == v2b) {
1563          Temp tmp = bld.tmp(v1);
1564          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1565             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f16, tmp, false);
1566          else
1567             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f16, tmp, true);
1568          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1569       } else if (dst.regClass() == v1) {
1570          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1571             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1572          else
1573             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1574       } else if (dst.regClass() == v2) {
1575          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1576                                      src0, src1);
1577          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1578          sub->neg[1] = true;
1579       } else {
1580          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1581          nir_print_instr(&instr->instr, stderr);
1582          fprintf(stderr, "\n");
1583       }
1584       break;
1585    }
1586    case nir_op_fmax: {
1587       Temp src0 = get_alu_src(ctx, instr->src[0]);
1588       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1589       if (dst.regClass() == v2b) {
1590          // TODO: check fp_mode.must_flush_denorms16_64
1591          Temp tmp = bld.tmp(v1);
1592          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f16, tmp, true);
1593          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1594       } else if (dst.regClass() == v1) {
1595          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1596       } else if (dst.regClass() == v2) {
1597          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1598             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2), src0, src1);
1599             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1600          } else {
1601             bld.vop3(aco_opcode::v_max_f64, Definition(dst), src0, src1);
1602          }
1603       } else {
1604          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1605          nir_print_instr(&instr->instr, stderr);
1606          fprintf(stderr, "\n");
1607       }
1608       break;
1609    }
1610    case nir_op_fmin: {
1611       Temp src0 = get_alu_src(ctx, instr->src[0]);
1612       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1613       if (dst.regClass() == v2b) {
1614          // TODO: check fp_mode.must_flush_denorms16_64
1615          Temp tmp = bld.tmp(v1);
1616          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f16, tmp, true);
1617          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1618       } else if (dst.regClass() == v1) {
1619          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1620       } else if (dst.regClass() == v2) {
1621          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1622             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), src0, src1);
1623             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1624          } else {
1625             bld.vop3(aco_opcode::v_min_f64, Definition(dst), src0, src1);
1626          }
1627       } else {
1628          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1629          nir_print_instr(&instr->instr, stderr);
1630          fprintf(stderr, "\n");
1631       }
1632       break;
1633    }
1634    case nir_op_fmax3: {
1635       if (dst.size() == 1) {
1636          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1637       } else {
1638          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1639          nir_print_instr(&instr->instr, stderr);
1640          fprintf(stderr, "\n");
1641       }
1642       break;
1643    }
1644    case nir_op_fmin3: {
1645       if (dst.size() == 1) {
1646          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1647       } else {
1648          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1649          nir_print_instr(&instr->instr, stderr);
1650          fprintf(stderr, "\n");
1651       }
1652       break;
1653    }
1654    case nir_op_fmed3: {
1655       if (dst.size() == 1) {
1656          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_umax3: {
1665       if (dst.size() == 1) {
1666          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_umin3: {
1675       if (dst.size() == 1) {
1676          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1677       } else {
1678          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1679          nir_print_instr(&instr->instr, stderr);
1680          fprintf(stderr, "\n");
1681       }
1682       break;
1683    }
1684    case nir_op_umed3: {
1685       if (dst.size() == 1) {
1686          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1687       } else {
1688          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1689          nir_print_instr(&instr->instr, stderr);
1690          fprintf(stderr, "\n");
1691       }
1692       break;
1693    }
1694    case nir_op_imax3: {
1695       if (dst.size() == 1) {
1696          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1697       } else {
1698          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1699          nir_print_instr(&instr->instr, stderr);
1700          fprintf(stderr, "\n");
1701       }
1702       break;
1703    }
1704    case nir_op_imin3: {
1705       if (dst.size() == 1) {
1706          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1707       } else {
1708          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1709          nir_print_instr(&instr->instr, stderr);
1710          fprintf(stderr, "\n");
1711       }
1712       break;
1713    }
1714    case nir_op_imed3: {
1715       if (dst.size() == 1) {
1716          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1717       } else {
1718          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1719          nir_print_instr(&instr->instr, stderr);
1720          fprintf(stderr, "\n");
1721       }
1722       break;
1723    }
1724    case nir_op_cube_face_coord: {
1725       Temp in = get_alu_src(ctx, instr->src[0], 3);
1726       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1727                       emit_extract_vector(ctx, in, 1, v1),
1728                       emit_extract_vector(ctx, in, 2, v1) };
1729       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1730       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1731       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1732       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1733       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1734       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1735       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1736       break;
1737    }
1738    case nir_op_cube_face_index: {
1739       Temp in = get_alu_src(ctx, instr->src[0], 3);
1740       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1741                       emit_extract_vector(ctx, in, 1, v1),
1742                       emit_extract_vector(ctx, in, 2, v1) };
1743       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1744       break;
1745    }
1746    case nir_op_bcsel: {
1747       emit_bcsel(ctx, instr, dst);
1748       break;
1749    }
1750    case nir_op_frsq: {
1751       Temp src = get_alu_src(ctx, instr->src[0]);
1752       if (dst.regClass() == v2b) {
1753          Temp tmp = bld.vop1(aco_opcode::v_rsq_f16, bld.def(v1), src);
1754          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1755       } else if (dst.regClass() == v1) {
1756          emit_rsq(ctx, bld, Definition(dst), src);
1757       } else if (dst.regClass() == v2) {
1758          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1759       } else {
1760          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1761          nir_print_instr(&instr->instr, stderr);
1762          fprintf(stderr, "\n");
1763       }
1764       break;
1765    }
1766    case nir_op_fneg: {
1767       Temp src = get_alu_src(ctx, instr->src[0]);
1768       if (dst.regClass() == v2b) {
1769          Temp tmp = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x8000u), as_vgpr(ctx, src));
1770          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1771       } else if (dst.regClass() == v1) {
1772          if (ctx->block->fp_mode.must_flush_denorms32)
1773             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1774          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1775       } else if (dst.regClass() == v2) {
1776          if (ctx->block->fp_mode.must_flush_denorms16_64)
1777             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1778          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1779          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1780          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1781          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1782       } else {
1783          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1784          nir_print_instr(&instr->instr, stderr);
1785          fprintf(stderr, "\n");
1786       }
1787       break;
1788    }
1789    case nir_op_fabs: {
1790       Temp src = get_alu_src(ctx, instr->src[0]);
1791       if (dst.regClass() == v2b) {
1792          Temp tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFu), as_vgpr(ctx, src));
1793          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1794       } else if (dst.regClass() == v1) {
1795          if (ctx->block->fp_mode.must_flush_denorms32)
1796             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1797          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1798       } else if (dst.regClass() == v2) {
1799          if (ctx->block->fp_mode.must_flush_denorms16_64)
1800             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1801          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1802          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1803          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1804          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1805       } else {
1806          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1807          nir_print_instr(&instr->instr, stderr);
1808          fprintf(stderr, "\n");
1809       }
1810       break;
1811    }
1812    case nir_op_fsat: {
1813       Temp src = get_alu_src(ctx, instr->src[0]);
1814       if (dst.size() == 1) {
1815          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1816          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1817          // TODO: confirm that this holds under any circumstances
1818       } else if (dst.size() == 2) {
1819          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1820          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1821          vop3->clamp = true;
1822       } else {
1823          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1824          nir_print_instr(&instr->instr, stderr);
1825          fprintf(stderr, "\n");
1826       }
1827       break;
1828    }
1829    case nir_op_flog2: {
1830       Temp src = get_alu_src(ctx, instr->src[0]);
1831       if (dst.regClass() == v2b) {
1832          Temp tmp = bld.vop1(aco_opcode::v_log_f16, bld.def(v1), src);
1833          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1834       } else if (dst.regClass() == v1) {
1835          emit_log2(ctx, bld, Definition(dst), src);
1836       } else {
1837          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1838          nir_print_instr(&instr->instr, stderr);
1839          fprintf(stderr, "\n");
1840       }
1841       break;
1842    }
1843    case nir_op_frcp: {
1844       Temp src = get_alu_src(ctx, instr->src[0]);
1845       if (dst.regClass() == v2b) {
1846          Temp tmp = bld.vop1(aco_opcode::v_rcp_f16, bld.def(v1), src);
1847          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1848       } else if (dst.regClass() == v1) {
1849          emit_rcp(ctx, bld, Definition(dst), src);
1850       } else if (dst.regClass() == v2) {
1851          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1852       } else {
1853          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1854          nir_print_instr(&instr->instr, stderr);
1855          fprintf(stderr, "\n");
1856       }
1857       break;
1858    }
1859    case nir_op_fexp2: {
1860       if (dst.regClass() == v2b) {
1861          Temp src = get_alu_src(ctx, instr->src[0]);
1862          Temp tmp = bld.vop1(aco_opcode::v_exp_f16, bld.def(v1), src);
1863          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1864       } else if (dst.regClass() == v1) {
1865          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1866       } else {
1867          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1868          nir_print_instr(&instr->instr, stderr);
1869          fprintf(stderr, "\n");
1870       }
1871       break;
1872    }
1873    case nir_op_fsqrt: {
1874       Temp src = get_alu_src(ctx, instr->src[0]);
1875       if (dst.regClass() == v2b) {
1876          Temp tmp = bld.vop1(aco_opcode::v_sqrt_f16, bld.def(v1), src);
1877          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1878       } else if (dst.regClass() == v1) {
1879          emit_sqrt(ctx, bld, Definition(dst), src);
1880       } else if (dst.regClass() == v2) {
1881          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1882       } else {
1883          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1884          nir_print_instr(&instr->instr, stderr);
1885          fprintf(stderr, "\n");
1886       }
1887       break;
1888    }
1889    case nir_op_ffract: {
1890       if (dst.regClass() == v2b) {
1891          Temp src = get_alu_src(ctx, instr->src[0]);
1892          Temp tmp = bld.vop1(aco_opcode::v_fract_f16, bld.def(v1), src);
1893          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1894       } else if (dst.regClass() == v1) {
1895          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1896       } else if (dst.regClass() == v2) {
1897          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1898       } else {
1899          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1900          nir_print_instr(&instr->instr, stderr);
1901          fprintf(stderr, "\n");
1902       }
1903       break;
1904    }
1905    case nir_op_ffloor: {
1906       Temp src = get_alu_src(ctx, instr->src[0]);
1907       if (dst.regClass() == v2b) {
1908          Temp tmp = bld.vop1(aco_opcode::v_floor_f16, bld.def(v1), src);
1909          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1910       } else if (dst.regClass() == v1) {
1911          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1912       } else if (dst.regClass() == v2) {
1913          emit_floor_f64(ctx, bld, Definition(dst), src);
1914       } else {
1915          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1916          nir_print_instr(&instr->instr, stderr);
1917          fprintf(stderr, "\n");
1918       }
1919       break;
1920    }
1921    case nir_op_fceil: {
1922       Temp src0 = get_alu_src(ctx, instr->src[0]);
1923       if (dst.regClass() == v2b) {
1924          Temp tmp = bld.vop1(aco_opcode::v_ceil_f16, bld.def(v1), src0);
1925          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1926       } else if (dst.regClass() == v1) {
1927          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1928       } else if (dst.regClass() == v2) {
1929          if (ctx->options->chip_class >= GFX7) {
1930             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1931          } else {
1932             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1933             /* trunc = trunc(src0)
1934              * if (src0 > 0.0 && src0 != trunc)
1935              *    trunc += 1.0
1936              */
1937             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1938             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1939             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1940             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1941             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1942             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1943             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1944          }
1945       } else {
1946          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1947          nir_print_instr(&instr->instr, stderr);
1948          fprintf(stderr, "\n");
1949       }
1950       break;
1951    }
1952    case nir_op_ftrunc: {
1953       Temp src = get_alu_src(ctx, instr->src[0]);
1954       if (dst.regClass() == v2b) {
1955          Temp tmp = bld.vop1(aco_opcode::v_trunc_f16, bld.def(v1), src);
1956          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1957       } else if (dst.regClass() == v1) {
1958          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1959       } else if (dst.regClass() == v2) {
1960          emit_trunc_f64(ctx, bld, Definition(dst), src);
1961       } else {
1962          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1963          nir_print_instr(&instr->instr, stderr);
1964          fprintf(stderr, "\n");
1965       }
1966       break;
1967    }
1968    case nir_op_fround_even: {
1969       Temp src0 = get_alu_src(ctx, instr->src[0]);
1970       if (dst.regClass() == v2b) {
1971          Temp tmp = bld.vop1(aco_opcode::v_rndne_f16, bld.def(v1), src0);
1972          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1973       } else if (dst.regClass() == v1) {
1974          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1975       } else if (dst.regClass() == v2) {
1976          if (ctx->options->chip_class >= GFX7) {
1977             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1978          } else {
1979             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1980             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1981             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1982
1983             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1984             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1985             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1986             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1987             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1988             tmp = sub->definitions[0].getTemp();
1989
1990             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1991             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1992             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1993             Temp cond = vop3->definitions[0].getTemp();
1994
1995             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1996             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1997             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1998             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1999
2000             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
2001          }
2002       } else {
2003          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2004          nir_print_instr(&instr->instr, stderr);
2005          fprintf(stderr, "\n");
2006       }
2007       break;
2008    }
2009    case nir_op_fsin:
2010    case nir_op_fcos: {
2011       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2012       aco_ptr<Instruction> norm;
2013       Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
2014       if (dst.regClass() == v2b) {
2015          Temp tmp = bld.vop2(aco_opcode::v_mul_f16, bld.def(v1), half_pi, src);
2016          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f16 : aco_opcode::v_cos_f16;
2017          tmp = bld.vop1(opcode, bld.def(v1), tmp);
2018          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2019       } else if (dst.regClass() == v1) {
2020          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
2021
2022          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
2023          if (ctx->options->chip_class < GFX9)
2024             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
2025
2026          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
2027          bld.vop1(opcode, Definition(dst), tmp);
2028       } else {
2029          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2030          nir_print_instr(&instr->instr, stderr);
2031          fprintf(stderr, "\n");
2032       }
2033       break;
2034    }
2035    case nir_op_ldexp: {
2036       if (dst.size() == 1) {
2037          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
2038                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
2039                   get_alu_src(ctx, instr->src[1]));
2040       } else if (dst.size() == 2) {
2041          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
2042                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
2043                   get_alu_src(ctx, instr->src[1]));
2044       } else {
2045          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2046          nir_print_instr(&instr->instr, stderr);
2047          fprintf(stderr, "\n");
2048       }
2049       break;
2050    }
2051    case nir_op_frexp_sig: {
2052       Temp src = get_alu_src(ctx, instr->src[0]);
2053       if (dst.regClass() == v2b) {
2054          Temp tmp = bld.vop1(aco_opcode::v_frexp_mant_f16, bld.def(v1), src);
2055          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2056       } else if (dst.regClass() == v1) {
2057          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst), src);
2058       } else if (dst.regClass() == v2) {
2059          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst), src);
2060       } else {
2061          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2062          nir_print_instr(&instr->instr, stderr);
2063          fprintf(stderr, "\n");
2064       }
2065       break;
2066    }
2067    case nir_op_frexp_exp: {
2068       Temp src = get_alu_src(ctx, instr->src[0]);
2069       if (instr->src[0].src.ssa->bit_size == 16) {
2070          Temp tmp = bld.vop1(aco_opcode::v_frexp_exp_i16_f16, bld.def(v1), src);
2071          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), tmp, Operand(0u));
2072       } else if (instr->src[0].src.ssa->bit_size == 32) {
2073          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst), src);
2074       } else if (instr->src[0].src.ssa->bit_size == 64) {
2075          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst), src);
2076       } else {
2077          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2078          nir_print_instr(&instr->instr, stderr);
2079          fprintf(stderr, "\n");
2080       }
2081       break;
2082    }
2083    case nir_op_fsign: {
2084       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2085       if (dst.size() == 1) {
2086          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2087          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
2088          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2089          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
2090       } else if (dst.size() == 2) {
2091          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2092          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
2093          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
2094
2095          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2096          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
2097          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
2098
2099          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2100       } else {
2101          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2102          nir_print_instr(&instr->instr, stderr);
2103          fprintf(stderr, "\n");
2104       }
2105       break;
2106    }
2107    case nir_op_f2f16:
2108    case nir_op_f2f16_rtne: {
2109       Temp src = get_alu_src(ctx, instr->src[0]);
2110       if (instr->src[0].src.ssa->bit_size == 64)
2111          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2112       src = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2113       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2114       break;
2115    }
2116    case nir_op_f2f16_rtz: {
2117       Temp src = get_alu_src(ctx, instr->src[0]);
2118       if (instr->src[0].src.ssa->bit_size == 64)
2119          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2120       src = bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, bld.def(v1), src, Operand(0u));
2121       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2122       break;
2123    }
2124    case nir_op_f2f32: {
2125       if (instr->src[0].src.ssa->bit_size == 16) {
2126          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f16, dst);
2127       } else if (instr->src[0].src.ssa->bit_size == 64) {
2128          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
2129       } else {
2130          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2131          nir_print_instr(&instr->instr, stderr);
2132          fprintf(stderr, "\n");
2133       }
2134       break;
2135    }
2136    case nir_op_f2f64: {
2137       Temp src = get_alu_src(ctx, instr->src[0]);
2138       if (instr->src[0].src.ssa->bit_size == 16)
2139          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2140       bld.vop1(aco_opcode::v_cvt_f64_f32, Definition(dst), src);
2141       break;
2142    }
2143    case nir_op_i2f32: {
2144       assert(dst.size() == 1);
2145       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
2146       break;
2147    }
2148    case nir_op_i2f64: {
2149       if (instr->src[0].src.ssa->bit_size == 32) {
2150          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
2151       } else if (instr->src[0].src.ssa->bit_size == 64) {
2152          Temp src = get_alu_src(ctx, instr->src[0]);
2153          RegClass rc = RegClass(src.type(), 1);
2154          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2155          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2156          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2157          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
2158          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2159          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2160
2161       } else {
2162          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2163          nir_print_instr(&instr->instr, stderr);
2164          fprintf(stderr, "\n");
2165       }
2166       break;
2167    }
2168    case nir_op_u2f32: {
2169       assert(dst.size() == 1);
2170       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
2171       break;
2172    }
2173    case nir_op_u2f64: {
2174       if (instr->src[0].src.ssa->bit_size == 32) {
2175          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
2176       } else if (instr->src[0].src.ssa->bit_size == 64) {
2177          Temp src = get_alu_src(ctx, instr->src[0]);
2178          RegClass rc = RegClass(src.type(), 1);
2179          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2180          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2181          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2182          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
2183          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2184          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2185       } else {
2186          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2187          nir_print_instr(&instr->instr, stderr);
2188          fprintf(stderr, "\n");
2189       }
2190       break;
2191    }
2192    case nir_op_f2i16: {
2193       Temp src = get_alu_src(ctx, instr->src[0]);
2194       if (instr->src[0].src.ssa->bit_size == 16)
2195          src = bld.vop1(aco_opcode::v_cvt_i16_f16, bld.def(v1), src);
2196       else if (instr->src[0].src.ssa->bit_size == 32)
2197          src = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src);
2198       else
2199          src = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src);
2200
2201       if (dst.type() == RegType::vgpr)
2202          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2203       else
2204          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2205       break;
2206    }
2207    case nir_op_f2u16: {
2208       Temp src = get_alu_src(ctx, instr->src[0]);
2209       if (instr->src[0].src.ssa->bit_size == 16)
2210          src = bld.vop1(aco_opcode::v_cvt_u16_f16, bld.def(v1), src);
2211       else if (instr->src[0].src.ssa->bit_size == 32)
2212          src = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src);
2213       else
2214          src = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src);
2215
2216       if (dst.type() == RegType::vgpr)
2217          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2218       else
2219          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2220       break;
2221    }
2222    case nir_op_f2i32: {
2223       Temp src = get_alu_src(ctx, instr->src[0]);
2224       if (instr->src[0].src.ssa->bit_size == 32) {
2225          if (dst.type() == RegType::vgpr)
2226             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
2227          else
2228             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2229                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
2230
2231       } else if (instr->src[0].src.ssa->bit_size == 64) {
2232          if (dst.type() == RegType::vgpr)
2233             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
2234          else
2235             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2236                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
2237
2238       } else {
2239          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2240          nir_print_instr(&instr->instr, stderr);
2241          fprintf(stderr, "\n");
2242       }
2243       break;
2244    }
2245    case nir_op_f2u32: {
2246       Temp src = get_alu_src(ctx, instr->src[0]);
2247       if (instr->src[0].src.ssa->bit_size == 32) {
2248          if (dst.type() == RegType::vgpr)
2249             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2250          else
2251             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2252                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2253
2254       } else if (instr->src[0].src.ssa->bit_size == 64) {
2255          if (dst.type() == RegType::vgpr)
2256             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2257          else
2258             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2259                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2260
2261       } else {
2262          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2263          nir_print_instr(&instr->instr, stderr);
2264          fprintf(stderr, "\n");
2265       }
2266       break;
2267    }
2268    case nir_op_f2i64: {
2269       Temp src = get_alu_src(ctx, instr->src[0]);
2270       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2271          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2272          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2273          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2274          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2275          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2276          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2277          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2278          Temp new_exponent = bld.tmp(v1);
2279          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2280          if (ctx->program->chip_class >= GFX8)
2281             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2282          else
2283             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2284          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2285          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2286          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2287          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2288          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2289          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2290          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2291          Temp new_lower = bld.tmp(v1);
2292          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2293          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2294          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2295
2296       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2297          if (src.type() == RegType::vgpr)
2298             src = bld.as_uniform(src);
2299          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2300          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2301          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2302          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2303          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2304          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2305          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2306          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2307          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2308          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2309          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2310          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2311          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2312          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2313          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2314          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2315          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2316          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2317          Temp borrow = bld.tmp(s1);
2318          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2319          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2320          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2321
2322       } else if (instr->src[0].src.ssa->bit_size == 64) {
2323          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2324          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2325          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2326          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2327          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2328          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2329          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2330          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2331          if (dst.type() == RegType::sgpr) {
2332             lower = bld.as_uniform(lower);
2333             upper = bld.as_uniform(upper);
2334          }
2335          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2336
2337       } else {
2338          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2339          nir_print_instr(&instr->instr, stderr);
2340          fprintf(stderr, "\n");
2341       }
2342       break;
2343    }
2344    case nir_op_f2u64: {
2345       Temp src = get_alu_src(ctx, instr->src[0]);
2346       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2347          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2348          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2349          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2350          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2351          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2352          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2353          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2354          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2355          Temp new_exponent = bld.tmp(v1);
2356          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2357          if (ctx->program->chip_class >= GFX8)
2358             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2359          else
2360             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2361          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2362          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2363          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2364          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2365          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2366          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2367          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2368
2369       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2370          if (src.type() == RegType::vgpr)
2371             src = bld.as_uniform(src);
2372          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2373          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2374          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2375          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2376          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2377          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2378          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2379          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2380          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2381          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2382          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2383          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2384          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2385          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2386          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2387          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2388          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2389          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2390
2391       } else if (instr->src[0].src.ssa->bit_size == 64) {
2392          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2393          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2394          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2395          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2396          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2397          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2398          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2399          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2400          if (dst.type() == RegType::sgpr) {
2401             lower = bld.as_uniform(lower);
2402             upper = bld.as_uniform(upper);
2403          }
2404          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2405
2406       } else {
2407          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2408          nir_print_instr(&instr->instr, stderr);
2409          fprintf(stderr, "\n");
2410       }
2411       break;
2412    }
2413    case nir_op_b2f32: {
2414       Temp src = get_alu_src(ctx, instr->src[0]);
2415       assert(src.regClass() == bld.lm);
2416
2417       if (dst.regClass() == s1) {
2418          src = bool_to_scalar_condition(ctx, src);
2419          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2420       } else if (dst.regClass() == v1) {
2421          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2422       } else {
2423          unreachable("Wrong destination register class for nir_op_b2f32.");
2424       }
2425       break;
2426    }
2427    case nir_op_b2f64: {
2428       Temp src = get_alu_src(ctx, instr->src[0]);
2429       assert(src.regClass() == bld.lm);
2430
2431       if (dst.regClass() == s2) {
2432          src = bool_to_scalar_condition(ctx, src);
2433          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2434       } else if (dst.regClass() == v2) {
2435          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2436          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2437          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2438       } else {
2439          unreachable("Wrong destination register class for nir_op_b2f64.");
2440       }
2441       break;
2442    }
2443    case nir_op_i2i8:
2444    case nir_op_u2u8: {
2445       Temp src = get_alu_src(ctx, instr->src[0]);
2446       /* we can actually just say dst = src */
2447       if (src.regClass() == s1)
2448          bld.copy(Definition(dst), src);
2449       else
2450          emit_extract_vector(ctx, src, 0, dst);
2451       break;
2452    }
2453    case nir_op_i2i16: {
2454       Temp src = get_alu_src(ctx, instr->src[0]);
2455       if (instr->src[0].src.ssa->bit_size == 8) {
2456          if (dst.regClass() == s1) {
2457             bld.sop1(aco_opcode::s_sext_i32_i8, Definition(dst), Operand(src));
2458          } else {
2459             assert(src.regClass() == v1b);
2460             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2461             sdwa->operands[0] = Operand(src);
2462             sdwa->definitions[0] = Definition(dst);
2463             sdwa->sel[0] = sdwa_sbyte;
2464             sdwa->dst_sel = sdwa_sword;
2465             ctx->block->instructions.emplace_back(std::move(sdwa));
2466          }
2467       } else {
2468          Temp src = get_alu_src(ctx, instr->src[0]);
2469          /* we can actually just say dst = src */
2470          if (src.regClass() == s1)
2471             bld.copy(Definition(dst), src);
2472          else
2473             emit_extract_vector(ctx, src, 0, dst);
2474       }
2475       break;
2476    }
2477    case nir_op_u2u16: {
2478       Temp src = get_alu_src(ctx, instr->src[0]);
2479       if (instr->src[0].src.ssa->bit_size == 8) {
2480          if (dst.regClass() == s1)
2481             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFu), src);
2482          else {
2483             assert(src.regClass() == v1b);
2484             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2485             sdwa->operands[0] = Operand(src);
2486             sdwa->definitions[0] = Definition(dst);
2487             sdwa->sel[0] = sdwa_ubyte;
2488             sdwa->dst_sel = sdwa_uword;
2489             ctx->block->instructions.emplace_back(std::move(sdwa));
2490          }
2491       } else {
2492          Temp src = get_alu_src(ctx, instr->src[0]);
2493          /* we can actually just say dst = src */
2494          if (src.regClass() == s1)
2495             bld.copy(Definition(dst), src);
2496          else
2497             emit_extract_vector(ctx, src, 0, dst);
2498       }
2499       break;
2500    }
2501    case nir_op_i2i32: {
2502       Temp src = get_alu_src(ctx, instr->src[0]);
2503       if (instr->src[0].src.ssa->bit_size == 8) {
2504          if (dst.regClass() == s1) {
2505             bld.sop1(aco_opcode::s_sext_i32_i8, Definition(dst), Operand(src));
2506          } else {
2507             assert(src.regClass() == v1b);
2508             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2509             sdwa->operands[0] = Operand(src);
2510             sdwa->definitions[0] = Definition(dst);
2511             sdwa->sel[0] = sdwa_sbyte;
2512             sdwa->dst_sel = sdwa_sdword;
2513             ctx->block->instructions.emplace_back(std::move(sdwa));
2514          }
2515       } else if (instr->src[0].src.ssa->bit_size == 16) {
2516          if (dst.regClass() == s1) {
2517             bld.sop1(aco_opcode::s_sext_i32_i16, Definition(dst), Operand(src));
2518          } else {
2519             assert(src.regClass() == v2b);
2520             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2521             sdwa->operands[0] = Operand(src);
2522             sdwa->definitions[0] = Definition(dst);
2523             sdwa->sel[0] = sdwa_sword;
2524             sdwa->dst_sel = sdwa_udword;
2525             ctx->block->instructions.emplace_back(std::move(sdwa));
2526          }
2527       } else if (instr->src[0].src.ssa->bit_size == 64) {
2528          /* we can actually just say dst = src, as it would map the lower register */
2529          emit_extract_vector(ctx, src, 0, dst);
2530       } else {
2531          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2532          nir_print_instr(&instr->instr, stderr);
2533          fprintf(stderr, "\n");
2534       }
2535       break;
2536    }
2537    case nir_op_u2u32: {
2538       Temp src = get_alu_src(ctx, instr->src[0]);
2539       if (instr->src[0].src.ssa->bit_size == 8) {
2540          if (dst.regClass() == s1)
2541             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFu), src);
2542          else {
2543             assert(src.regClass() == v1b);
2544             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2545             sdwa->operands[0] = Operand(src);
2546             sdwa->definitions[0] = Definition(dst);
2547             sdwa->sel[0] = sdwa_ubyte;
2548             sdwa->dst_sel = sdwa_udword;
2549             ctx->block->instructions.emplace_back(std::move(sdwa));
2550          }
2551       } else if (instr->src[0].src.ssa->bit_size == 16) {
2552          if (dst.regClass() == s1) {
2553             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2554          } else {
2555             assert(src.regClass() == v2b);
2556             aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
2557             sdwa->operands[0] = Operand(src);
2558             sdwa->definitions[0] = Definition(dst);
2559             sdwa->sel[0] = sdwa_uword;
2560             sdwa->dst_sel = sdwa_udword;
2561             ctx->block->instructions.emplace_back(std::move(sdwa));
2562          }
2563       } else if (instr->src[0].src.ssa->bit_size == 64) {
2564          /* we can actually just say dst = src, as it would map the lower register */
2565          emit_extract_vector(ctx, src, 0, dst);
2566       } else {
2567          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2568          nir_print_instr(&instr->instr, stderr);
2569          fprintf(stderr, "\n");
2570       }
2571       break;
2572    }
2573    case nir_op_i2i64: {
2574       Temp src = get_alu_src(ctx, instr->src[0]);
2575       if (src.regClass() == s1) {
2576          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2577          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2578       } else if (src.regClass() == v1) {
2579          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2580          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2581       } else {
2582          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2583          nir_print_instr(&instr->instr, stderr);
2584          fprintf(stderr, "\n");
2585       }
2586       break;
2587    }
2588    case nir_op_u2u64: {
2589       Temp src = get_alu_src(ctx, instr->src[0]);
2590       if (instr->src[0].src.ssa->bit_size == 32) {
2591          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2592       } else {
2593          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2594          nir_print_instr(&instr->instr, stderr);
2595          fprintf(stderr, "\n");
2596       }
2597       break;
2598    }
2599    case nir_op_b2b32:
2600    case nir_op_b2i32: {
2601       Temp src = get_alu_src(ctx, instr->src[0]);
2602       assert(src.regClass() == bld.lm);
2603
2604       if (dst.regClass() == s1) {
2605          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2606          bool_to_scalar_condition(ctx, src, dst);
2607       } else if (dst.regClass() == v1) {
2608          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2609       } else {
2610          unreachable("Invalid register class for b2i32");
2611       }
2612       break;
2613    }
2614    case nir_op_b2b1:
2615    case nir_op_i2b1: {
2616       Temp src = get_alu_src(ctx, instr->src[0]);
2617       assert(dst.regClass() == bld.lm);
2618
2619       if (src.type() == RegType::vgpr) {
2620          assert(src.regClass() == v1 || src.regClass() == v2);
2621          assert(dst.regClass() == bld.lm);
2622          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2623                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2624       } else {
2625          assert(src.regClass() == s1 || src.regClass() == s2);
2626          Temp tmp;
2627          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2628             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2629          } else {
2630             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2631                            bld.scc(bld.def(s1)), Operand(0u), src);
2632          }
2633          bool_to_vector_condition(ctx, tmp, dst);
2634       }
2635       break;
2636    }
2637    case nir_op_pack_64_2x32_split: {
2638       Temp src0 = get_alu_src(ctx, instr->src[0]);
2639       Temp src1 = get_alu_src(ctx, instr->src[1]);
2640
2641       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2642       break;
2643    }
2644    case nir_op_unpack_64_2x32_split_x:
2645       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2646       break;
2647    case nir_op_unpack_64_2x32_split_y:
2648       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2649       break;
2650    case nir_op_unpack_32_2x16_split_x:
2651       if (dst.type() == RegType::vgpr) {
2652          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2653       } else {
2654          bld.copy(Definition(dst), get_alu_src(ctx, instr->src[0]));
2655       }
2656       break;
2657    case nir_op_unpack_32_2x16_split_y:
2658       if (dst.type() == RegType::vgpr) {
2659          bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2660       } else {
2661          bld.sop2(aco_opcode::s_bfe_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), Operand(uint32_t(16 << 16 | 16)));
2662       }
2663       break;
2664    case nir_op_pack_32_2x16_split: {
2665       Temp src0 = get_alu_src(ctx, instr->src[0]);
2666       Temp src1 = get_alu_src(ctx, instr->src[1]);
2667       if (dst.regClass() == v1) {
2668          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2669       } else {
2670          src0 = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), src0, Operand(0xFFFFu));
2671          src1 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), src1, Operand(16u));
2672          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), src0, src1);
2673       }
2674       break;
2675    }
2676    case nir_op_pack_half_2x16: {
2677       Temp src = get_alu_src(ctx, instr->src[0], 2);
2678
2679       if (dst.regClass() == v1) {
2680          Temp src0 = bld.tmp(v1);
2681          Temp src1 = bld.tmp(v1);
2682          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2683          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2684             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2685          else
2686             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2687                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2688                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2689       } else {
2690          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2691          nir_print_instr(&instr->instr, stderr);
2692          fprintf(stderr, "\n");
2693       }
2694       break;
2695    }
2696    case nir_op_unpack_half_2x16_split_x: {
2697       if (dst.regClass() == v1) {
2698          Builder bld(ctx->program, ctx->block);
2699          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2700       } else {
2701          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2702          nir_print_instr(&instr->instr, stderr);
2703          fprintf(stderr, "\n");
2704       }
2705       break;
2706    }
2707    case nir_op_unpack_half_2x16_split_y: {
2708       if (dst.regClass() == v1) {
2709          Builder bld(ctx->program, ctx->block);
2710          /* TODO: use SDWA here */
2711          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2712                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2713       } else {
2714          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2715          nir_print_instr(&instr->instr, stderr);
2716          fprintf(stderr, "\n");
2717       }
2718       break;
2719    }
2720    case nir_op_fquantize2f16: {
2721       Temp src = get_alu_src(ctx, instr->src[0]);
2722       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2723       Temp f32, cmp_res;
2724
2725       if (ctx->program->chip_class >= GFX8) {
2726          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2727          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2728          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2729       } else {
2730          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2731           * so compare the result and flush to 0 if it's smaller.
2732           */
2733          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2734          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2735          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2736          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2737          cmp_res = vop3->definitions[0].getTemp();
2738       }
2739
2740       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2741          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2742          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2743       } else {
2744          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2745       }
2746       break;
2747    }
2748    case nir_op_bfm: {
2749       Temp bits = get_alu_src(ctx, instr->src[0]);
2750       Temp offset = get_alu_src(ctx, instr->src[1]);
2751
2752       if (dst.regClass() == s1) {
2753          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2754       } else if (dst.regClass() == v1) {
2755          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2756       } else {
2757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2758          nir_print_instr(&instr->instr, stderr);
2759          fprintf(stderr, "\n");
2760       }
2761       break;
2762    }
2763    case nir_op_bitfield_select: {
2764       /* (mask & insert) | (~mask & base) */
2765       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2766       Temp insert = get_alu_src(ctx, instr->src[1]);
2767       Temp base = get_alu_src(ctx, instr->src[2]);
2768
2769       /* dst = (insert & bitmask) | (base & ~bitmask) */
2770       if (dst.regClass() == s1) {
2771          aco_ptr<Instruction> sop2;
2772          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2773          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2774          Operand lhs;
2775          if (const_insert && const_bitmask) {
2776             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2777          } else {
2778             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2779             lhs = Operand(insert);
2780          }
2781
2782          Operand rhs;
2783          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2784          if (const_base && const_bitmask) {
2785             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2786          } else {
2787             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2788             rhs = Operand(base);
2789          }
2790
2791          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2792
2793       } else if (dst.regClass() == v1) {
2794          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2795             base = as_vgpr(ctx, base);
2796          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2797             insert = as_vgpr(ctx, insert);
2798
2799          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2800
2801       } else {
2802          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2803          nir_print_instr(&instr->instr, stderr);
2804          fprintf(stderr, "\n");
2805       }
2806       break;
2807    }
2808    case nir_op_ubfe:
2809    case nir_op_ibfe: {
2810       Temp base = get_alu_src(ctx, instr->src[0]);
2811       Temp offset = get_alu_src(ctx, instr->src[1]);
2812       Temp bits = get_alu_src(ctx, instr->src[2]);
2813
2814       if (dst.type() == RegType::sgpr) {
2815          Operand extract;
2816          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2817          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2818          if (const_offset && const_bits) {
2819             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2820             extract = Operand(const_extract);
2821          } else {
2822             Operand width;
2823             if (const_bits) {
2824                width = Operand(const_bits->u32 << 16);
2825             } else {
2826                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2827             }
2828             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2829          }
2830
2831          aco_opcode opcode;
2832          if (dst.regClass() == s1) {
2833             if (instr->op == nir_op_ubfe)
2834                opcode = aco_opcode::s_bfe_u32;
2835             else
2836                opcode = aco_opcode::s_bfe_i32;
2837          } else if (dst.regClass() == s2) {
2838             if (instr->op == nir_op_ubfe)
2839                opcode = aco_opcode::s_bfe_u64;
2840             else
2841                opcode = aco_opcode::s_bfe_i64;
2842          } else {
2843             unreachable("Unsupported BFE bit size");
2844          }
2845
2846          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2847
2848       } else {
2849          aco_opcode opcode;
2850          if (dst.regClass() == v1) {
2851             if (instr->op == nir_op_ubfe)
2852                opcode = aco_opcode::v_bfe_u32;
2853             else
2854                opcode = aco_opcode::v_bfe_i32;
2855          } else {
2856             unreachable("Unsupported BFE bit size");
2857          }
2858
2859          emit_vop3a_instruction(ctx, instr, opcode, dst);
2860       }
2861       break;
2862    }
2863    case nir_op_bit_count: {
2864       Temp src = get_alu_src(ctx, instr->src[0]);
2865       if (src.regClass() == s1) {
2866          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2867       } else if (src.regClass() == v1) {
2868          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2869       } else if (src.regClass() == v2) {
2870          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2871                   emit_extract_vector(ctx, src, 1, v1),
2872                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2873                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2874       } else if (src.regClass() == s2) {
2875          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2876       } else {
2877          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2878          nir_print_instr(&instr->instr, stderr);
2879          fprintf(stderr, "\n");
2880       }
2881       break;
2882    }
2883    case nir_op_flt: {
2884       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2885       break;
2886    }
2887    case nir_op_fge: {
2888       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2889       break;
2890    }
2891    case nir_op_feq: {
2892       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2893       break;
2894    }
2895    case nir_op_fne: {
2896       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2897       break;
2898    }
2899    case nir_op_ilt: {
2900       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2901       break;
2902    }
2903    case nir_op_ige: {
2904       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2905       break;
2906    }
2907    case nir_op_ieq: {
2908       if (instr->src[0].src.ssa->bit_size == 1)
2909          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2910       else
2911          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2912                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2913       break;
2914    }
2915    case nir_op_ine: {
2916       if (instr->src[0].src.ssa->bit_size == 1)
2917          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2918       else
2919          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2920                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2921       break;
2922    }
2923    case nir_op_ult: {
2924       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2925       break;
2926    }
2927    case nir_op_uge: {
2928       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2929       break;
2930    }
2931    case nir_op_fddx:
2932    case nir_op_fddy:
2933    case nir_op_fddx_fine:
2934    case nir_op_fddy_fine:
2935    case nir_op_fddx_coarse:
2936    case nir_op_fddy_coarse: {
2937       Temp src = get_alu_src(ctx, instr->src[0]);
2938       uint16_t dpp_ctrl1, dpp_ctrl2;
2939       if (instr->op == nir_op_fddx_fine) {
2940          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2941          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2942       } else if (instr->op == nir_op_fddy_fine) {
2943          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2944          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2945       } else {
2946          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2947          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2948             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2949          else
2950             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2951       }
2952
2953       Temp tmp;
2954       if (ctx->program->chip_class >= GFX8) {
2955          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2956          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2957       } else {
2958          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2959          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2960          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2961       }
2962       emit_wqm(ctx, tmp, dst, true);
2963       break;
2964    }
2965    default:
2966       fprintf(stderr, "Unknown NIR ALU instr: ");
2967       nir_print_instr(&instr->instr, stderr);
2968       fprintf(stderr, "\n");
2969    }
2970 }
2971
2972 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2973 {
2974    Temp dst = get_ssa_temp(ctx, &instr->def);
2975
2976    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2977    // which get truncated the lsb if double and msb if int
2978    // for now, we only use s_mov_b64 with 64bit inline constants
2979    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2980    assert(dst.type() == RegType::sgpr);
2981
2982    Builder bld(ctx->program, ctx->block);
2983
2984    if (instr->def.bit_size == 1) {
2985       assert(dst.regClass() == bld.lm);
2986       int val = instr->value[0].b ? -1 : 0;
2987       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2988       bld.sop1(Builder::s_mov, Definition(dst), op);
2989    } else if (dst.size() == 1) {
2990       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2991    } else {
2992       assert(dst.size() != 1);
2993       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2994       if (instr->def.bit_size == 64)
2995          for (unsigned i = 0; i < dst.size(); i++)
2996             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2997       else {
2998          for (unsigned i = 0; i < dst.size(); i++)
2999             vec->operands[i] = Operand{instr->value[i].u32};
3000       }
3001       vec->definitions[0] = Definition(dst);
3002       ctx->block->instructions.emplace_back(std::move(vec));
3003    }
3004 }
3005
3006 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
3007 {
3008    uint32_t new_mask = 0;
3009    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
3010       if (mask & (1u << i))
3011          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
3012    return new_mask;
3013 }
3014
3015 Operand load_lds_size_m0(isel_context *ctx)
3016 {
3017    /* TODO: m0 does not need to be initialized on GFX9+ */
3018    Builder bld(ctx->program, ctx->block);
3019    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
3020 }
3021
3022 Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
3023               Temp address, unsigned base_offset, unsigned align)
3024 {
3025    assert(util_is_power_of_two_nonzero(align) && align >= 4);
3026
3027    Builder bld(ctx->program, ctx->block);
3028
3029    Operand m = load_lds_size_m0(ctx);
3030
3031    unsigned num_components = dst.size() * 4u / elem_size_bytes;
3032    unsigned bytes_read = 0;
3033    unsigned result_size = 0;
3034    unsigned total_bytes = num_components * elem_size_bytes;
3035    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
3036    bool large_ds_read = ctx->options->chip_class >= GFX7;
3037    bool usable_read2 = ctx->options->chip_class >= GFX7;
3038
3039    while (bytes_read < total_bytes) {
3040       unsigned todo = total_bytes - bytes_read;
3041       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
3042       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
3043
3044       aco_opcode op = aco_opcode::last_opcode;
3045       bool read2 = false;
3046       if (todo >= 16 && aligned16 && large_ds_read) {
3047          op = aco_opcode::ds_read_b128;
3048          todo = 16;
3049       } else if (todo >= 16 && aligned8 && usable_read2) {
3050          op = aco_opcode::ds_read2_b64;
3051          read2 = true;
3052          todo = 16;
3053       } else if (todo >= 12 && aligned16 && large_ds_read) {
3054          op = aco_opcode::ds_read_b96;
3055          todo = 12;
3056       } else if (todo >= 8 && aligned8) {
3057          op = aco_opcode::ds_read_b64;
3058          todo = 8;
3059       } else if (todo >= 8 && usable_read2) {
3060          op = aco_opcode::ds_read2_b32;
3061          read2 = true;
3062          todo = 8;
3063       } else if (todo >= 4) {
3064          op = aco_opcode::ds_read_b32;
3065          todo = 4;
3066       } else {
3067          assert(false);
3068       }
3069       assert(todo % elem_size_bytes == 0);
3070       unsigned num_elements = todo / elem_size_bytes;
3071       unsigned offset = base_offset + bytes_read;
3072       unsigned max_offset = read2 ? 1019 : 65535;
3073
3074       Temp address_offset = address;
3075       if (offset > max_offset) {
3076          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3077          offset = bytes_read;
3078       }
3079       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
3080
3081       Temp res;
3082       if (num_components == 1 && dst.type() == RegType::vgpr)
3083          res = dst;
3084       else
3085          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
3086
3087       if (read2)
3088          res = bld.ds(op, Definition(res), address_offset, m, offset / (todo / 2), (offset / (todo / 2)) + 1);
3089       else
3090          res = bld.ds(op, Definition(res), address_offset, m, offset);
3091
3092       if (num_components == 1) {
3093          assert(todo == total_bytes);
3094          if (dst.type() == RegType::sgpr)
3095             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
3096          return dst;
3097       }
3098
3099       if (dst.type() == RegType::sgpr) {
3100          Temp new_res = bld.tmp(RegType::sgpr, res.size());
3101          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
3102          res = new_res;
3103       }
3104
3105       if (num_elements == 1) {
3106          result[result_size++] = res;
3107       } else {
3108          assert(res != dst && res.size() % num_elements == 0);
3109          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
3110          split->operands[0] = Operand(res);
3111          for (unsigned i = 0; i < num_elements; i++)
3112             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
3113          ctx->block->instructions.emplace_back(std::move(split));
3114       }
3115
3116       bytes_read += todo;
3117    }
3118
3119    assert(result_size == num_components && result_size > 1);
3120    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
3121    for (unsigned i = 0; i < result_size; i++)
3122       vec->operands[i] = Operand(result[i]);
3123    vec->definitions[0] = Definition(dst);
3124    ctx->block->instructions.emplace_back(std::move(vec));
3125    ctx->allocated_vec.emplace(dst.id(), result);
3126
3127    return dst;
3128 }
3129
3130 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
3131 {
3132    if (start == 0 && size == data.size())
3133       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
3134
3135    unsigned size_hint = 1;
3136    auto it = ctx->allocated_vec.find(data.id());
3137    if (it != ctx->allocated_vec.end())
3138       size_hint = it->second[0].size();
3139    if (size % size_hint || start % size_hint)
3140       size_hint = 1;
3141
3142    start /= size_hint;
3143    size /= size_hint;
3144
3145    Temp elems[size];
3146    for (unsigned i = 0; i < size; i++)
3147       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
3148
3149    if (size == 1)
3150       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
3151
3152    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
3153    for (unsigned i = 0; i < size; i++)
3154       vec->operands[i] = Operand(elems[i]);
3155    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
3156    vec->definitions[0] = Definition(res);
3157    ctx->block->instructions.emplace_back(std::move(vec));
3158    return res;
3159 }
3160
3161 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
3162 {
3163    Builder bld(ctx->program, ctx->block);
3164    unsigned bytes_written = 0;
3165    bool large_ds_write = ctx->options->chip_class >= GFX7;
3166    bool usable_write2 = ctx->options->chip_class >= GFX7;
3167
3168    while (bytes_written < total_size * 4) {
3169       unsigned todo = total_size * 4 - bytes_written;
3170       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
3171       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
3172
3173       aco_opcode op = aco_opcode::last_opcode;
3174       bool write2 = false;
3175       unsigned size = 0;
3176       if (todo >= 16 && aligned16 && large_ds_write) {
3177          op = aco_opcode::ds_write_b128;
3178          size = 4;
3179       } else if (todo >= 16 && aligned8 && usable_write2) {
3180          op = aco_opcode::ds_write2_b64;
3181          write2 = true;
3182          size = 4;
3183       } else if (todo >= 12 && aligned16 && large_ds_write) {
3184          op = aco_opcode::ds_write_b96;
3185          size = 3;
3186       } else if (todo >= 8 && aligned8) {
3187          op = aco_opcode::ds_write_b64;
3188          size = 2;
3189       } else if (todo >= 8 && usable_write2) {
3190          op = aco_opcode::ds_write2_b32;
3191          write2 = true;
3192          size = 2;
3193       } else if (todo >= 4) {
3194          op = aco_opcode::ds_write_b32;
3195          size = 1;
3196       } else {
3197          assert(false);
3198       }
3199
3200       unsigned offset = offset0 + offset1 + bytes_written;
3201       unsigned max_offset = write2 ? 1020 : 65535;
3202       Temp address_offset = address;
3203       if (offset > max_offset) {
3204          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
3205          offset = offset1 + bytes_written;
3206       }
3207       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
3208
3209       if (write2) {
3210          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
3211          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
3212          bld.ds(op, address_offset, val0, val1, m, offset / size / 2, (offset / size / 2) + 1);
3213       } else {
3214          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
3215          bld.ds(op, address_offset, val, m, offset);
3216       }
3217
3218       bytes_written += size * 4;
3219    }
3220 }
3221
3222 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
3223                Temp address, unsigned base_offset, unsigned align)
3224 {
3225    assert(util_is_power_of_two_nonzero(align) && align >= 4);
3226    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3227
3228    Operand m = load_lds_size_m0(ctx);
3229
3230    /* we need at most two stores, assuming that the writemask is at most 4 bits wide */
3231    assert(wrmask <= 0x0f);
3232    int start[2], count[2];
3233    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
3234    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
3235    assert(wrmask == 0);
3236
3237    /* one combined store is sufficient */
3238    if (count[0] == count[1] && (align % elem_size_bytes) == 0 && (base_offset % elem_size_bytes) == 0) {
3239       Builder bld(ctx->program, ctx->block);
3240
3241       Temp address_offset = address;
3242       if ((base_offset / elem_size_bytes) + start[1] > 255) {
3243          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3244          base_offset = 0;
3245       }
3246
3247       assert(count[0] == 1);
3248       RegClass xtract_rc(RegType::vgpr, elem_size_bytes / 4);
3249
3250       Temp val0 = emit_extract_vector(ctx, data, start[0], xtract_rc);
3251       Temp val1 = emit_extract_vector(ctx, data, start[1], xtract_rc);
3252       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
3253       base_offset = base_offset / elem_size_bytes;
3254       bld.ds(op, address_offset, val0, val1, m,
3255              base_offset + start[0], base_offset + start[1]);
3256       return;
3257    }
3258
3259    for (unsigned i = 0; i < 2; i++) {
3260       if (count[i] == 0)
3261          continue;
3262
3263       unsigned elem_size_words = elem_size_bytes / 4;
3264       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
3265                       base_offset, start[i] * elem_size_bytes, align);
3266    }
3267    return;
3268 }
3269
3270 unsigned calculate_lds_alignment(isel_context *ctx, unsigned const_offset)
3271 {
3272    unsigned align = 16;
3273    if (const_offset)
3274       align = std::min(align, 1u << (ffs(const_offset) - 1));
3275
3276    return align;
3277 }
3278
3279
3280 Temp create_vec_from_array(isel_context *ctx, Temp arr[], unsigned cnt, RegType reg_type, unsigned elem_size_bytes,
3281                            unsigned split_cnt = 0u, Temp dst = Temp())
3282 {
3283    Builder bld(ctx->program, ctx->block);
3284    unsigned dword_size = elem_size_bytes / 4;
3285
3286    if (!dst.id())
3287       dst = bld.tmp(RegClass(reg_type, cnt * dword_size));
3288
3289    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
3290    aco_ptr<Pseudo_instruction> instr {create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, cnt, 1)};
3291    instr->definitions[0] = Definition(dst);
3292
3293    for (unsigned i = 0; i < cnt; ++i) {
3294       if (arr[i].id()) {
3295          assert(arr[i].size() == dword_size);
3296          allocated_vec[i] = arr[i];
3297          instr->operands[i] = Operand(arr[i]);
3298       } else {
3299          Temp zero = bld.copy(bld.def(RegClass(reg_type, dword_size)), Operand(0u, dword_size == 2));
3300          allocated_vec[i] = zero;
3301          instr->operands[i] = Operand(zero);
3302       }
3303    }
3304
3305    bld.insert(std::move(instr));
3306
3307    if (split_cnt)
3308       emit_split_vector(ctx, dst, split_cnt);
3309    else
3310       ctx->allocated_vec.emplace(dst.id(), allocated_vec); /* emit_split_vector already does this */
3311
3312    return dst;
3313 }
3314
3315 inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, unsigned const_offset)
3316 {
3317    if (const_offset >= 4096) {
3318       unsigned excess_const_offset = const_offset / 4096u * 4096u;
3319       const_offset %= 4096u;
3320
3321       if (!voffset.id())
3322          voffset = bld.copy(bld.def(v1), Operand(excess_const_offset));
3323       else if (unlikely(voffset.regClass() == s1))
3324          voffset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), Operand(excess_const_offset), Operand(voffset));
3325       else if (likely(voffset.regClass() == v1))
3326          voffset = bld.vadd32(bld.def(v1), Operand(voffset), Operand(excess_const_offset));
3327       else
3328          unreachable("Unsupported register class of voffset");
3329    }
3330
3331    return const_offset;
3332 }
3333
3334 void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata,
3335                              unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false)
3336 {
3337    assert(vdata.id());
3338    assert(vdata.size() != 3 || ctx->program->chip_class != GFX6);
3339    assert(vdata.size() >= 1 && vdata.size() <= 4);
3340
3341    Builder bld(ctx->program, ctx->block);
3342    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_store_dword + vdata.size() - 1);
3343    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3344
3345    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3346    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3347    Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset,
3348                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3349                                  /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc);
3350
3351    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3352 }
3353
3354 void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset,
3355                                    unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask,
3356                                    bool allow_combining = true, bool reorder = true, bool slc = false)
3357 {
3358    Builder bld(ctx->program, ctx->block);
3359    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3360    assert(write_mask);
3361
3362    if (elem_size_bytes == 8) {
3363       elem_size_bytes = 4;
3364       write_mask = widen_mask(write_mask, 2);
3365    }
3366
3367    while (write_mask) {
3368       int start = 0;
3369       int count = 0;
3370       u_bit_scan_consecutive_range(&write_mask, &start, &count);
3371       assert(count > 0);
3372       assert(start >= 0);
3373
3374       while (count > 0) {
3375          unsigned sub_count = allow_combining ? MIN2(count, 4) : 1;
3376          unsigned const_offset = (unsigned) start * elem_size_bytes + base_const_offset;
3377
3378          /* GFX6 doesn't have buffer_store_dwordx3, so make sure not to emit that here either. */
3379          if (unlikely(ctx->program->chip_class == GFX6 && sub_count == 3))
3380             sub_count = 2;
3381
3382          Temp elem = extract_subvector(ctx, src, start, sub_count, RegType::vgpr);
3383          emit_single_mubuf_store(ctx, descriptor, voffset, soffset, elem, const_offset, reorder, slc);
3384
3385          count -= sub_count;
3386          start += sub_count;
3387       }
3388
3389       assert(count == 0);
3390    }
3391 }
3392
3393 Temp emit_single_mubuf_load(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset,
3394                             unsigned const_offset, unsigned size_dwords, bool allow_reorder = true)
3395 {
3396    assert(size_dwords != 3 || ctx->program->chip_class != GFX6);
3397    assert(size_dwords >= 1 && size_dwords <= 4);
3398
3399    Builder bld(ctx->program, ctx->block);
3400    Temp vdata = bld.tmp(RegClass(RegType::vgpr, size_dwords));
3401    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_load_dword + size_dwords - 1);
3402    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3403
3404    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3405    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3406    Builder::Result r = bld.mubuf(op, Definition(vdata), Operand(descriptor), voffset_op, soffset_op, const_offset,
3407                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3408                                  /* disable_wqm */ false, /* glc */ true,
3409                                  /* dlc*/ ctx->program->chip_class >= GFX10, /* slc */ false);
3410
3411    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3412
3413    return vdata;
3414 }
3415
3416 void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, Temp soffset,
3417                      unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components,
3418                      unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true)
3419 {
3420    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3421    assert((num_components * elem_size_bytes / 4) == dst.size());
3422    assert(!!stride != allow_combining);
3423
3424    Builder bld(ctx->program, ctx->block);
3425    unsigned split_cnt = num_components;
3426
3427    if (elem_size_bytes == 8) {
3428       elem_size_bytes = 4;
3429       num_components *= 2;
3430    }
3431
3432    if (!stride)
3433       stride = elem_size_bytes;
3434
3435    unsigned load_size = 1;
3436    if (allow_combining) {
3437       if ((num_components % 4) == 0)
3438          load_size = 4;
3439       else if ((num_components % 3) == 0 && ctx->program->chip_class != GFX6)
3440          load_size = 3;
3441       else if ((num_components % 2) == 0)
3442          load_size = 2;
3443    }
3444
3445    unsigned num_loads = num_components / load_size;
3446    std::array<Temp, NIR_MAX_VEC_COMPONENTS> elems;
3447
3448    for (unsigned i = 0; i < num_loads; ++i) {
3449       unsigned const_offset = i * stride * load_size + base_const_offset;
3450       elems[i] = emit_single_mubuf_load(ctx, descriptor, voffset, soffset, const_offset, load_size, allow_reorder);
3451    }
3452
3453    create_vec_from_array(ctx, elems.data(), num_loads, RegType::vgpr, load_size * 4u, split_cnt, dst);
3454 }
3455
3456 std::pair<Temp, unsigned> offset_add_from_nir(isel_context *ctx, const std::pair<Temp, unsigned> &base_offset, nir_src *off_src, unsigned stride = 1u)
3457 {
3458    Builder bld(ctx->program, ctx->block);
3459    Temp offset = base_offset.first;
3460    unsigned const_offset = base_offset.second;
3461
3462    if (!nir_src_is_const(*off_src)) {
3463       Temp indirect_offset_arg = get_ssa_temp(ctx, off_src->ssa);
3464       Temp with_stride;
3465
3466       /* Calculate indirect offset with stride */
3467       if (likely(indirect_offset_arg.regClass() == v1))
3468          with_stride = bld.v_mul_imm(bld.def(v1), indirect_offset_arg, stride);
3469       else if (indirect_offset_arg.regClass() == s1)
3470          with_stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), indirect_offset_arg);
3471       else
3472          unreachable("Unsupported register class of indirect offset");
3473
3474       /* Add to the supplied base offset */
3475       if (offset.id() == 0)
3476          offset = with_stride;
3477       else if (unlikely(offset.regClass() == s1 && with_stride.regClass() == s1))
3478          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), with_stride, offset);
3479       else if (offset.size() == 1 && with_stride.size() == 1)
3480          offset = bld.vadd32(bld.def(v1), with_stride, offset);
3481       else
3482          unreachable("Unsupported register class of indirect offset");
3483    } else {
3484       unsigned const_offset_arg = nir_src_as_uint(*off_src);
3485       const_offset += const_offset_arg * stride;
3486    }
3487
3488    return std::make_pair(offset, const_offset);
3489 }
3490
3491 std::pair<Temp, unsigned> offset_add(isel_context *ctx, const std::pair<Temp, unsigned> &off1, const std::pair<Temp, unsigned> &off2)
3492 {
3493    Builder bld(ctx->program, ctx->block);
3494    Temp offset;
3495
3496    if (off1.first.id() && off2.first.id()) {
3497       if (unlikely(off1.first.regClass() == s1 && off2.first.regClass() == s1))
3498          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), off1.first, off2.first);
3499       else if (off1.first.size() == 1 && off2.first.size() == 1)
3500          offset = bld.vadd32(bld.def(v1), off1.first, off2.first);
3501       else
3502          unreachable("Unsupported register class of indirect offset");
3503    } else {
3504       offset = off1.first.id() ? off1.first : off2.first;
3505    }
3506
3507    return std::make_pair(offset, off1.second + off2.second);
3508 }
3509
3510 std::pair<Temp, unsigned> offset_mul(isel_context *ctx, const std::pair<Temp, unsigned> &offs, unsigned multiplier)
3511 {
3512    Builder bld(ctx->program, ctx->block);
3513    unsigned const_offset = offs.second * multiplier;
3514
3515    if (!offs.first.id())
3516       return std::make_pair(offs.first, const_offset);
3517
3518    Temp offset = unlikely(offs.first.regClass() == s1)
3519                  ? bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(multiplier), offs.first)
3520                  : bld.v_mul_imm(bld.def(v1), offs.first, multiplier);
3521
3522    return std::make_pair(offset, const_offset);
3523 }
3524
3525 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride, unsigned component_stride)
3526 {
3527    Builder bld(ctx->program, ctx->block);
3528
3529    /* base is the driver_location, which is already multiplied by 4, so is in dwords */
3530    unsigned const_offset = nir_intrinsic_base(instr) * base_stride;
3531    /* component is in bytes */
3532    const_offset += nir_intrinsic_component(instr) * component_stride;
3533
3534    /* offset should be interpreted in relation to the base, so the instruction effectively reads/writes another input/output when it has an offset */
3535    nir_src *off_src = nir_get_io_offset_src(instr);
3536    return offset_add_from_nir(ctx, std::make_pair(Temp(), const_offset), off_src, 4u * base_stride);
3537 }
3538
3539 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned stride = 1u)
3540 {
3541    return get_intrinsic_io_basic_offset(ctx, instr, stride, stride);
3542 }
3543
3544 Temp get_tess_rel_patch_id(isel_context *ctx)
3545 {
3546    Builder bld(ctx->program, ctx->block);
3547
3548    switch (ctx->shader->info.stage) {
3549    case MESA_SHADER_TESS_CTRL:
3550       return bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffu),
3551                       get_arg(ctx, ctx->args->ac.tcs_rel_ids));
3552    case MESA_SHADER_TESS_EVAL:
3553       return get_arg(ctx, ctx->args->tes_rel_patch_id);
3554    default:
3555       unreachable("Unsupported stage in get_tess_rel_patch_id");
3556    }
3557 }
3558
3559 std::pair<Temp, unsigned> get_tcs_per_vertex_input_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3560 {
3561    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3562    Builder bld(ctx->program, ctx->block);
3563
3564    uint32_t tcs_in_patch_stride = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 4;
3565    uint32_t tcs_in_vertex_stride = ctx->tcs_num_inputs * 4;
3566
3567    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr);
3568
3569    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3570    offs = offset_add_from_nir(ctx, offs, vertex_index_src, tcs_in_vertex_stride);
3571
3572    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3573    Temp tcs_in_current_patch_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, tcs_in_patch_stride);
3574    offs = offset_add(ctx, offs, std::make_pair(tcs_in_current_patch_offset, 0));
3575
3576    return offset_mul(ctx, offs, 4u);
3577 }
3578
3579 std::pair<Temp, unsigned> get_tcs_output_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, bool per_vertex = false)
3580 {
3581    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3582    Builder bld(ctx->program, ctx->block);
3583
3584    uint32_t input_patch_size = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 16;
3585    uint32_t num_tcs_outputs = util_last_bit64(ctx->args->shader_info->tcs.outputs_written);
3586    uint32_t num_tcs_patch_outputs = util_last_bit64(ctx->args->shader_info->tcs.patch_outputs_written);
3587    uint32_t output_vertex_size = num_tcs_outputs * 16;
3588    uint32_t pervertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3589    uint32_t output_patch_stride = pervertex_output_patch_size + num_tcs_patch_outputs * 16;
3590
3591    std::pair<Temp, unsigned> offs = instr
3592                                     ? get_intrinsic_io_basic_offset(ctx, instr, 4u)
3593                                     : std::make_pair(Temp(), 0u);
3594
3595    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3596    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, output_patch_stride);
3597
3598    if (per_vertex) {
3599       assert(instr);
3600
3601       nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3602       offs = offset_add_from_nir(ctx, offs, vertex_index_src, output_vertex_size);
3603
3604       uint32_t output_patch0_offset = (input_patch_size * ctx->tcs_num_patches);
3605       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_offset));
3606    } else {
3607       uint32_t output_patch0_patch_data_offset = (input_patch_size * ctx->tcs_num_patches + pervertex_output_patch_size);
3608       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_patch_data_offset));
3609    }
3610
3611    return offs;
3612 }
3613
3614 std::pair<Temp, unsigned> get_tcs_per_vertex_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3615 {
3616    Builder bld(ctx->program, ctx->block);
3617
3618    unsigned vertices_per_patch = ctx->shader->info.tess.tcs_vertices_out;
3619    unsigned attr_stride = vertices_per_patch * ctx->tcs_num_patches;
3620
3621    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u);
3622
3623    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3624    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, vertices_per_patch * 16u);
3625    offs = offset_add(ctx, offs, std::make_pair(patch_off, 0u));
3626
3627    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3628    offs = offset_add_from_nir(ctx, offs, vertex_index_src, 16u);
3629
3630    return offs;
3631 }
3632
3633 std::pair<Temp, unsigned> get_tcs_per_patch_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, unsigned const_base_offset = 0u)
3634 {
3635    Builder bld(ctx->program, ctx->block);
3636
3637    unsigned num_tcs_outputs = ctx->shader->info.stage == MESA_SHADER_TESS_CTRL
3638                               ? util_last_bit64(ctx->args->shader_info->tcs.outputs_written)
3639                               : ctx->args->options->key.tes.tcs_num_outputs;
3640
3641    unsigned output_vertex_size = num_tcs_outputs * 16;
3642    unsigned per_vertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3643    unsigned per_patch_data_offset = per_vertex_output_patch_size * ctx->tcs_num_patches;
3644    unsigned attr_stride = ctx->tcs_num_patches;
3645
3646    std::pair<Temp, unsigned> offs = instr
3647                                     ? get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u)
3648                                     : std::make_pair(Temp(), 0u);
3649
3650    if (const_base_offset)
3651       offs.second += const_base_offset * attr_stride;
3652
3653    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3654    Temp patch_off = bld.v_mul_imm(bld.def(v1), rel_patch_id, 16u);
3655    offs = offset_add(ctx, offs, std::make_pair(patch_off, per_patch_data_offset));
3656
3657    return offs;
3658 }
3659
3660 bool tcs_driver_location_matches_api_mask(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex, uint64_t mask, bool *indirect)
3661 {
3662    unsigned off = nir_intrinsic_base(instr) * 4u;
3663    nir_src *off_src = nir_get_io_offset_src(instr);
3664
3665    if (!nir_src_is_const(*off_src)) {
3666       *indirect = true;
3667       return false;
3668    }
3669
3670    *indirect = false;
3671    off += nir_src_as_uint(*off_src) * 16u;
3672
3673    while (mask) {
3674       unsigned slot = u_bit_scan64(&mask) + (per_vertex ? 0 : VARYING_SLOT_PATCH0);
3675       if (off == shader_io_get_unique_index((gl_varying_slot) slot) * 16u)
3676          return true;
3677    }
3678
3679    return false;
3680 }
3681
3682 bool store_output_to_temps(isel_context *ctx, nir_intrinsic_instr *instr)
3683 {
3684    unsigned write_mask = nir_intrinsic_write_mask(instr);
3685    unsigned component = nir_intrinsic_component(instr);
3686    unsigned idx = nir_intrinsic_base(instr) + component;
3687
3688    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3689    if (off_instr->type != nir_instr_type_load_const)
3690       return false;
3691
3692    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3693    idx += nir_src_as_uint(instr->src[1]) * 4u;
3694
3695    if (instr->src[0].ssa->bit_size == 64)
3696       write_mask = widen_mask(write_mask, 2);
3697
3698    for (unsigned i = 0; i < 8; ++i) {
3699       if (write_mask & (1 << i)) {
3700          ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
3701          ctx->outputs.temps[idx] = emit_extract_vector(ctx, src, i, v1);
3702       }
3703       idx++;
3704    }
3705
3706    return true;
3707 }
3708
3709 bool load_input_from_temps(isel_context *ctx, nir_intrinsic_instr *instr, Temp dst)
3710 {
3711    /* Only TCS per-vertex inputs are supported by this function.
3712     * Per-vertex inputs only match between the VS/TCS invocation id when the number of invocations is the same.
3713     */
3714    if (ctx->shader->info.stage != MESA_SHADER_TESS_CTRL || !ctx->tcs_in_out_eq)
3715       return false;
3716
3717    nir_src *off_src = nir_get_io_offset_src(instr);
3718    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3719    nir_instr *vertex_index_instr = vertex_index_src->ssa->parent_instr;
3720    bool can_use_temps = nir_src_is_const(*off_src) &&
3721                         vertex_index_instr->type == nir_instr_type_intrinsic &&
3722                         nir_instr_as_intrinsic(vertex_index_instr)->intrinsic == nir_intrinsic_load_invocation_id;
3723
3724    if (!can_use_temps)
3725       return false;
3726
3727    unsigned idx = nir_intrinsic_base(instr) + nir_intrinsic_component(instr) + 4 * nir_src_as_uint(*off_src);
3728    Temp *src = &ctx->inputs.temps[idx];
3729    Temp vec = create_vec_from_array(ctx, src, dst.size(), dst.regClass().type(), 4u);
3730    assert(vec.size() == dst.size());
3731
3732    Builder bld(ctx->program, ctx->block);
3733    bld.copy(Definition(dst), vec);
3734    return true;
3735 }
3736
3737 void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr)
3738 {
3739    Builder bld(ctx->program, ctx->block);
3740
3741    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, 4u);
3742    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3743    unsigned write_mask = nir_intrinsic_write_mask(instr);
3744    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
3745
3746    if (ctx->tcs_in_out_eq && store_output_to_temps(ctx, instr)) {
3747       /* When the TCS only reads this output directly and for the same vertices as its invocation id, it is unnecessary to store the VS output to LDS. */
3748       bool indirect_write;
3749       bool temp_only_input = tcs_driver_location_matches_api_mask(ctx, instr, true, ctx->tcs_temp_only_inputs, &indirect_write);
3750       if (temp_only_input && !indirect_write)
3751          return;
3752    }
3753
3754    if (ctx->stage == vertex_es || ctx->stage == tess_eval_es) {
3755       /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */
3756       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
3757       Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset);
3758       store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true);
3759    } else {
3760       Temp lds_base;
3761
3762       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
3763          /* GFX9+: ES stage is merged into GS, data is passed between them using LDS. */
3764          unsigned itemsize = ctx->stage == vertex_geometry_gs
3765                              ? ctx->program->info->vs.es_info.esgs_itemsize
3766                              : ctx->program->info->tes.es_info.esgs_itemsize;
3767          Temp thread_id = emit_mbcnt(ctx, bld.def(v1));
3768          Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
3769          Temp vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), thread_id,
3770                                bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
3771          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
3772       } else if (ctx->stage == vertex_ls || ctx->stage == vertex_tess_control_hs) {
3773          /* GFX6-8: VS runs on LS stage when tessellation is used, but LS shares LDS space with HS.
3774           * GFX9+: LS is merged into HS, but still uses the same LDS layout.
3775           */
3776          unsigned num_tcs_inputs = util_last_bit64(ctx->args->shader_info->vs.ls_outputs_written);
3777          Temp vertex_idx = get_arg(ctx, ctx->args->rel_auto_id);
3778          lds_base = bld.v_mul_imm(bld.def(v1), vertex_idx, num_tcs_inputs * 16u);
3779       } else {
3780          unreachable("Invalid LS or ES stage");
3781       }
3782
3783       offs = offset_add(ctx, offs, std::make_pair(lds_base, 0u));
3784       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3785       store_lds(ctx, elem_size_bytes, src, write_mask, offs.first, offs.second, lds_align);
3786    }
3787 }
3788
3789 bool should_write_tcs_patch_output_to_vmem(isel_context *ctx, nir_intrinsic_instr *instr)
3790 {
3791    unsigned off = nir_intrinsic_base(instr) * 4u;
3792    return off != ctx->tcs_tess_lvl_out_loc &&
3793           off != ctx->tcs_tess_lvl_in_loc;
3794 }
3795
3796 bool should_write_tcs_output_to_lds(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3797 {
3798    /* When none of the appropriate outputs are read, we are OK to never write to LDS */
3799    if (per_vertex ? ctx->shader->info.outputs_read == 0U : ctx->shader->info.patch_outputs_read == 0u)
3800       return false;
3801
3802    uint64_t mask = per_vertex
3803                    ? ctx->shader->info.outputs_read
3804                    : ctx->shader->info.patch_outputs_read;
3805    bool indirect_write;
3806    bool output_read = tcs_driver_location_matches_api_mask(ctx, instr, per_vertex, mask, &indirect_write);
3807    return indirect_write || output_read;
3808 }
3809
3810 void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3811 {
3812    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3813    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3814
3815    Builder bld(ctx->program, ctx->block);
3816
3817    Temp store_val = get_ssa_temp(ctx, instr->src[0].ssa);
3818    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3819    unsigned write_mask = nir_intrinsic_write_mask(instr);
3820
3821    /* Only write to VMEM if the output is per-vertex or it's per-patch non tess factor */
3822    bool write_to_vmem = per_vertex || should_write_tcs_patch_output_to_vmem(ctx, instr);
3823    /* Only write to LDS if the output is read by the shader, or it's per-patch tess factor */
3824    bool write_to_lds = !write_to_vmem || should_write_tcs_output_to_lds(ctx, instr, per_vertex);
3825
3826    if (write_to_vmem) {
3827       std::pair<Temp, unsigned> vmem_offs = per_vertex
3828                                             ? get_tcs_per_vertex_output_vmem_offset(ctx, instr)
3829                                             : get_tcs_per_patch_output_vmem_offset(ctx, instr);
3830
3831       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
3832       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
3833       store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, false);
3834    }
3835
3836    if (write_to_lds) {
3837       std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3838       unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3839       store_lds(ctx, elem_size_bytes, store_val, write_mask, lds_offs.first, lds_offs.second, lds_align);
3840    }
3841 }
3842
3843 void visit_load_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3844 {
3845    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3846    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3847
3848    Builder bld(ctx->program, ctx->block);
3849
3850    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3851    std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3852    unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3853    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3854
3855    load_lds(ctx, elem_size_bytes, dst, lds_offs.first, lds_offs.second, lds_align);
3856 }
3857
3858 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
3859 {
3860    if (ctx->stage == vertex_vs ||
3861        ctx->stage == tess_eval_vs ||
3862        ctx->stage == fragment_fs ||
3863        ctx->stage == ngg_vertex_gs ||
3864        ctx->stage == ngg_tess_eval_gs ||
3865        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
3866       bool stored_to_temps = store_output_to_temps(ctx, instr);
3867       if (!stored_to_temps) {
3868          fprintf(stderr, "Unimplemented output offset instruction:\n");
3869          nir_print_instr(instr->src[1].ssa->parent_instr, stderr);
3870          fprintf(stderr, "\n");
3871          abort();
3872       }
3873    } else if (ctx->stage == vertex_es ||
3874               ctx->stage == vertex_ls ||
3875               ctx->stage == tess_eval_es ||
3876               (ctx->stage == vertex_tess_control_hs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3877               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3878               (ctx->stage == tess_eval_geometry_gs && ctx->shader->info.stage == MESA_SHADER_TESS_EVAL)) {
3879       visit_store_ls_or_es_output(ctx, instr);
3880    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
3881       visit_store_tcs_output(ctx, instr, false);
3882    } else {
3883       unreachable("Shader stage not implemented");
3884    }
3885 }
3886
3887 void visit_load_output(isel_context *ctx, nir_intrinsic_instr *instr)
3888 {
3889    visit_load_tcs_output(ctx, instr, false);
3890 }
3891
3892 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3893 {
3894    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3895    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3896
3897    Builder bld(ctx->program, ctx->block);
3898    Builder::Result interp_p1 = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3899    if (ctx->program->has_16bank_lds)
3900       interp_p1.instr->operands[0].setLateKill(true);
3901    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), interp_p1, idx, component);
3902 }
3903
3904 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3905 {
3906    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3907    for (unsigned i = 0; i < num_components; i++)
3908       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3909    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3910       assert(num_components == 4);
3911       Builder bld(ctx->program, ctx->block);
3912       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3913    }
3914
3915    for (Operand& op : vec->operands)
3916       op = op.isUndefined() ? Operand(0u) : op;
3917
3918    vec->definitions[0] = Definition(dst);
3919    ctx->block->instructions.emplace_back(std::move(vec));
3920    emit_split_vector(ctx, dst, num_components);
3921    return;
3922 }
3923
3924 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3925 {
3926    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3927    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3928    unsigned idx = nir_intrinsic_base(instr);
3929    unsigned component = nir_intrinsic_component(instr);
3930    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3931
3932    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3933    if (offset) {
3934       assert(offset->u32 == 0);
3935    } else {
3936       /* the lower 15bit of the prim_mask contain the offset into LDS
3937        * while the upper bits contain the number of prims */
3938       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3939       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3940       Builder bld(ctx->program, ctx->block);
3941       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3942       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3943       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3944       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3945       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3946    }
3947
3948    if (instr->dest.ssa.num_components == 1) {
3949       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3950    } else {
3951       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3952       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3953       {
3954          Temp tmp = {ctx->program->allocateId(), v1};
3955          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3956          vec->operands[i] = Operand(tmp);
3957       }
3958       vec->definitions[0] = Definition(dst);
3959       ctx->block->instructions.emplace_back(std::move(vec));
3960    }
3961 }
3962
3963 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3964                              unsigned offset, unsigned stride, unsigned channels)
3965 {
3966    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3967    if (vtx_info->chan_byte_size != 4 && channels == 3)
3968       return false;
3969    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3970           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3971 }
3972
3973 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3974                               unsigned offset, unsigned stride, unsigned *channels)
3975 {
3976    if (!vtx_info->chan_byte_size) {
3977       *channels = vtx_info->num_channels;
3978       return vtx_info->chan_format;
3979    }
3980
3981    unsigned num_channels = *channels;
3982    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3983       unsigned new_channels = num_channels + 1;
3984       /* first, assume more loads is worse and try using a larger data format */
3985       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3986          new_channels++;
3987          /* don't make the attribute potentially out-of-bounds */
3988          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3989             new_channels = 5;
3990       }
3991
3992       if (new_channels == 5) {
3993          /* then try decreasing load size (at the cost of more loads) */
3994          new_channels = *channels;
3995          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3996             new_channels--;
3997       }
3998
3999       if (new_channels < *channels)
4000          *channels = new_channels;
4001       num_channels = new_channels;
4002    }
4003
4004    switch (vtx_info->chan_format) {
4005    case V_008F0C_BUF_DATA_FORMAT_8:
4006       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
4007                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
4008    case V_008F0C_BUF_DATA_FORMAT_16:
4009       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
4010                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
4011    case V_008F0C_BUF_DATA_FORMAT_32:
4012       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
4013                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
4014    }
4015    unreachable("shouldn't reach here");
4016    return V_008F0C_BUF_DATA_FORMAT_INVALID;
4017 }
4018
4019 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
4020  * so we may need to fix it up. */
4021 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
4022 {
4023    Builder bld(ctx->program, ctx->block);
4024
4025    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
4026       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
4027
4028    /* For the integer-like cases, do a natural sign extension.
4029     *
4030     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
4031     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
4032     * exponent.
4033     */
4034    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
4035    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
4036
4037    /* Convert back to the right type. */
4038    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
4039       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4040       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
4041       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
4042    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
4043       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4044    }
4045
4046    return alpha;
4047 }
4048
4049 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
4050 {
4051    Builder bld(ctx->program, ctx->block);
4052    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4053    if (ctx->shader->info.stage == MESA_SHADER_VERTEX) {
4054
4055       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
4056       if (off_instr->type != nir_instr_type_load_const) {
4057          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4058          nir_print_instr(off_instr, stderr);
4059          fprintf(stderr, "\n");
4060       }
4061       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
4062
4063       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
4064
4065       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
4066       unsigned component = nir_intrinsic_component(instr);
4067       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
4068       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
4069       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
4070       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
4071
4072       unsigned dfmt = attrib_format & 0xf;
4073       unsigned nfmt = (attrib_format >> 4) & 0x7;
4074       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
4075
4076       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
4077       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
4078       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
4079       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
4080       if (post_shuffle)
4081          num_channels = MAX2(num_channels, 3);
4082
4083       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
4084       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
4085
4086       Temp index;
4087       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
4088          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
4089          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
4090          if (divisor) {
4091             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
4092             if (divisor != 1) {
4093                Temp divided = bld.tmp(v1);
4094                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
4095                index = bld.vadd32(bld.def(v1), start_instance, divided);
4096             } else {
4097                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
4098             }
4099          } else {
4100             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
4101          }
4102       } else {
4103          index = bld.vadd32(bld.def(v1),
4104                             get_arg(ctx, ctx->args->ac.base_vertex),
4105                             get_arg(ctx, ctx->args->ac.vertex_id));
4106       }
4107
4108       Temp channels[num_channels];
4109       unsigned channel_start = 0;
4110       bool direct_fetch = false;
4111
4112       /* skip unused channels at the start */
4113       if (vtx_info->chan_byte_size && !post_shuffle) {
4114          channel_start = ffs(mask) - 1;
4115          for (unsigned i = 0; i < channel_start; i++)
4116             channels[i] = Temp(0, s1);
4117       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
4118          num_channels = 3 - (ffs(mask) - 1);
4119       }
4120
4121       /* load channels */
4122       while (channel_start < num_channels) {
4123          unsigned fetch_size = num_channels - channel_start;
4124          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
4125          bool expanded = false;
4126
4127          /* use MUBUF when possible to avoid possible alignment issues */
4128          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
4129          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
4130                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
4131                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
4132                           vtx_info->chan_byte_size == 4;
4133          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
4134          if (!use_mubuf) {
4135             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
4136          } else {
4137             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
4138                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
4139                fetch_size = 4;
4140                expanded = true;
4141             }
4142          }
4143
4144          Temp fetch_index = index;
4145          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
4146             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
4147             fetch_offset = fetch_offset % attrib_stride;
4148          }
4149
4150          Operand soffset(0u);
4151          if (fetch_offset >= 4096) {
4152             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
4153             fetch_offset %= 4096;
4154          }
4155
4156          aco_opcode opcode;
4157          switch (fetch_size) {
4158          case 1:
4159             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
4160             break;
4161          case 2:
4162             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
4163             break;
4164          case 3:
4165             assert(ctx->options->chip_class >= GFX7 ||
4166                    (!use_mubuf && ctx->options->chip_class == GFX6));
4167             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
4168             break;
4169          case 4:
4170             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
4171             break;
4172          default:
4173             unreachable("Unimplemented load_input vector size");
4174          }
4175
4176          Temp fetch_dst;
4177          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
4178              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
4179                            num_channels <= 3)) {
4180             direct_fetch = true;
4181             fetch_dst = dst;
4182          } else {
4183             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
4184          }
4185
4186          if (use_mubuf) {
4187             Instruction *mubuf = bld.mubuf(opcode,
4188                                            Definition(fetch_dst), list, fetch_index, soffset,
4189                                            fetch_offset, false, true).instr;
4190             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
4191          } else {
4192             Instruction *mtbuf = bld.mtbuf(opcode,
4193                                            Definition(fetch_dst), list, fetch_index, soffset,
4194                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
4195             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
4196          }
4197
4198          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
4199
4200          if (fetch_size == 1) {
4201             channels[channel_start] = fetch_dst;
4202          } else {
4203             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
4204                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
4205          }
4206
4207          channel_start += fetch_size;
4208       }
4209
4210       if (!direct_fetch) {
4211          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
4212                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
4213
4214          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
4215          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
4216          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
4217
4218          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4219          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4220          unsigned num_temp = 0;
4221          for (unsigned i = 0; i < dst.size(); i++) {
4222             unsigned idx = i + component;
4223             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
4224                Temp channel = channels[swizzle[idx]];
4225                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
4226                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
4227                vec->operands[i] = Operand(channel);
4228
4229                num_temp++;
4230                elems[i] = channel;
4231             } else if (is_float && idx == 3) {
4232                vec->operands[i] = Operand(0x3f800000u);
4233             } else if (!is_float && idx == 3) {
4234                vec->operands[i] = Operand(1u);
4235             } else {
4236                vec->operands[i] = Operand(0u);
4237             }
4238          }
4239          vec->definitions[0] = Definition(dst);
4240          ctx->block->instructions.emplace_back(std::move(vec));
4241          emit_split_vector(ctx, dst, dst.size());
4242
4243          if (num_temp == dst.size())
4244             ctx->allocated_vec.emplace(dst.id(), elems);
4245       }
4246    } else if (ctx->shader->info.stage == MESA_SHADER_FRAGMENT) {
4247       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
4248       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
4249       if (off_instr->type != nir_instr_type_load_const ||
4250           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
4251          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4252          nir_print_instr(off_instr, stderr);
4253          fprintf(stderr, "\n");
4254       }
4255
4256       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
4257       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
4258       if (offset) {
4259          assert(offset->u32 == 0);
4260       } else {
4261          /* the lower 15bit of the prim_mask contain the offset into LDS
4262           * while the upper bits contain the number of prims */
4263          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
4264          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
4265          Builder bld(ctx->program, ctx->block);
4266          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
4267          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
4268          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
4269          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
4270          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
4271       }
4272
4273       unsigned idx = nir_intrinsic_base(instr);
4274       unsigned component = nir_intrinsic_component(instr);
4275       unsigned vertex_id = 2; /* P0 */
4276
4277       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
4278          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
4279          switch (src0->u32) {
4280          case 0:
4281             vertex_id = 2; /* P0 */
4282             break;
4283          case 1:
4284             vertex_id = 0; /* P10 */
4285             break;
4286          case 2:
4287             vertex_id = 1; /* P20 */
4288             break;
4289          default:
4290             unreachable("invalid vertex index");
4291          }
4292       }
4293
4294       if (dst.size() == 1) {
4295          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
4296       } else {
4297          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4298          for (unsigned i = 0; i < dst.size(); i++)
4299             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
4300          vec->definitions[0] = Definition(dst);
4301          bld.insert(std::move(vec));
4302       }
4303
4304    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_EVAL) {
4305       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4306       Temp soffset = get_arg(ctx, ctx->args->oc_lds);
4307       std::pair<Temp, unsigned> offs = get_tcs_per_patch_output_vmem_offset(ctx, instr);
4308       unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8u;
4309
4310       load_vmem_mubuf(ctx, dst, ring, offs.first, soffset, offs.second, elem_size_bytes, instr->dest.ssa.num_components);
4311    } else {
4312       unreachable("Shader stage not implemented");
4313    }
4314 }
4315
4316 std::pair<Temp, unsigned> get_gs_per_vertex_input_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride = 1u)
4317 {
4318    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4319
4320    Builder bld(ctx->program, ctx->block);
4321    nir_src *vertex_src = nir_get_io_vertex_index_src(instr);
4322    Temp vertex_offset;
4323
4324    if (!nir_src_is_const(*vertex_src)) {
4325       /* better code could be created, but this case probably doesn't happen
4326        * much in practice */
4327       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, vertex_src->ssa));
4328       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
4329          Temp elem;
4330
4331          if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4332             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
4333             if (i % 2u)
4334                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
4335          } else {
4336             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
4337          }
4338
4339          if (vertex_offset.id()) {
4340             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)),
4341                                  Operand(i), indirect_vertex);
4342             vertex_offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), vertex_offset, elem, cond);
4343          } else {
4344             vertex_offset = elem;
4345          }
4346       }
4347
4348       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4349          vertex_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), vertex_offset);
4350    } else {
4351       unsigned vertex = nir_src_as_uint(*vertex_src);
4352       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4353          vertex_offset = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
4354                                   get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
4355                                   Operand((vertex % 2u) * 16u), Operand(16u));
4356       else
4357          vertex_offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
4358    }
4359
4360    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, base_stride);
4361    offs = offset_add(ctx, offs, std::make_pair(vertex_offset, 0u));
4362    return offset_mul(ctx, offs, 4u);
4363 }
4364
4365 void visit_load_gs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4366 {
4367    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4368
4369    Builder bld(ctx->program, ctx->block);
4370    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4371    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4372
4373    if (ctx->stage == geometry_gs) {
4374       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr, ctx->program->wave_size);
4375       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
4376       load_vmem_mubuf(ctx, dst, ring, offs.first, Temp(), offs.second, elem_size_bytes, instr->dest.ssa.num_components, 4u * ctx->program->wave_size, false, true);
4377    } else if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4378       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr);
4379       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4380       load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4381    } else {
4382       unreachable("Unsupported GS stage.");
4383    }
4384 }
4385
4386 void visit_load_tcs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4387 {
4388    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4389
4390    Builder bld(ctx->program, ctx->block);
4391    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4392
4393    if (load_input_from_temps(ctx, instr, dst))
4394       return;
4395
4396    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_input_lds_offset(ctx, instr);
4397    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4398    unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4399
4400    load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4401 }
4402
4403 void visit_load_tes_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4404 {
4405    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4406
4407    Builder bld(ctx->program, ctx->block);
4408
4409    Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4410    Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
4411    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4412
4413    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4414    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_output_vmem_offset(ctx, instr);
4415
4416    load_vmem_mubuf(ctx, dst, ring, offs.first, oc_lds, offs.second, elem_size_bytes, instr->dest.ssa.num_components, 0u, true, true);
4417 }
4418
4419 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4420 {
4421    switch (ctx->shader->info.stage) {
4422    case MESA_SHADER_GEOMETRY:
4423       visit_load_gs_per_vertex_input(ctx, instr);
4424       break;
4425    case MESA_SHADER_TESS_CTRL:
4426       visit_load_tcs_per_vertex_input(ctx, instr);
4427       break;
4428    case MESA_SHADER_TESS_EVAL:
4429       visit_load_tes_per_vertex_input(ctx, instr);
4430       break;
4431    default:
4432       unreachable("Unimplemented shader stage");
4433    }
4434 }
4435
4436 void visit_load_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4437 {
4438    visit_load_tcs_output(ctx, instr, true);
4439 }
4440
4441 void visit_store_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4442 {
4443    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
4444    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4445
4446    visit_store_tcs_output(ctx, instr, true);
4447 }
4448
4449 void visit_load_tess_coord(isel_context *ctx, nir_intrinsic_instr *instr)
4450 {
4451    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4452
4453    Builder bld(ctx->program, ctx->block);
4454    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4455
4456    Operand tes_u(get_arg(ctx, ctx->args->tes_u));
4457    Operand tes_v(get_arg(ctx, ctx->args->tes_v));
4458    Operand tes_w(0u);
4459
4460    if (ctx->shader->info.tess.primitive_mode == GL_TRIANGLES) {
4461       Temp tmp = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), tes_u, tes_v);
4462       tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0x3f800000u /* 1.0f */), tmp);
4463       tes_w = Operand(tmp);
4464    }
4465
4466    Temp tess_coord = bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tes_u, tes_v, tes_w);
4467    emit_split_vector(ctx, tess_coord, 3);
4468 }
4469
4470 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
4471 {
4472    if (ctx->program->info->need_indirect_descriptor_sets) {
4473       Builder bld(ctx->program, ctx->block);
4474       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
4475       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
4476       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
4477    }
4478
4479    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
4480 }
4481
4482
4483 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
4484 {
4485    Builder bld(ctx->program, ctx->block);
4486    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
4487    if (!ctx->divergent_vals[instr->dest.ssa.index])
4488       index = bld.as_uniform(index);
4489    unsigned desc_set = nir_intrinsic_desc_set(instr);
4490    unsigned binding = nir_intrinsic_binding(instr);
4491
4492    Temp desc_ptr;
4493    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
4494    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
4495    unsigned offset = layout->binding[binding].offset;
4496    unsigned stride;
4497    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
4498        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
4499       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
4500       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
4501       offset = pipeline_layout->push_constant_size + 16 * idx;
4502       stride = 16;
4503    } else {
4504       desc_ptr = load_desc_ptr(ctx, desc_set);
4505       stride = layout->binding[binding].size;
4506    }
4507
4508    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
4509    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
4510    if (stride != 1) {
4511       if (nir_const_index) {
4512          const_index = const_index * stride;
4513       } else if (index.type() == RegType::vgpr) {
4514          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
4515          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
4516       } else {
4517          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
4518       }
4519    }
4520    if (offset) {
4521       if (nir_const_index) {
4522          const_index = const_index + offset;
4523       } else if (index.type() == RegType::vgpr) {
4524          index = bld.vadd32(bld.def(v1), Operand(offset), index);
4525       } else {
4526          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
4527       }
4528    }
4529
4530    if (nir_const_index && const_index == 0) {
4531       index = desc_ptr;
4532    } else if (index.type() == RegType::vgpr) {
4533       index = bld.vadd32(bld.def(v1),
4534                          nir_const_index ? Operand(const_index) : Operand(index),
4535                          Operand(desc_ptr));
4536    } else {
4537       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4538                        nir_const_index ? Operand(const_index) : Operand(index),
4539                        Operand(desc_ptr));
4540    }
4541
4542    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
4543 }
4544
4545 void load_buffer(isel_context *ctx, unsigned num_components, unsigned component_size,
4546                  Temp dst, Temp rsrc, Temp offset, int byte_align,
4547                  bool glc=false, bool readonly=true)
4548 {
4549    Builder bld(ctx->program, ctx->block);
4550    bool dlc = glc && ctx->options->chip_class >= GFX10;
4551    unsigned num_bytes = num_components * component_size;
4552
4553    aco_opcode op;
4554    if (dst.type() == RegType::vgpr || ((ctx->options->chip_class < GFX8 || component_size < 4) && !readonly)) {
4555       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4556       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4557       unsigned const_offset = 0;
4558
4559       /* for small bit sizes add buffer for unaligned loads */
4560       if (byte_align) {
4561          if (num_bytes > 2)
4562             num_bytes += byte_align == -1 ? 4 - component_size : byte_align;
4563          else
4564             byte_align = 0;
4565       }
4566
4567       Temp lower = Temp();
4568       if (num_bytes > 16) {
4569          assert(num_components == 3 || num_components == 4);
4570          op = aco_opcode::buffer_load_dwordx4;
4571          lower = bld.tmp(v4);
4572          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4573          mubuf->definitions[0] = Definition(lower);
4574          mubuf->operands[0] = Operand(rsrc);
4575          mubuf->operands[1] = vaddr;
4576          mubuf->operands[2] = soffset;
4577          mubuf->offen = (offset.type() == RegType::vgpr);
4578          mubuf->glc = glc;
4579          mubuf->dlc = dlc;
4580          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4581          mubuf->can_reorder = readonly;
4582          bld.insert(std::move(mubuf));
4583          emit_split_vector(ctx, lower, 2);
4584          num_bytes -= 16;
4585          const_offset = 16;
4586       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
4587          /* GFX6 doesn't support loading vec3, expand to vec4. */
4588          num_bytes = 16;
4589       }
4590
4591       switch (num_bytes) {
4592          case 1:
4593             op = aco_opcode::buffer_load_ubyte;
4594             break;
4595          case 2:
4596             op = aco_opcode::buffer_load_ushort;
4597             break;
4598          case 3:
4599          case 4:
4600             op = aco_opcode::buffer_load_dword;
4601             break;
4602          case 5:
4603          case 6:
4604          case 7:
4605          case 8:
4606             op = aco_opcode::buffer_load_dwordx2;
4607             break;
4608          case 10:
4609          case 12:
4610             assert(ctx->options->chip_class > GFX6);
4611             op = aco_opcode::buffer_load_dwordx3;
4612             break;
4613          case 16:
4614             op = aco_opcode::buffer_load_dwordx4;
4615             break;
4616          default:
4617             unreachable("Load SSBO not implemented for this size.");
4618       }
4619       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4620       mubuf->operands[0] = Operand(rsrc);
4621       mubuf->operands[1] = vaddr;
4622       mubuf->operands[2] = soffset;
4623       mubuf->offen = (offset.type() == RegType::vgpr);
4624       mubuf->glc = glc;
4625       mubuf->dlc = dlc;
4626       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4627       mubuf->can_reorder = readonly;
4628       mubuf->offset = const_offset;
4629       aco_ptr<Instruction> instr = std::move(mubuf);
4630
4631       if (component_size < 4) {
4632          Temp vec = num_bytes <= 4 ? bld.tmp(v1) : num_bytes <= 8 ? bld.tmp(v2) : bld.tmp(v3);
4633          instr->definitions[0] = Definition(vec);
4634          bld.insert(std::move(instr));
4635
4636          if (byte_align == -1 || (byte_align && dst.type() == RegType::sgpr)) {
4637             Operand align = byte_align == -1 ? Operand(offset) : Operand((uint32_t)byte_align);
4638             Temp tmp[3] = {vec, vec, vec};
4639
4640             if (vec.size() == 3) {
4641                tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1);
4642                bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), vec);
4643             } else if (vec.size() == 2) {
4644                tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = tmp[1];
4645                bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), vec);
4646             }
4647             for (unsigned i = 0; i < dst.size(); i++)
4648                tmp[i] = bld.vop3(aco_opcode::v_alignbyte_b32, bld.def(v1), tmp[i + 1], tmp[i], align);
4649
4650             vec = tmp[0];
4651             if (dst.size() == 2)
4652                vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), tmp[0], tmp[1]);
4653
4654             byte_align = 0;
4655          }
4656
4657          if (dst.type() == RegType::vgpr && num_components == 1) {
4658             bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), vec, Operand(byte_align / component_size));
4659          } else {
4660             trim_subdword_vector(ctx, vec, dst, 4 * vec.size() / component_size, ((1 << num_components) - 1) << byte_align / component_size);
4661          }
4662
4663          return;
4664
4665       } else if (dst.size() > 4) {
4666          assert(lower != Temp());
4667          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
4668          instr->definitions[0] = Definition(upper);
4669          bld.insert(std::move(instr));
4670          if (dst.size() == 8)
4671             emit_split_vector(ctx, upper, 2);
4672          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
4673          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
4674          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
4675          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
4676          if (dst.size() == 8)
4677             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
4678       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
4679          Temp vec = bld.tmp(v4);
4680          instr->definitions[0] = Definition(vec);
4681          bld.insert(std::move(instr));
4682          emit_split_vector(ctx, vec, 4);
4683
4684          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4685          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4686          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4687          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4688       }
4689
4690       if (dst.type() == RegType::sgpr) {
4691          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4692          instr->definitions[0] = Definition(vec);
4693          bld.insert(std::move(instr));
4694          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4695       } else {
4696          instr->definitions[0] = Definition(dst);
4697          bld.insert(std::move(instr));
4698          emit_split_vector(ctx, dst, num_components);
4699       }
4700    } else {
4701       /* for small bit sizes add buffer for unaligned loads */
4702       if (byte_align)
4703          num_bytes += byte_align == -1 ? 4 - component_size : byte_align;
4704
4705       switch (num_bytes) {
4706          case 1:
4707          case 2:
4708          case 3:
4709          case 4:
4710             op = aco_opcode::s_buffer_load_dword;
4711             break;
4712          case 5:
4713          case 6:
4714          case 7:
4715          case 8:
4716             op = aco_opcode::s_buffer_load_dwordx2;
4717             break;
4718          case 10:
4719          case 12:
4720          case 16:
4721             op = aco_opcode::s_buffer_load_dwordx4;
4722             break;
4723          case 24:
4724          case 32:
4725             op = aco_opcode::s_buffer_load_dwordx8;
4726             break;
4727          default:
4728             unreachable("Load SSBO not implemented for this size.");
4729       }
4730       offset = bld.as_uniform(offset);
4731       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4732       load->operands[0] = Operand(rsrc);
4733       load->operands[1] = Operand(offset);
4734       assert(load->operands[1].getTemp().type() == RegType::sgpr);
4735       load->definitions[0] = Definition(dst);
4736       load->glc = glc;
4737       load->dlc = dlc;
4738       load->barrier = readonly ? barrier_none : barrier_buffer;
4739       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
4740       assert(ctx->options->chip_class >= GFX8 || !glc);
4741
4742       /* adjust misaligned small bit size loads */
4743       if (byte_align) {
4744          Temp vec = num_bytes <= 4 ? bld.tmp(s1) : num_bytes <= 8 ? bld.tmp(s2) : bld.tmp(s4);
4745          load->definitions[0] = Definition(vec);
4746          bld.insert(std::move(load));
4747          Operand byte_offset = byte_align > 0 ? Operand(uint32_t(byte_align)) : Operand(offset);
4748          byte_align_scalar(ctx, vec, byte_offset, dst);
4749
4750       /* trim vector */
4751       } else if (dst.size() == 3) {
4752          Temp vec = bld.tmp(s4);
4753          load->definitions[0] = Definition(vec);
4754          bld.insert(std::move(load));
4755          emit_split_vector(ctx, vec, 4);
4756
4757          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4758                     emit_extract_vector(ctx, vec, 0, s1),
4759                     emit_extract_vector(ctx, vec, 1, s1),
4760                     emit_extract_vector(ctx, vec, 2, s1));
4761       } else if (dst.size() == 6) {
4762          Temp vec = bld.tmp(s8);
4763          load->definitions[0] = Definition(vec);
4764          bld.insert(std::move(load));
4765          emit_split_vector(ctx, vec, 4);
4766
4767          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4768                     emit_extract_vector(ctx, vec, 0, s2),
4769                     emit_extract_vector(ctx, vec, 1, s2),
4770                     emit_extract_vector(ctx, vec, 2, s2));
4771       } else {
4772          bld.insert(std::move(load));
4773       }
4774       emit_split_vector(ctx, dst, num_components);
4775    }
4776 }
4777
4778 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
4779 {
4780    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4781    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
4782
4783    Builder bld(ctx->program, ctx->block);
4784
4785    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
4786    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
4787    unsigned binding = nir_intrinsic_binding(idx_instr);
4788    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
4789
4790    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
4791       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4792                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4793                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4794                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4795       if (ctx->options->chip_class >= GFX10) {
4796          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4797                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4798                       S_008F0C_RESOURCE_LEVEL(1);
4799       } else {
4800          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4801                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4802       }
4803       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
4804                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
4805                                      Operand(0xFFFFFFFFu),
4806                                      Operand(desc_type));
4807       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4808                         rsrc, upper_dwords);
4809    } else {
4810       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
4811       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4812    }
4813    unsigned size = instr->dest.ssa.bit_size / 8;
4814    int byte_align = 0;
4815    if (size < 4) {
4816       unsigned align_mul = nir_intrinsic_align_mul(instr);
4817       unsigned align_offset = nir_intrinsic_align_offset(instr);
4818       byte_align = align_mul % 4 == 0 ? align_offset : -1;
4819    }
4820    load_buffer(ctx, instr->num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), byte_align);
4821 }
4822
4823 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4824 {
4825    Builder bld(ctx->program, ctx->block);
4826    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4827    unsigned offset = nir_intrinsic_base(instr);
4828    unsigned count = instr->dest.ssa.num_components;
4829    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
4830
4831    if (index_cv && instr->dest.ssa.bit_size == 32) {
4832       unsigned start = (offset + index_cv->u32) / 4u;
4833       start -= ctx->args->ac.base_inline_push_consts;
4834       if (start + count <= ctx->args->ac.num_inline_push_consts) {
4835          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4836          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4837          for (unsigned i = 0; i < count; ++i) {
4838             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
4839             vec->operands[i] = Operand{elems[i]};
4840          }
4841          vec->definitions[0] = Definition(dst);
4842          ctx->block->instructions.emplace_back(std::move(vec));
4843          ctx->allocated_vec.emplace(dst.id(), elems);
4844          return;
4845       }
4846    }
4847
4848    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
4849    if (offset != 0) // TODO check if index != 0 as well
4850       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
4851    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
4852    Temp vec = dst;
4853    bool trim = false;
4854    bool aligned = true;
4855
4856    if (instr->dest.ssa.bit_size == 8) {
4857       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
4858       bool fits_in_dword = count == 1 || (index_cv && ((offset + index_cv->u32) % 4 + count) <= 4);
4859       if (!aligned)
4860          vec = fits_in_dword ? bld.tmp(s1) : bld.tmp(s2);
4861    } else if (instr->dest.ssa.bit_size == 16) {
4862       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
4863       if (!aligned)
4864          vec = count == 4 ? bld.tmp(s4) : count > 1 ? bld.tmp(s2) : bld.tmp(s1);
4865    }
4866
4867    aco_opcode op;
4868
4869    switch (vec.size()) {
4870    case 1:
4871       op = aco_opcode::s_load_dword;
4872       break;
4873    case 2:
4874       op = aco_opcode::s_load_dwordx2;
4875       break;
4876    case 3:
4877       vec = bld.tmp(s4);
4878       trim = true;
4879    case 4:
4880       op = aco_opcode::s_load_dwordx4;
4881       break;
4882    case 6:
4883       vec = bld.tmp(s8);
4884       trim = true;
4885    case 8:
4886       op = aco_opcode::s_load_dwordx8;
4887       break;
4888    default:
4889       unreachable("unimplemented or forbidden load_push_constant.");
4890    }
4891
4892    bld.smem(op, Definition(vec), ptr, index);
4893
4894    if (!aligned) {
4895       Operand byte_offset = index_cv ? Operand((offset + index_cv->u32) % 4) : Operand(index);
4896       byte_align_scalar(ctx, vec, byte_offset, dst);
4897       return;
4898    }
4899
4900    if (trim) {
4901       emit_split_vector(ctx, vec, 4);
4902       RegClass rc = dst.size() == 3 ? s1 : s2;
4903       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4904                  emit_extract_vector(ctx, vec, 0, rc),
4905                  emit_extract_vector(ctx, vec, 1, rc),
4906                  emit_extract_vector(ctx, vec, 2, rc));
4907
4908    }
4909    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4910 }
4911
4912 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4913 {
4914    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4915
4916    Builder bld(ctx->program, ctx->block);
4917
4918    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4919                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4920                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4921                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4922    if (ctx->options->chip_class >= GFX10) {
4923       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4924                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4925                    S_008F0C_RESOURCE_LEVEL(1);
4926    } else {
4927       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4928                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4929    }
4930
4931    unsigned base = nir_intrinsic_base(instr);
4932    unsigned range = nir_intrinsic_range(instr);
4933
4934    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
4935    if (base && offset.type() == RegType::sgpr)
4936       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
4937    else if (base && offset.type() == RegType::vgpr)
4938       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
4939
4940    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4941                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
4942                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
4943                           Operand(desc_type));
4944    unsigned size = instr->dest.ssa.bit_size / 8;
4945    // TODO: get alignment information for subdword constants
4946    unsigned byte_align = size < 4 ? -1 : 0;
4947    load_buffer(ctx, instr->num_components, size, dst, rsrc, offset, byte_align);
4948 }
4949
4950 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
4951 {
4952    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4953       ctx->cf_info.exec_potentially_empty_discard = true;
4954
4955    ctx->program->needs_exact = true;
4956
4957    // TODO: optimize uniform conditions
4958    Builder bld(ctx->program, ctx->block);
4959    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
4960    assert(src.regClass() == bld.lm);
4961    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
4962    bld.pseudo(aco_opcode::p_discard_if, src);
4963    ctx->block->kind |= block_kind_uses_discard_if;
4964    return;
4965 }
4966
4967 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
4968 {
4969    Builder bld(ctx->program, ctx->block);
4970
4971    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4972       ctx->cf_info.exec_potentially_empty_discard = true;
4973
4974    bool divergent = ctx->cf_info.parent_if.is_divergent ||
4975                     ctx->cf_info.parent_loop.has_divergent_continue;
4976
4977    if (ctx->block->loop_nest_depth &&
4978        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
4979       /* we handle discards the same way as jump instructions */
4980       append_logical_end(ctx->block);
4981
4982       /* in loops, discard behaves like break */
4983       Block *linear_target = ctx->cf_info.parent_loop.exit;
4984       ctx->block->kind |= block_kind_discard;
4985
4986       if (!divergent) {
4987          /* uniform discard - loop ends here */
4988          assert(nir_instr_is_last(&instr->instr));
4989          ctx->block->kind |= block_kind_uniform;
4990          ctx->cf_info.has_branch = true;
4991          bld.branch(aco_opcode::p_branch);
4992          add_linear_edge(ctx->block->index, linear_target);
4993          return;
4994       }
4995
4996       /* we add a break right behind the discard() instructions */
4997       ctx->block->kind |= block_kind_break;
4998       unsigned idx = ctx->block->index;
4999
5000       ctx->cf_info.parent_loop.has_divergent_branch = true;
5001       ctx->cf_info.nir_to_aco[instr->instr.block->index] = idx;
5002
5003       /* remove critical edges from linear CFG */
5004       bld.branch(aco_opcode::p_branch);
5005       Block* break_block = ctx->program->create_and_insert_block();
5006       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5007       break_block->kind |= block_kind_uniform;
5008       add_linear_edge(idx, break_block);
5009       add_linear_edge(break_block->index, linear_target);
5010       bld.reset(break_block);
5011       bld.branch(aco_opcode::p_branch);
5012
5013       Block* continue_block = ctx->program->create_and_insert_block();
5014       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5015       add_linear_edge(idx, continue_block);
5016       append_logical_start(continue_block);
5017       ctx->block = continue_block;
5018
5019       return;
5020    }
5021
5022    /* it can currently happen that NIR doesn't remove the unreachable code */
5023    if (!nir_instr_is_last(&instr->instr)) {
5024       ctx->program->needs_exact = true;
5025       /* save exec somewhere temporarily so that it doesn't get
5026        * overwritten before the discard from outer exec masks */
5027       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
5028       bld.pseudo(aco_opcode::p_discard_if, cond);
5029       ctx->block->kind |= block_kind_uses_discard_if;
5030       return;
5031    }
5032
5033    /* This condition is incorrect for uniformly branched discards in a loop
5034     * predicated by a divergent condition, but the above code catches that case
5035     * and the discard would end up turning into a discard_if.
5036     * For example:
5037     * if (divergent) {
5038     *    while (...) {
5039     *       if (uniform) {
5040     *          discard;
5041     *       }
5042     *    }
5043     * }
5044     */
5045    if (!ctx->cf_info.parent_if.is_divergent) {
5046       /* program just ends here */
5047       ctx->block->kind |= block_kind_uniform;
5048       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
5049               0 /* enabled mask */, 9 /* dest */,
5050               false /* compressed */, true/* done */, true /* valid mask */);
5051       bld.sopp(aco_opcode::s_endpgm);
5052       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
5053    } else {
5054       ctx->block->kind |= block_kind_discard;
5055       /* branch and linear edge is added by visit_if() */
5056    }
5057 }
5058
5059 enum aco_descriptor_type {
5060    ACO_DESC_IMAGE,
5061    ACO_DESC_FMASK,
5062    ACO_DESC_SAMPLER,
5063    ACO_DESC_BUFFER,
5064    ACO_DESC_PLANE_0,
5065    ACO_DESC_PLANE_1,
5066    ACO_DESC_PLANE_2,
5067 };
5068
5069 static bool
5070 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
5071    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
5072       return false;
5073    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
5074    return dim == ac_image_cube ||
5075           dim == ac_image_1darray ||
5076           dim == ac_image_2darray ||
5077           dim == ac_image_2darraymsaa;
5078 }
5079
5080 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
5081                       enum aco_descriptor_type desc_type,
5082                       const nir_tex_instr *tex_instr, bool image, bool write)
5083 {
5084 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
5085    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
5086    if (it != ctx->tex_desc.end())
5087       return it->second;
5088 */
5089    Temp index = Temp();
5090    bool index_set = false;
5091    unsigned constant_index = 0;
5092    unsigned descriptor_set;
5093    unsigned base_index;
5094    Builder bld(ctx->program, ctx->block);
5095
5096    if (!deref_instr) {
5097       assert(tex_instr && !image);
5098       descriptor_set = 0;
5099       base_index = tex_instr->sampler_index;
5100    } else {
5101       while(deref_instr->deref_type != nir_deref_type_var) {
5102          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
5103          if (!array_size)
5104             array_size = 1;
5105
5106          assert(deref_instr->deref_type == nir_deref_type_array);
5107          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
5108          if (const_value) {
5109             constant_index += array_size * const_value->u32;
5110          } else {
5111             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
5112             if (indirect.type() == RegType::vgpr)
5113                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
5114
5115             if (array_size != 1)
5116                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
5117
5118             if (!index_set) {
5119                index = indirect;
5120                index_set = true;
5121             } else {
5122                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
5123             }
5124          }
5125
5126          deref_instr = nir_src_as_deref(deref_instr->parent);
5127       }
5128       descriptor_set = deref_instr->var->data.descriptor_set;
5129       base_index = deref_instr->var->data.binding;
5130    }
5131
5132    Temp list = load_desc_ptr(ctx, descriptor_set);
5133    list = convert_pointer_to_64_bit(ctx, list);
5134
5135    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
5136    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
5137    unsigned offset = binding->offset;
5138    unsigned stride = binding->size;
5139    aco_opcode opcode;
5140    RegClass type;
5141
5142    assert(base_index < layout->binding_count);
5143
5144    switch (desc_type) {
5145    case ACO_DESC_IMAGE:
5146       type = s8;
5147       opcode = aco_opcode::s_load_dwordx8;
5148       break;
5149    case ACO_DESC_FMASK:
5150       type = s8;
5151       opcode = aco_opcode::s_load_dwordx8;
5152       offset += 32;
5153       break;
5154    case ACO_DESC_SAMPLER:
5155       type = s4;
5156       opcode = aco_opcode::s_load_dwordx4;
5157       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
5158          offset += radv_combined_image_descriptor_sampler_offset(binding);
5159       break;
5160    case ACO_DESC_BUFFER:
5161       type = s4;
5162       opcode = aco_opcode::s_load_dwordx4;
5163       break;
5164    case ACO_DESC_PLANE_0:
5165    case ACO_DESC_PLANE_1:
5166       type = s8;
5167       opcode = aco_opcode::s_load_dwordx8;
5168       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
5169       break;
5170    case ACO_DESC_PLANE_2:
5171       type = s4;
5172       opcode = aco_opcode::s_load_dwordx4;
5173       offset += 64;
5174       break;
5175    default:
5176       unreachable("invalid desc_type\n");
5177    }
5178
5179    offset += constant_index * stride;
5180
5181    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
5182       (!index_set || binding->immutable_samplers_equal)) {
5183       if (binding->immutable_samplers_equal)
5184          constant_index = 0;
5185
5186       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
5187       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5188                         Operand(samplers[constant_index * 4 + 0]),
5189                         Operand(samplers[constant_index * 4 + 1]),
5190                         Operand(samplers[constant_index * 4 + 2]),
5191                         Operand(samplers[constant_index * 4 + 3]));
5192    }
5193
5194    Operand off;
5195    if (!index_set) {
5196       off = bld.copy(bld.def(s1), Operand(offset));
5197    } else {
5198       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
5199                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
5200    }
5201
5202    Temp res = bld.smem(opcode, bld.def(type), list, off);
5203
5204    if (desc_type == ACO_DESC_PLANE_2) {
5205       Temp components[8];
5206       for (unsigned i = 0; i < 8; i++)
5207          components[i] = bld.tmp(s1);
5208       bld.pseudo(aco_opcode::p_split_vector,
5209                  Definition(components[0]),
5210                  Definition(components[1]),
5211                  Definition(components[2]),
5212                  Definition(components[3]),
5213                  res);
5214
5215       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
5216       bld.pseudo(aco_opcode::p_split_vector,
5217                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
5218                  Definition(components[4]),
5219                  Definition(components[5]),
5220                  Definition(components[6]),
5221                  Definition(components[7]),
5222                  desc2);
5223
5224       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
5225                        components[0], components[1], components[2], components[3],
5226                        components[4], components[5], components[6], components[7]);
5227    }
5228
5229    return res;
5230 }
5231
5232 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
5233 {
5234    switch (dim) {
5235    case GLSL_SAMPLER_DIM_BUF:
5236       return 1;
5237    case GLSL_SAMPLER_DIM_1D:
5238       return array ? 2 : 1;
5239    case GLSL_SAMPLER_DIM_2D:
5240       return array ? 3 : 2;
5241    case GLSL_SAMPLER_DIM_MS:
5242       return array ? 4 : 3;
5243    case GLSL_SAMPLER_DIM_3D:
5244    case GLSL_SAMPLER_DIM_CUBE:
5245       return 3;
5246    case GLSL_SAMPLER_DIM_RECT:
5247    case GLSL_SAMPLER_DIM_SUBPASS:
5248       return 2;
5249    case GLSL_SAMPLER_DIM_SUBPASS_MS:
5250       return 3;
5251    default:
5252       break;
5253    }
5254    return 0;
5255 }
5256
5257
5258 /* Adjust the sample index according to FMASK.
5259  *
5260  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
5261  * which is the identity mapping. Each nibble says which physical sample
5262  * should be fetched to get that sample.
5263  *
5264  * For example, 0x11111100 means there are only 2 samples stored and
5265  * the second sample covers 3/4 of the pixel. When reading samples 0
5266  * and 1, return physical sample 0 (determined by the first two 0s
5267  * in FMASK), otherwise return physical sample 1.
5268  *
5269  * The sample index should be adjusted as follows:
5270  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
5271  */
5272 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
5273 {
5274    Builder bld(ctx->program, ctx->block);
5275    Temp fmask = bld.tmp(v1);
5276    unsigned dim = ctx->options->chip_class >= GFX10
5277                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
5278                   : 0;
5279
5280    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
5281                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
5282    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
5283    load->operands[0] = Operand(fmask_desc_ptr);
5284    load->operands[1] = Operand(s4); /* no sampler */
5285    load->operands[2] = Operand(coord);
5286    load->definitions[0] = Definition(fmask);
5287    load->glc = false;
5288    load->dlc = false;
5289    load->dmask = 0x1;
5290    load->unrm = true;
5291    load->da = da;
5292    load->dim = dim;
5293    load->can_reorder = true; /* fmask images shouldn't be modified */
5294    ctx->block->instructions.emplace_back(std::move(load));
5295
5296    Operand sample_index4;
5297    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
5298       sample_index4 = Operand(sample_index.constantValue() << 2);
5299    } else if (sample_index.regClass() == s1) {
5300       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
5301    } else {
5302       assert(sample_index.regClass() == v1);
5303       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
5304    }
5305
5306    Temp final_sample;
5307    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
5308       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
5309    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
5310       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
5311    else
5312       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
5313
5314    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
5315     * resource descriptor is 0 (invalid),
5316     */
5317    Temp compare = bld.tmp(bld.lm);
5318    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
5319                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
5320
5321    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
5322
5323    /* Replace the MSAA sample index. */
5324    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
5325 }
5326
5327 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
5328 {
5329
5330    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
5331    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5332    bool is_array = glsl_sampler_type_is_array(type);
5333    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5334    assert(!add_frag_pos && "Input attachments should be lowered.");
5335    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5336    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
5337    int count = image_type_to_components_count(dim, is_array);
5338    std::vector<Temp> coords(count);
5339    Builder bld(ctx->program, ctx->block);
5340
5341    if (is_ms) {
5342       count--;
5343       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
5344       /* get sample index */
5345       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
5346          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
5347          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
5348          std::vector<Temp> fmask_load_address;
5349          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
5350             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
5351
5352          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
5353          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
5354       } else {
5355          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
5356       }
5357    }
5358
5359    if (gfx9_1d) {
5360       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
5361       coords.resize(coords.size() + 1);
5362       coords[1] = bld.copy(bld.def(v1), Operand(0u));
5363       if (is_array)
5364          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
5365    } else {
5366       for (int i = 0; i < count; i++)
5367          coords[i] = emit_extract_vector(ctx, src0, i, v1);
5368    }
5369
5370    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
5371        instr->intrinsic == nir_intrinsic_image_deref_store) {
5372       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
5373       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
5374
5375       if (!level_zero)
5376          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
5377    }
5378
5379    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
5380    for (unsigned i = 0; i < coords.size(); i++)
5381       vec->operands[i] = Operand(coords[i]);
5382    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
5383    vec->definitions[0] = Definition(res);
5384    ctx->block->instructions.emplace_back(std::move(vec));
5385    return res;
5386 }
5387
5388
5389 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
5390 {
5391    Builder bld(ctx->program, ctx->block);
5392    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5393    const struct glsl_type *type = glsl_without_array(var->type);
5394    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5395    bool is_array = glsl_sampler_type_is_array(type);
5396    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5397
5398    if (dim == GLSL_SAMPLER_DIM_BUF) {
5399       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
5400       unsigned num_channels = util_last_bit(mask);
5401       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5402       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5403
5404       aco_opcode opcode;
5405       switch (num_channels) {
5406       case 1:
5407          opcode = aco_opcode::buffer_load_format_x;
5408          break;
5409       case 2:
5410          opcode = aco_opcode::buffer_load_format_xy;
5411          break;
5412       case 3:
5413          opcode = aco_opcode::buffer_load_format_xyz;
5414          break;
5415       case 4:
5416          opcode = aco_opcode::buffer_load_format_xyzw;
5417          break;
5418       default:
5419          unreachable(">4 channel buffer image load");
5420       }
5421       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
5422       load->operands[0] = Operand(rsrc);
5423       load->operands[1] = Operand(vindex);
5424       load->operands[2] = Operand((uint32_t) 0);
5425       Temp tmp;
5426       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5427          tmp = dst;
5428       else
5429          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
5430       load->definitions[0] = Definition(tmp);
5431       load->idxen = true;
5432       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
5433       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5434       load->barrier = barrier_image;
5435       ctx->block->instructions.emplace_back(std::move(load));
5436
5437       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
5438       return;
5439    }
5440
5441    Temp coords = get_image_coords(ctx, instr, type);
5442    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5443
5444    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
5445    unsigned num_components = util_bitcount(dmask);
5446    Temp tmp;
5447    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5448       tmp = dst;
5449    else
5450       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
5451
5452    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
5453    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
5454
5455    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
5456    load->operands[0] = Operand(resource);
5457    load->operands[1] = Operand(s4); /* no sampler */
5458    load->operands[2] = Operand(coords);
5459    load->definitions[0] = Definition(tmp);
5460    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
5461    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5462    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5463    load->dmask = dmask;
5464    load->unrm = true;
5465    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5466    load->barrier = barrier_image;
5467    ctx->block->instructions.emplace_back(std::move(load));
5468
5469    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
5470    return;
5471 }
5472
5473 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
5474 {
5475    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5476    const struct glsl_type *type = glsl_without_array(var->type);
5477    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5478    bool is_array = glsl_sampler_type_is_array(type);
5479    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5480
5481    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
5482
5483    if (dim == GLSL_SAMPLER_DIM_BUF) {
5484       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5485       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5486       aco_opcode opcode;
5487       switch (data.size()) {
5488       case 1:
5489          opcode = aco_opcode::buffer_store_format_x;
5490          break;
5491       case 2:
5492          opcode = aco_opcode::buffer_store_format_xy;
5493          break;
5494       case 3:
5495          opcode = aco_opcode::buffer_store_format_xyz;
5496          break;
5497       case 4:
5498          opcode = aco_opcode::buffer_store_format_xyzw;
5499          break;
5500       default:
5501          unreachable(">4 channel buffer image store");
5502       }
5503       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
5504       store->operands[0] = Operand(rsrc);
5505       store->operands[1] = Operand(vindex);
5506       store->operands[2] = Operand((uint32_t) 0);
5507       store->operands[3] = Operand(data);
5508       store->idxen = true;
5509       store->glc = glc;
5510       store->dlc = false;
5511       store->disable_wqm = true;
5512       store->barrier = barrier_image;
5513       ctx->program->needs_exact = true;
5514       ctx->block->instructions.emplace_back(std::move(store));
5515       return;
5516    }
5517
5518    assert(data.type() == RegType::vgpr);
5519    Temp coords = get_image_coords(ctx, instr, type);
5520    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5521
5522    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
5523    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
5524
5525    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
5526    store->operands[0] = Operand(resource);
5527    store->operands[1] = Operand(data);
5528    store->operands[2] = Operand(coords);
5529    store->glc = glc;
5530    store->dlc = false;
5531    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5532    store->dmask = (1 << data.size()) - 1;
5533    store->unrm = true;
5534    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5535    store->disable_wqm = true;
5536    store->barrier = barrier_image;
5537    ctx->program->needs_exact = true;
5538    ctx->block->instructions.emplace_back(std::move(store));
5539    return;
5540 }
5541
5542 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5543 {
5544    /* return the previous value if dest is ever used */
5545    bool return_previous = false;
5546    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5547       return_previous = true;
5548       break;
5549    }
5550    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5551       return_previous = true;
5552       break;
5553    }
5554
5555    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5556    const struct glsl_type *type = glsl_without_array(var->type);
5557    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5558    bool is_array = glsl_sampler_type_is_array(type);
5559    Builder bld(ctx->program, ctx->block);
5560
5561    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5562    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
5563
5564    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
5565       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
5566
5567    aco_opcode buf_op, image_op;
5568    switch (instr->intrinsic) {
5569       case nir_intrinsic_image_deref_atomic_add:
5570          buf_op = aco_opcode::buffer_atomic_add;
5571          image_op = aco_opcode::image_atomic_add;
5572          break;
5573       case nir_intrinsic_image_deref_atomic_umin:
5574          buf_op = aco_opcode::buffer_atomic_umin;
5575          image_op = aco_opcode::image_atomic_umin;
5576          break;
5577       case nir_intrinsic_image_deref_atomic_imin:
5578          buf_op = aco_opcode::buffer_atomic_smin;
5579          image_op = aco_opcode::image_atomic_smin;
5580          break;
5581       case nir_intrinsic_image_deref_atomic_umax:
5582          buf_op = aco_opcode::buffer_atomic_umax;
5583          image_op = aco_opcode::image_atomic_umax;
5584          break;
5585       case nir_intrinsic_image_deref_atomic_imax:
5586          buf_op = aco_opcode::buffer_atomic_smax;
5587          image_op = aco_opcode::image_atomic_smax;
5588          break;
5589       case nir_intrinsic_image_deref_atomic_and:
5590          buf_op = aco_opcode::buffer_atomic_and;
5591          image_op = aco_opcode::image_atomic_and;
5592          break;
5593       case nir_intrinsic_image_deref_atomic_or:
5594          buf_op = aco_opcode::buffer_atomic_or;
5595          image_op = aco_opcode::image_atomic_or;
5596          break;
5597       case nir_intrinsic_image_deref_atomic_xor:
5598          buf_op = aco_opcode::buffer_atomic_xor;
5599          image_op = aco_opcode::image_atomic_xor;
5600          break;
5601       case nir_intrinsic_image_deref_atomic_exchange:
5602          buf_op = aco_opcode::buffer_atomic_swap;
5603          image_op = aco_opcode::image_atomic_swap;
5604          break;
5605       case nir_intrinsic_image_deref_atomic_comp_swap:
5606          buf_op = aco_opcode::buffer_atomic_cmpswap;
5607          image_op = aco_opcode::image_atomic_cmpswap;
5608          break;
5609       default:
5610          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
5611    }
5612
5613    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5614
5615    if (dim == GLSL_SAMPLER_DIM_BUF) {
5616       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5617       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5618       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
5619       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5620       mubuf->operands[0] = Operand(resource);
5621       mubuf->operands[1] = Operand(vindex);
5622       mubuf->operands[2] = Operand((uint32_t)0);
5623       mubuf->operands[3] = Operand(data);
5624       if (return_previous)
5625          mubuf->definitions[0] = Definition(dst);
5626       mubuf->offset = 0;
5627       mubuf->idxen = true;
5628       mubuf->glc = return_previous;
5629       mubuf->dlc = false; /* Not needed for atomics */
5630       mubuf->disable_wqm = true;
5631       mubuf->barrier = barrier_image;
5632       ctx->program->needs_exact = true;
5633       ctx->block->instructions.emplace_back(std::move(mubuf));
5634       return;
5635    }
5636
5637    Temp coords = get_image_coords(ctx, instr, type);
5638    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5639    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
5640    mimg->operands[0] = Operand(resource);
5641    mimg->operands[1] = Operand(data);
5642    mimg->operands[2] = Operand(coords);
5643    if (return_previous)
5644       mimg->definitions[0] = Definition(dst);
5645    mimg->glc = return_previous;
5646    mimg->dlc = false; /* Not needed for atomics */
5647    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5648    mimg->dmask = (1 << data.size()) - 1;
5649    mimg->unrm = true;
5650    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5651    mimg->disable_wqm = true;
5652    mimg->barrier = barrier_image;
5653    ctx->program->needs_exact = true;
5654    ctx->block->instructions.emplace_back(std::move(mimg));
5655    return;
5656 }
5657
5658 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
5659 {
5660    if (in_elements && ctx->options->chip_class == GFX8) {
5661       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
5662       Builder bld(ctx->program, ctx->block);
5663
5664       Temp size = emit_extract_vector(ctx, desc, 2, s1);
5665
5666       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
5667       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
5668
5669       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
5670       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
5671
5672       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
5673       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
5674
5675       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
5676       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
5677                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
5678       if (dst.type() == RegType::vgpr)
5679          bld.copy(Definition(dst), shr_dst);
5680
5681       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
5682    } else {
5683       emit_extract_vector(ctx, desc, 2, dst);
5684    }
5685 }
5686
5687 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
5688 {
5689    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5690    const struct glsl_type *type = glsl_without_array(var->type);
5691    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5692    bool is_array = glsl_sampler_type_is_array(type);
5693    Builder bld(ctx->program, ctx->block);
5694
5695    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
5696       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
5697       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
5698    }
5699
5700    /* LOD */
5701    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
5702
5703    /* Resource */
5704    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
5705
5706    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5707
5708    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
5709    mimg->operands[0] = Operand(resource);
5710    mimg->operands[1] = Operand(s4); /* no sampler */
5711    mimg->operands[2] = Operand(lod);
5712    uint8_t& dmask = mimg->dmask;
5713    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5714    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
5715    mimg->da = glsl_sampler_type_is_array(type);
5716    mimg->can_reorder = true;
5717    Definition& def = mimg->definitions[0];
5718    ctx->block->instructions.emplace_back(std::move(mimg));
5719
5720    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
5721        glsl_sampler_type_is_array(type)) {
5722
5723       assert(instr->dest.ssa.num_components == 3);
5724       Temp tmp = {ctx->program->allocateId(), v3};
5725       def = Definition(tmp);
5726       emit_split_vector(ctx, tmp, 3);
5727
5728       /* divide 3rd value by 6 by multiplying with magic number */
5729       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
5730       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
5731
5732       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5733                  emit_extract_vector(ctx, tmp, 0, v1),
5734                  emit_extract_vector(ctx, tmp, 1, v1),
5735                  by_6);
5736
5737    } else if (ctx->options->chip_class == GFX9 &&
5738               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
5739               glsl_sampler_type_is_array(type)) {
5740       assert(instr->dest.ssa.num_components == 2);
5741       def = Definition(dst);
5742       dmask = 0x5;
5743    } else {
5744       def = Definition(dst);
5745    }
5746
5747    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
5748 }
5749
5750 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5751 {
5752    Builder bld(ctx->program, ctx->block);
5753    unsigned num_components = instr->num_components;
5754
5755    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5756    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5757    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5758
5759    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5760    unsigned size = instr->dest.ssa.bit_size / 8;
5761    int byte_align = 0;
5762    if (size < 4) {
5763       unsigned align_mul = nir_intrinsic_align_mul(instr);
5764       unsigned align_offset = nir_intrinsic_align_offset(instr);
5765       byte_align = align_mul % 4 == 0 ? align_offset : -1;
5766    }
5767    load_buffer(ctx, num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), byte_align, glc, false);
5768 }
5769
5770 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5771 {
5772    Builder bld(ctx->program, ctx->block);
5773    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5774    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5775    unsigned writemask = nir_intrinsic_write_mask(instr);
5776    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
5777
5778    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5779    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5780
5781    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
5782                ctx->options->chip_class >= GFX8 &&
5783                elem_size_bytes >= 4;
5784    if (smem)
5785       offset = bld.as_uniform(offset);
5786    bool smem_nonfs = smem && ctx->stage != fragment_fs;
5787
5788    while (writemask) {
5789       int start, count;
5790       u_bit_scan_consecutive_range(&writemask, &start, &count);
5791       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
5792          /* GFX6 doesn't support storing vec3, split it. */
5793          writemask |= 1u << (start + 2);
5794          count = 2;
5795       }
5796       int num_bytes = count * elem_size_bytes;
5797
5798       /* dword or larger stores have to be dword-aligned */
5799       if (elem_size_bytes < 4 && num_bytes > 2) {
5800          // TODO: improve alignment check of sub-dword stores
5801          unsigned count_new = 2 / elem_size_bytes;
5802          writemask |= ((1 << (count - count_new)) - 1) << (start + count_new);
5803          count = count_new;
5804          num_bytes = 2;
5805       }
5806
5807       if (num_bytes > 16) {
5808          assert(elem_size_bytes == 8);
5809          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5810          count = 2;
5811          num_bytes = 16;
5812       }
5813
5814       Temp write_data;
5815       if (elem_size_bytes < 4) {
5816          if (data.type() == RegType::sgpr) {
5817             data = as_vgpr(ctx, data);
5818             emit_split_vector(ctx, data, 4 * data.size() / elem_size_bytes);
5819          }
5820          RegClass rc = RegClass(RegType::vgpr, elem_size_bytes).as_subdword();
5821          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5822          for (int i = 0; i < count; i++)
5823             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, rc));
5824          write_data = bld.tmp(RegClass(RegType::vgpr, num_bytes).as_subdword());
5825          vec->definitions[0] = Definition(write_data);
5826          bld.insert(std::move(vec));
5827       } else if (count != instr->num_components) {
5828          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5829          for (int i = 0; i < count; i++) {
5830             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
5831             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
5832          }
5833          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
5834          vec->definitions[0] = Definition(write_data);
5835          ctx->block->instructions.emplace_back(std::move(vec));
5836       } else if (!smem && data.type() != RegType::vgpr) {
5837          assert(num_bytes % 4 == 0);
5838          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
5839       } else if (smem_nonfs && data.type() == RegType::vgpr) {
5840          assert(num_bytes % 4 == 0);
5841          write_data = bld.as_uniform(data);
5842       } else {
5843          write_data = data;
5844       }
5845
5846       aco_opcode vmem_op, smem_op = aco_opcode::last_opcode;
5847       switch (num_bytes) {
5848          case 1:
5849             vmem_op = aco_opcode::buffer_store_byte;
5850             break;
5851          case 2:
5852             vmem_op = aco_opcode::buffer_store_short;
5853             break;
5854          case 4:
5855             vmem_op = aco_opcode::buffer_store_dword;
5856             smem_op = aco_opcode::s_buffer_store_dword;
5857             break;
5858          case 8:
5859             vmem_op = aco_opcode::buffer_store_dwordx2;
5860             smem_op = aco_opcode::s_buffer_store_dwordx2;
5861             break;
5862          case 12:
5863             vmem_op = aco_opcode::buffer_store_dwordx3;
5864             assert(!smem && ctx->options->chip_class > GFX6);
5865             break;
5866          case 16:
5867             vmem_op = aco_opcode::buffer_store_dwordx4;
5868             smem_op = aco_opcode::s_buffer_store_dwordx4;
5869             break;
5870          default:
5871             unreachable("Store SSBO not implemented for this size.");
5872       }
5873       if (ctx->stage == fragment_fs)
5874          smem_op = aco_opcode::p_fs_buffer_store_smem;
5875
5876       if (smem) {
5877          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
5878          store->operands[0] = Operand(rsrc);
5879          if (start) {
5880             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
5881                                 offset, Operand(start * elem_size_bytes));
5882             store->operands[1] = Operand(off);
5883          } else {
5884             store->operands[1] = Operand(offset);
5885          }
5886          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
5887             store->operands[1].setFixed(m0);
5888          store->operands[2] = Operand(write_data);
5889          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5890          store->dlc = false;
5891          store->disable_wqm = true;
5892          store->barrier = barrier_buffer;
5893          ctx->block->instructions.emplace_back(std::move(store));
5894          ctx->program->wb_smem_l1_on_end = true;
5895          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
5896             ctx->block->kind |= block_kind_needs_lowering;
5897             ctx->program->needs_exact = true;
5898          }
5899       } else {
5900          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
5901          store->operands[0] = Operand(rsrc);
5902          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5903          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5904          store->operands[3] = Operand(write_data);
5905          store->offset = start * elem_size_bytes;
5906          store->offen = (offset.type() == RegType::vgpr);
5907          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5908          store->dlc = false;
5909          store->disable_wqm = true;
5910          store->barrier = barrier_buffer;
5911          ctx->program->needs_exact = true;
5912          ctx->block->instructions.emplace_back(std::move(store));
5913       }
5914    }
5915 }
5916
5917 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5918 {
5919    /* return the previous value if dest is ever used */
5920    bool return_previous = false;
5921    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5922       return_previous = true;
5923       break;
5924    }
5925    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5926       return_previous = true;
5927       break;
5928    }
5929
5930    Builder bld(ctx->program, ctx->block);
5931    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
5932
5933    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
5934       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5935                         get_ssa_temp(ctx, instr->src[3].ssa), data);
5936
5937    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
5938    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5939    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5940
5941    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5942
5943    aco_opcode op32, op64;
5944    switch (instr->intrinsic) {
5945       case nir_intrinsic_ssbo_atomic_add:
5946          op32 = aco_opcode::buffer_atomic_add;
5947          op64 = aco_opcode::buffer_atomic_add_x2;
5948          break;
5949       case nir_intrinsic_ssbo_atomic_imin:
5950          op32 = aco_opcode::buffer_atomic_smin;
5951          op64 = aco_opcode::buffer_atomic_smin_x2;
5952          break;
5953       case nir_intrinsic_ssbo_atomic_umin:
5954          op32 = aco_opcode::buffer_atomic_umin;
5955          op64 = aco_opcode::buffer_atomic_umin_x2;
5956          break;
5957       case nir_intrinsic_ssbo_atomic_imax:
5958          op32 = aco_opcode::buffer_atomic_smax;
5959          op64 = aco_opcode::buffer_atomic_smax_x2;
5960          break;
5961       case nir_intrinsic_ssbo_atomic_umax:
5962          op32 = aco_opcode::buffer_atomic_umax;
5963          op64 = aco_opcode::buffer_atomic_umax_x2;
5964          break;
5965       case nir_intrinsic_ssbo_atomic_and:
5966          op32 = aco_opcode::buffer_atomic_and;
5967          op64 = aco_opcode::buffer_atomic_and_x2;
5968          break;
5969       case nir_intrinsic_ssbo_atomic_or:
5970          op32 = aco_opcode::buffer_atomic_or;
5971          op64 = aco_opcode::buffer_atomic_or_x2;
5972          break;
5973       case nir_intrinsic_ssbo_atomic_xor:
5974          op32 = aco_opcode::buffer_atomic_xor;
5975          op64 = aco_opcode::buffer_atomic_xor_x2;
5976          break;
5977       case nir_intrinsic_ssbo_atomic_exchange:
5978          op32 = aco_opcode::buffer_atomic_swap;
5979          op64 = aco_opcode::buffer_atomic_swap_x2;
5980          break;
5981       case nir_intrinsic_ssbo_atomic_comp_swap:
5982          op32 = aco_opcode::buffer_atomic_cmpswap;
5983          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5984          break;
5985       default:
5986          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
5987    }
5988    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5989    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5990    mubuf->operands[0] = Operand(rsrc);
5991    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5992    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5993    mubuf->operands[3] = Operand(data);
5994    if (return_previous)
5995       mubuf->definitions[0] = Definition(dst);
5996    mubuf->offset = 0;
5997    mubuf->offen = (offset.type() == RegType::vgpr);
5998    mubuf->glc = return_previous;
5999    mubuf->dlc = false; /* Not needed for atomics */
6000    mubuf->disable_wqm = true;
6001    mubuf->barrier = barrier_buffer;
6002    ctx->program->needs_exact = true;
6003    ctx->block->instructions.emplace_back(std::move(mubuf));
6004 }
6005
6006 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
6007
6008    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6009    Builder bld(ctx->program, ctx->block);
6010    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
6011    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
6012 }
6013
6014 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
6015 {
6016    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6017                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6018
6019    if (addr.type() == RegType::vgpr)
6020       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
6021    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
6022 }
6023
6024 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
6025 {
6026    Builder bld(ctx->program, ctx->block);
6027    unsigned num_components = instr->num_components;
6028    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
6029
6030    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6031    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6032
6033    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
6034    bool dlc = glc && ctx->options->chip_class >= GFX10;
6035    aco_opcode op;
6036    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
6037       bool global = ctx->options->chip_class >= GFX9;
6038
6039       if (ctx->options->chip_class >= GFX7) {
6040          aco_opcode op;
6041          switch (num_bytes) {
6042          case 4:
6043             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
6044             break;
6045          case 8:
6046             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
6047             break;
6048          case 12:
6049             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
6050             break;
6051          case 16:
6052             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
6053             break;
6054          default:
6055             unreachable("load_global not implemented for this size.");
6056          }
6057
6058          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
6059          flat->operands[0] = Operand(addr);
6060          flat->operands[1] = Operand(s1);
6061          flat->glc = glc;
6062          flat->dlc = dlc;
6063          flat->barrier = barrier_buffer;
6064
6065          if (dst.type() == RegType::sgpr) {
6066             Temp vec = bld.tmp(RegType::vgpr, dst.size());
6067             flat->definitions[0] = Definition(vec);
6068             ctx->block->instructions.emplace_back(std::move(flat));
6069             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
6070          } else {
6071             flat->definitions[0] = Definition(dst);
6072             ctx->block->instructions.emplace_back(std::move(flat));
6073          }
6074          emit_split_vector(ctx, dst, num_components);
6075       } else {
6076          assert(ctx->options->chip_class == GFX6);
6077
6078          /* GFX6 doesn't support loading vec3, expand to vec4. */
6079          num_bytes = num_bytes == 12 ? 16 : num_bytes;
6080
6081          aco_opcode op;
6082          switch (num_bytes) {
6083          case 4:
6084             op = aco_opcode::buffer_load_dword;
6085             break;
6086          case 8:
6087             op = aco_opcode::buffer_load_dwordx2;
6088             break;
6089          case 16:
6090             op = aco_opcode::buffer_load_dwordx4;
6091             break;
6092          default:
6093             unreachable("load_global not implemented for this size.");
6094          }
6095
6096          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6097
6098          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6099          mubuf->operands[0] = Operand(rsrc);
6100          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6101          mubuf->operands[2] = Operand(0u);
6102          mubuf->glc = glc;
6103          mubuf->dlc = false;
6104          mubuf->offset = 0;
6105          mubuf->addr64 = addr.type() == RegType::vgpr;
6106          mubuf->disable_wqm = false;
6107          mubuf->barrier = barrier_buffer;
6108          aco_ptr<Instruction> instr = std::move(mubuf);
6109
6110          /* expand vector */
6111          if (dst.size() == 3) {
6112             Temp vec = bld.tmp(v4);
6113             instr->definitions[0] = Definition(vec);
6114             bld.insert(std::move(instr));
6115             emit_split_vector(ctx, vec, 4);
6116
6117             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
6118             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
6119             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
6120             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
6121          }
6122
6123          if (dst.type() == RegType::sgpr) {
6124             Temp vec = bld.tmp(RegType::vgpr, dst.size());
6125             instr->definitions[0] = Definition(vec);
6126             bld.insert(std::move(instr));
6127             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
6128             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
6129          } else {
6130             instr->definitions[0] = Definition(dst);
6131             bld.insert(std::move(instr));
6132             emit_split_vector(ctx, dst, num_components);
6133          }
6134       }
6135    } else {
6136       switch (num_bytes) {
6137          case 4:
6138             op = aco_opcode::s_load_dword;
6139             break;
6140          case 8:
6141             op = aco_opcode::s_load_dwordx2;
6142             break;
6143          case 12:
6144          case 16:
6145             op = aco_opcode::s_load_dwordx4;
6146             break;
6147          default:
6148             unreachable("load_global not implemented for this size.");
6149       }
6150       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
6151       load->operands[0] = Operand(addr);
6152       load->operands[1] = Operand(0u);
6153       load->definitions[0] = Definition(dst);
6154       load->glc = glc;
6155       load->dlc = dlc;
6156       load->barrier = barrier_buffer;
6157       assert(ctx->options->chip_class >= GFX8 || !glc);
6158
6159       if (dst.size() == 3) {
6160          /* trim vector */
6161          Temp vec = bld.tmp(s4);
6162          load->definitions[0] = Definition(vec);
6163          ctx->block->instructions.emplace_back(std::move(load));
6164          emit_split_vector(ctx, vec, 4);
6165
6166          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6167                     emit_extract_vector(ctx, vec, 0, s1),
6168                     emit_extract_vector(ctx, vec, 1, s1),
6169                     emit_extract_vector(ctx, vec, 2, s1));
6170       } else {
6171          ctx->block->instructions.emplace_back(std::move(load));
6172       }
6173    }
6174 }
6175
6176 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
6177 {
6178    Builder bld(ctx->program, ctx->block);
6179    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6180
6181    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6182    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
6183
6184    if (ctx->options->chip_class >= GFX7)
6185       addr = as_vgpr(ctx, addr);
6186
6187    unsigned writemask = nir_intrinsic_write_mask(instr);
6188    while (writemask) {
6189       int start, count;
6190       u_bit_scan_consecutive_range(&writemask, &start, &count);
6191       if (count == 3 && ctx->options->chip_class == GFX6) {
6192          /* GFX6 doesn't support storing vec3, split it. */
6193          writemask |= 1u << (start + 2);
6194          count = 2;
6195       }
6196       unsigned num_bytes = count * elem_size_bytes;
6197
6198       Temp write_data = data;
6199       if (count != instr->num_components) {
6200          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6201          for (int i = 0; i < count; i++)
6202             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
6203          write_data = bld.tmp(RegType::vgpr, count);
6204          vec->definitions[0] = Definition(write_data);
6205          ctx->block->instructions.emplace_back(std::move(vec));
6206       }
6207
6208       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
6209       unsigned offset = start * elem_size_bytes;
6210
6211       if (ctx->options->chip_class >= GFX7) {
6212          if (offset > 0 && ctx->options->chip_class < GFX9) {
6213             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
6214             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
6215             Temp carry = bld.tmp(bld.lm);
6216             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
6217
6218             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
6219                      Operand(offset), addr0);
6220             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
6221                      Operand(0u), addr1,
6222                      carry).def(1).setHint(vcc);
6223
6224             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
6225
6226             offset = 0;
6227          }
6228
6229          bool global = ctx->options->chip_class >= GFX9;
6230          aco_opcode op;
6231          switch (num_bytes) {
6232          case 4:
6233             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
6234             break;
6235          case 8:
6236             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
6237             break;
6238          case 12:
6239             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
6240             break;
6241          case 16:
6242             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
6243             break;
6244          default:
6245             unreachable("store_global not implemented for this size.");
6246          }
6247
6248          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
6249          flat->operands[0] = Operand(addr);
6250          flat->operands[1] = Operand(s1);
6251          flat->operands[2] = Operand(data);
6252          flat->glc = glc;
6253          flat->dlc = false;
6254          flat->offset = offset;
6255          flat->disable_wqm = true;
6256          flat->barrier = barrier_buffer;
6257          ctx->program->needs_exact = true;
6258          ctx->block->instructions.emplace_back(std::move(flat));
6259       } else {
6260          assert(ctx->options->chip_class == GFX6);
6261
6262          aco_opcode op;
6263          switch (num_bytes) {
6264          case 4:
6265             op = aco_opcode::buffer_store_dword;
6266             break;
6267          case 8:
6268             op = aco_opcode::buffer_store_dwordx2;
6269             break;
6270          case 16:
6271             op = aco_opcode::buffer_store_dwordx4;
6272             break;
6273          default:
6274             unreachable("store_global not implemented for this size.");
6275          }
6276
6277          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6278
6279          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
6280          mubuf->operands[0] = Operand(rsrc);
6281          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6282          mubuf->operands[2] = Operand(0u);
6283          mubuf->operands[3] = Operand(write_data);
6284          mubuf->glc = glc;
6285          mubuf->dlc = false;
6286          mubuf->offset = offset;
6287          mubuf->addr64 = addr.type() == RegType::vgpr;
6288          mubuf->disable_wqm = true;
6289          mubuf->barrier = barrier_buffer;
6290          ctx->program->needs_exact = true;
6291          ctx->block->instructions.emplace_back(std::move(mubuf));
6292       }
6293    }
6294 }
6295
6296 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6297 {
6298    /* return the previous value if dest is ever used */
6299    bool return_previous = false;
6300    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6301       return_previous = true;
6302       break;
6303    }
6304    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6305       return_previous = true;
6306       break;
6307    }
6308
6309    Builder bld(ctx->program, ctx->block);
6310    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6311    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6312
6313    if (ctx->options->chip_class >= GFX7)
6314       addr = as_vgpr(ctx, addr);
6315
6316    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
6317       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
6318                         get_ssa_temp(ctx, instr->src[2].ssa), data);
6319
6320    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6321
6322    aco_opcode op32, op64;
6323
6324    if (ctx->options->chip_class >= GFX7) {
6325       bool global = ctx->options->chip_class >= GFX9;
6326       switch (instr->intrinsic) {
6327          case nir_intrinsic_global_atomic_add:
6328             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
6329             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
6330             break;
6331          case nir_intrinsic_global_atomic_imin:
6332             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
6333             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
6334             break;
6335          case nir_intrinsic_global_atomic_umin:
6336             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
6337             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
6338             break;
6339          case nir_intrinsic_global_atomic_imax:
6340             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
6341             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
6342             break;
6343          case nir_intrinsic_global_atomic_umax:
6344             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
6345             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
6346             break;
6347          case nir_intrinsic_global_atomic_and:
6348             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
6349             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
6350             break;
6351          case nir_intrinsic_global_atomic_or:
6352             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
6353             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
6354             break;
6355          case nir_intrinsic_global_atomic_xor:
6356             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
6357             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
6358             break;
6359          case nir_intrinsic_global_atomic_exchange:
6360             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
6361             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
6362             break;
6363          case nir_intrinsic_global_atomic_comp_swap:
6364             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
6365             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
6366             break;
6367          default:
6368             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6369       }
6370
6371       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6372       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
6373       flat->operands[0] = Operand(addr);
6374       flat->operands[1] = Operand(s1);
6375       flat->operands[2] = Operand(data);
6376       if (return_previous)
6377          flat->definitions[0] = Definition(dst);
6378       flat->glc = return_previous;
6379       flat->dlc = false; /* Not needed for atomics */
6380       flat->offset = 0;
6381       flat->disable_wqm = true;
6382       flat->barrier = barrier_buffer;
6383       ctx->program->needs_exact = true;
6384       ctx->block->instructions.emplace_back(std::move(flat));
6385    } else {
6386       assert(ctx->options->chip_class == GFX6);
6387
6388       switch (instr->intrinsic) {
6389          case nir_intrinsic_global_atomic_add:
6390             op32 = aco_opcode::buffer_atomic_add;
6391             op64 = aco_opcode::buffer_atomic_add_x2;
6392             break;
6393          case nir_intrinsic_global_atomic_imin:
6394             op32 = aco_opcode::buffer_atomic_smin;
6395             op64 = aco_opcode::buffer_atomic_smin_x2;
6396             break;
6397          case nir_intrinsic_global_atomic_umin:
6398             op32 = aco_opcode::buffer_atomic_umin;
6399             op64 = aco_opcode::buffer_atomic_umin_x2;
6400             break;
6401          case nir_intrinsic_global_atomic_imax:
6402             op32 = aco_opcode::buffer_atomic_smax;
6403             op64 = aco_opcode::buffer_atomic_smax_x2;
6404             break;
6405          case nir_intrinsic_global_atomic_umax:
6406             op32 = aco_opcode::buffer_atomic_umax;
6407             op64 = aco_opcode::buffer_atomic_umax_x2;
6408             break;
6409          case nir_intrinsic_global_atomic_and:
6410             op32 = aco_opcode::buffer_atomic_and;
6411             op64 = aco_opcode::buffer_atomic_and_x2;
6412             break;
6413          case nir_intrinsic_global_atomic_or:
6414             op32 = aco_opcode::buffer_atomic_or;
6415             op64 = aco_opcode::buffer_atomic_or_x2;
6416             break;
6417          case nir_intrinsic_global_atomic_xor:
6418             op32 = aco_opcode::buffer_atomic_xor;
6419             op64 = aco_opcode::buffer_atomic_xor_x2;
6420             break;
6421          case nir_intrinsic_global_atomic_exchange:
6422             op32 = aco_opcode::buffer_atomic_swap;
6423             op64 = aco_opcode::buffer_atomic_swap_x2;
6424             break;
6425          case nir_intrinsic_global_atomic_comp_swap:
6426             op32 = aco_opcode::buffer_atomic_cmpswap;
6427             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
6428             break;
6429          default:
6430             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6431       }
6432
6433       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6434
6435       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6436
6437       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
6438       mubuf->operands[0] = Operand(rsrc);
6439       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6440       mubuf->operands[2] = Operand(0u);
6441       mubuf->operands[3] = Operand(data);
6442       if (return_previous)
6443          mubuf->definitions[0] = Definition(dst);
6444       mubuf->glc = return_previous;
6445       mubuf->dlc = false;
6446       mubuf->offset = 0;
6447       mubuf->addr64 = addr.type() == RegType::vgpr;
6448       mubuf->disable_wqm = true;
6449       mubuf->barrier = barrier_buffer;
6450       ctx->program->needs_exact = true;
6451       ctx->block->instructions.emplace_back(std::move(mubuf));
6452    }
6453 }
6454
6455 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
6456    Builder bld(ctx->program, ctx->block);
6457    switch(instr->intrinsic) {
6458       case nir_intrinsic_group_memory_barrier:
6459       case nir_intrinsic_memory_barrier:
6460          bld.barrier(aco_opcode::p_memory_barrier_common);
6461          break;
6462       case nir_intrinsic_memory_barrier_buffer:
6463          bld.barrier(aco_opcode::p_memory_barrier_buffer);
6464          break;
6465       case nir_intrinsic_memory_barrier_image:
6466          bld.barrier(aco_opcode::p_memory_barrier_image);
6467          break;
6468       case nir_intrinsic_memory_barrier_tcs_patch:
6469       case nir_intrinsic_memory_barrier_shared:
6470          bld.barrier(aco_opcode::p_memory_barrier_shared);
6471          break;
6472       default:
6473          unreachable("Unimplemented memory barrier intrinsic");
6474          break;
6475    }
6476 }
6477
6478 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6479 {
6480    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
6481    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6482    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
6483    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6484    Builder bld(ctx->program, ctx->block);
6485
6486    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
6487    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6488    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
6489 }
6490
6491 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6492 {
6493    unsigned writemask = nir_intrinsic_write_mask(instr);
6494    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
6495    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6496    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6497    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
6498
6499    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6500    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
6501 }
6502
6503 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6504 {
6505    unsigned offset = nir_intrinsic_base(instr);
6506    Operand m = load_lds_size_m0(ctx);
6507    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6508    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6509
6510    unsigned num_operands = 3;
6511    aco_opcode op32, op64, op32_rtn, op64_rtn;
6512    switch(instr->intrinsic) {
6513       case nir_intrinsic_shared_atomic_add:
6514          op32 = aco_opcode::ds_add_u32;
6515          op64 = aco_opcode::ds_add_u64;
6516          op32_rtn = aco_opcode::ds_add_rtn_u32;
6517          op64_rtn = aco_opcode::ds_add_rtn_u64;
6518          break;
6519       case nir_intrinsic_shared_atomic_imin:
6520          op32 = aco_opcode::ds_min_i32;
6521          op64 = aco_opcode::ds_min_i64;
6522          op32_rtn = aco_opcode::ds_min_rtn_i32;
6523          op64_rtn = aco_opcode::ds_min_rtn_i64;
6524          break;
6525       case nir_intrinsic_shared_atomic_umin:
6526          op32 = aco_opcode::ds_min_u32;
6527          op64 = aco_opcode::ds_min_u64;
6528          op32_rtn = aco_opcode::ds_min_rtn_u32;
6529          op64_rtn = aco_opcode::ds_min_rtn_u64;
6530          break;
6531       case nir_intrinsic_shared_atomic_imax:
6532          op32 = aco_opcode::ds_max_i32;
6533          op64 = aco_opcode::ds_max_i64;
6534          op32_rtn = aco_opcode::ds_max_rtn_i32;
6535          op64_rtn = aco_opcode::ds_max_rtn_i64;
6536          break;
6537       case nir_intrinsic_shared_atomic_umax:
6538          op32 = aco_opcode::ds_max_u32;
6539          op64 = aco_opcode::ds_max_u64;
6540          op32_rtn = aco_opcode::ds_max_rtn_u32;
6541          op64_rtn = aco_opcode::ds_max_rtn_u64;
6542          break;
6543       case nir_intrinsic_shared_atomic_and:
6544          op32 = aco_opcode::ds_and_b32;
6545          op64 = aco_opcode::ds_and_b64;
6546          op32_rtn = aco_opcode::ds_and_rtn_b32;
6547          op64_rtn = aco_opcode::ds_and_rtn_b64;
6548          break;
6549       case nir_intrinsic_shared_atomic_or:
6550          op32 = aco_opcode::ds_or_b32;
6551          op64 = aco_opcode::ds_or_b64;
6552          op32_rtn = aco_opcode::ds_or_rtn_b32;
6553          op64_rtn = aco_opcode::ds_or_rtn_b64;
6554          break;
6555       case nir_intrinsic_shared_atomic_xor:
6556          op32 = aco_opcode::ds_xor_b32;
6557          op64 = aco_opcode::ds_xor_b64;
6558          op32_rtn = aco_opcode::ds_xor_rtn_b32;
6559          op64_rtn = aco_opcode::ds_xor_rtn_b64;
6560          break;
6561       case nir_intrinsic_shared_atomic_exchange:
6562          op32 = aco_opcode::ds_write_b32;
6563          op64 = aco_opcode::ds_write_b64;
6564          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
6565          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
6566          break;
6567       case nir_intrinsic_shared_atomic_comp_swap:
6568          op32 = aco_opcode::ds_cmpst_b32;
6569          op64 = aco_opcode::ds_cmpst_b64;
6570          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
6571          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
6572          num_operands = 4;
6573          break;
6574       default:
6575          unreachable("Unhandled shared atomic intrinsic");
6576    }
6577
6578    /* return the previous value if dest is ever used */
6579    bool return_previous = false;
6580    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6581       return_previous = true;
6582       break;
6583    }
6584    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6585       return_previous = true;
6586       break;
6587    }
6588
6589    aco_opcode op;
6590    if (data.size() == 1) {
6591       assert(instr->dest.ssa.bit_size == 32);
6592       op = return_previous ? op32_rtn : op32;
6593    } else {
6594       assert(instr->dest.ssa.bit_size == 64);
6595       op = return_previous ? op64_rtn : op64;
6596    }
6597
6598    if (offset > 65535) {
6599       Builder bld(ctx->program, ctx->block);
6600       address = bld.vadd32(bld.def(v1), Operand(offset), address);
6601       offset = 0;
6602    }
6603
6604    aco_ptr<DS_instruction> ds;
6605    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
6606    ds->operands[0] = Operand(address);
6607    ds->operands[1] = Operand(data);
6608    if (num_operands == 4)
6609       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
6610    ds->operands[num_operands - 1] = m;
6611    ds->offset0 = offset;
6612    if (return_previous)
6613       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
6614    ctx->block->instructions.emplace_back(std::move(ds));
6615 }
6616
6617 Temp get_scratch_resource(isel_context *ctx)
6618 {
6619    Builder bld(ctx->program, ctx->block);
6620    Temp scratch_addr = ctx->program->private_segment_buffer;
6621    if (ctx->stage != compute_cs)
6622       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
6623
6624    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
6625                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
6626
6627    if (ctx->program->chip_class >= GFX10) {
6628       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
6629                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
6630                    S_008F0C_RESOURCE_LEVEL(1);
6631    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
6632       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6633                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6634    }
6635
6636    /* older generations need element size = 16 bytes. element size removed in GFX9 */
6637    if (ctx->program->chip_class <= GFX8)
6638       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
6639
6640    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
6641 }
6642
6643 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6644    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
6645    Builder bld(ctx->program, ctx->block);
6646    Temp rsrc = get_scratch_resource(ctx);
6647    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6648    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6649
6650    aco_opcode op;
6651    switch (dst.size()) {
6652       case 1:
6653          op = aco_opcode::buffer_load_dword;
6654          break;
6655       case 2:
6656          op = aco_opcode::buffer_load_dwordx2;
6657          break;
6658       case 3:
6659          op = aco_opcode::buffer_load_dwordx3;
6660          break;
6661       case 4:
6662          op = aco_opcode::buffer_load_dwordx4;
6663          break;
6664       case 6:
6665       case 8: {
6666          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
6667          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
6668                                 bld.def(v4), rsrc, offset,
6669                                 ctx->program->scratch_offset, 0, true);
6670          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
6671                                                   aco_opcode::buffer_load_dwordx4,
6672                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
6673                                 rsrc, offset, ctx->program->scratch_offset, 16, true);
6674          emit_split_vector(ctx, lower, 2);
6675          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
6676          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
6677          if (dst.size() == 8) {
6678             emit_split_vector(ctx, upper, 2);
6679             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
6680             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
6681          } else {
6682             elems[2] = upper;
6683          }
6684
6685          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6686                                                                          Format::PSEUDO, dst.size() / 2, 1)};
6687          for (unsigned i = 0; i < dst.size() / 2; i++)
6688             vec->operands[i] = Operand(elems[i]);
6689          vec->definitions[0] = Definition(dst);
6690          bld.insert(std::move(vec));
6691          ctx->allocated_vec.emplace(dst.id(), elems);
6692          return;
6693       }
6694       default:
6695          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
6696    }
6697
6698    bld.mubuf(op, Definition(dst), rsrc, offset, ctx->program->scratch_offset, 0, true);
6699    emit_split_vector(ctx, dst, instr->num_components);
6700 }
6701
6702 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6703    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
6704    Builder bld(ctx->program, ctx->block);
6705    Temp rsrc = get_scratch_resource(ctx);
6706    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6707    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6708
6709    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6710    unsigned writemask = nir_intrinsic_write_mask(instr);
6711
6712    while (writemask) {
6713       int start, count;
6714       u_bit_scan_consecutive_range(&writemask, &start, &count);
6715       int num_bytes = count * elem_size_bytes;
6716
6717       if (num_bytes > 16) {
6718          assert(elem_size_bytes == 8);
6719          writemask |= (((count - 2) << 1) - 1) << (start + 2);
6720          count = 2;
6721          num_bytes = 16;
6722       }
6723
6724       // TODO: check alignment of sub-dword stores
6725       // TODO: split 3 bytes. there is no store instruction for that
6726
6727       Temp write_data;
6728       if (count != instr->num_components) {
6729          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6730          for (int i = 0; i < count; i++) {
6731             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
6732             vec->operands[i] = Operand(elem);
6733          }
6734          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
6735          vec->definitions[0] = Definition(write_data);
6736          ctx->block->instructions.emplace_back(std::move(vec));
6737       } else {
6738          write_data = data;
6739       }
6740
6741       aco_opcode op;
6742       switch (num_bytes) {
6743          case 4:
6744             op = aco_opcode::buffer_store_dword;
6745             break;
6746          case 8:
6747             op = aco_opcode::buffer_store_dwordx2;
6748             break;
6749          case 12:
6750             op = aco_opcode::buffer_store_dwordx3;
6751             break;
6752          case 16:
6753             op = aco_opcode::buffer_store_dwordx4;
6754             break;
6755          default:
6756             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
6757       }
6758
6759       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
6760    }
6761 }
6762
6763 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
6764    uint8_t log2_ps_iter_samples;
6765    if (ctx->program->info->ps.force_persample) {
6766       log2_ps_iter_samples =
6767          util_logbase2(ctx->options->key.fs.num_samples);
6768    } else {
6769       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
6770    }
6771
6772    /* The bit pattern matches that used by fixed function fragment
6773     * processing. */
6774    static const unsigned ps_iter_masks[] = {
6775       0xffff, /* not used */
6776       0x5555,
6777       0x1111,
6778       0x0101,
6779       0x0001,
6780    };
6781    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
6782
6783    Builder bld(ctx->program, ctx->block);
6784
6785    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
6786                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6787    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
6788    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
6789    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6790    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
6791 }
6792
6793 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
6794    Builder bld(ctx->program, ctx->block);
6795
6796    unsigned stream = nir_intrinsic_stream_id(instr);
6797    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6798    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
6799    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
6800
6801    /* get GSVS ring */
6802    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
6803
6804    unsigned num_components =
6805       ctx->program->info->gs.num_stream_output_components[stream];
6806    assert(num_components);
6807
6808    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
6809    unsigned stream_offset = 0;
6810    for (unsigned i = 0; i < stream; i++) {
6811       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
6812       stream_offset += prev_stride * ctx->program->wave_size;
6813    }
6814
6815    /* Limit on the stride field for <= GFX7. */
6816    assert(stride < (1 << 14));
6817
6818    Temp gsvs_dwords[4];
6819    for (unsigned i = 0; i < 4; i++)
6820       gsvs_dwords[i] = bld.tmp(s1);
6821    bld.pseudo(aco_opcode::p_split_vector,
6822               Definition(gsvs_dwords[0]),
6823               Definition(gsvs_dwords[1]),
6824               Definition(gsvs_dwords[2]),
6825               Definition(gsvs_dwords[3]),
6826               gsvs_ring);
6827
6828    if (stream_offset) {
6829       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
6830
6831       Temp carry = bld.tmp(s1);
6832       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
6833       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
6834    }
6835
6836    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
6837    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
6838
6839    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6840                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
6841
6842    unsigned offset = 0;
6843    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
6844       if (ctx->program->info->gs.output_streams[i] != stream)
6845          continue;
6846
6847       for (unsigned j = 0; j < 4; j++) {
6848          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
6849             continue;
6850
6851          if (ctx->outputs.mask[i] & (1 << j)) {
6852             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
6853             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
6854             if (const_offset >= 4096u) {
6855                if (vaddr_offset.isUndefined())
6856                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
6857                else
6858                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
6859                const_offset %= 4096u;
6860             }
6861
6862             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
6863             mtbuf->operands[0] = Operand(gsvs_ring);
6864             mtbuf->operands[1] = vaddr_offset;
6865             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
6866             mtbuf->operands[3] = Operand(ctx->outputs.temps[i * 4u + j]);
6867             mtbuf->offen = !vaddr_offset.isUndefined();
6868             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
6869             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
6870             mtbuf->offset = const_offset;
6871             mtbuf->glc = true;
6872             mtbuf->slc = true;
6873             mtbuf->barrier = barrier_gs_data;
6874             mtbuf->can_reorder = true;
6875             bld.insert(std::move(mtbuf));
6876          }
6877
6878          offset += ctx->shader->info.gs.vertices_out;
6879       }
6880
6881       /* outputs for the next vertex are undefined and keeping them around can
6882        * create invalid IR with control flow */
6883       ctx->outputs.mask[i] = 0;
6884    }
6885
6886    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
6887 }
6888
6889 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
6890 {
6891    Builder bld(ctx->program, ctx->block);
6892
6893    if (cluster_size == 1) {
6894       return src;
6895    } if (op == nir_op_iand && cluster_size == 4) {
6896       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
6897       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6898       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
6899                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
6900    } else if (op == nir_op_ior && cluster_size == 4) {
6901       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
6902       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
6903                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
6904    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
6905       //subgroupAnd(val) -> (exec & ~val) == 0
6906       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6907       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6908       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
6909    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
6910       //subgroupOr(val) -> (val & exec) != 0
6911       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
6912       return bool_to_vector_condition(ctx, tmp);
6913    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
6914       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
6915       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6916       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
6917       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
6918       return bool_to_vector_condition(ctx, tmp);
6919    } else {
6920       //subgroupClustered{And,Or,Xor}(val, n) ->
6921       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
6922       //cluster_offset = ~(n - 1) & lane_id
6923       //cluster_mask = ((1 << n) - 1)
6924       //subgroupClusteredAnd():
6925       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
6926       //subgroupClusteredOr():
6927       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
6928       //subgroupClusteredXor():
6929       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
6930       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
6931       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
6932
6933       Temp tmp;
6934       if (op == nir_op_iand)
6935          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6936       else
6937          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6938
6939       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
6940
6941       if (ctx->program->chip_class <= GFX7)
6942          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
6943       else if (ctx->program->wave_size == 64)
6944          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
6945       else
6946          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
6947       tmp = emit_extract_vector(ctx, tmp, 0, v1);
6948       if (cluster_mask != 0xffffffff)
6949          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
6950
6951       Definition cmp_def = Definition();
6952       if (op == nir_op_iand) {
6953          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
6954       } else if (op == nir_op_ior) {
6955          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6956       } else if (op == nir_op_ixor) {
6957          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
6958                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
6959          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6960       }
6961       cmp_def.setHint(vcc);
6962       return cmp_def.getTemp();
6963    }
6964 }
6965
6966 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
6967 {
6968    Builder bld(ctx->program, ctx->block);
6969
6970    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
6971    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
6972    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
6973    Temp tmp;
6974    if (op == nir_op_iand)
6975       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6976    else
6977       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
6978
6979    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
6980    Temp lo = lohi.def(0).getTemp();
6981    Temp hi = lohi.def(1).getTemp();
6982    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
6983
6984    Definition cmp_def = Definition();
6985    if (op == nir_op_iand)
6986       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6987    else if (op == nir_op_ior)
6988       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6989    else if (op == nir_op_ixor)
6990       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
6991                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
6992    cmp_def.setHint(vcc);
6993    return cmp_def.getTemp();
6994 }
6995
6996 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
6997 {
6998    Builder bld(ctx->program, ctx->block);
6999
7000    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
7001    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
7002    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
7003    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
7004    if (op == nir_op_iand)
7005       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7006    else if (op == nir_op_ior)
7007       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7008    else if (op == nir_op_ixor)
7009       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7010
7011    assert(false);
7012    return Temp();
7013 }
7014
7015 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
7016 {
7017    Builder bld(ctx->program, ctx->block);
7018    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
7019    if (src.regClass().type() == RegType::vgpr) {
7020       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
7021    } else if (src.regClass() == s1) {
7022       bld.sop1(aco_opcode::s_mov_b32, dst, src);
7023    } else if (src.regClass() == s2) {
7024       bld.sop1(aco_opcode::s_mov_b64, dst, src);
7025    } else {
7026       fprintf(stderr, "Unimplemented NIR instr bit size: ");
7027       nir_print_instr(&instr->instr, stderr);
7028       fprintf(stderr, "\n");
7029    }
7030 }
7031
7032 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
7033 {
7034    Builder bld(ctx->program, ctx->block);
7035    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
7036    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
7037    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
7038
7039    Temp ddx_1, ddx_2, ddy_1, ddy_2;
7040    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
7041    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
7042    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
7043
7044    /* Build DD X/Y */
7045    if (ctx->program->chip_class >= GFX8) {
7046       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
7047       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
7048       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
7049       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
7050       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
7051       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
7052    } else {
7053       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
7054       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
7055       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
7056       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
7057       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
7058       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
7059       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
7060       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
7061       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
7062       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
7063    }
7064
7065    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
7066    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
7067    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
7068    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
7069    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
7070    Temp wqm1 = bld.tmp(v1);
7071    emit_wqm(ctx, tmp1, wqm1, true);
7072    Temp wqm2 = bld.tmp(v1);
7073    emit_wqm(ctx, tmp2, wqm2, true);
7074    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
7075    return;
7076 }
7077
7078 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
7079 {
7080    Builder bld(ctx->program, ctx->block);
7081    switch(instr->intrinsic) {
7082    case nir_intrinsic_load_barycentric_sample:
7083    case nir_intrinsic_load_barycentric_pixel:
7084    case nir_intrinsic_load_barycentric_centroid: {
7085       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
7086       Temp bary = Temp(0, s2);
7087       switch (mode) {
7088       case INTERP_MODE_SMOOTH:
7089       case INTERP_MODE_NONE:
7090          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7091             bary = get_arg(ctx, ctx->args->ac.persp_center);
7092          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7093             bary = ctx->persp_centroid;
7094          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7095             bary = get_arg(ctx, ctx->args->ac.persp_sample);
7096          break;
7097       case INTERP_MODE_NOPERSPECTIVE:
7098          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7099             bary = get_arg(ctx, ctx->args->ac.linear_center);
7100          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7101             bary = ctx->linear_centroid;
7102          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7103             bary = get_arg(ctx, ctx->args->ac.linear_sample);
7104          break;
7105       default:
7106          break;
7107       }
7108       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7109       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
7110       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
7111       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7112                  Operand(p1), Operand(p2));
7113       emit_split_vector(ctx, dst, 2);
7114       break;
7115    }
7116    case nir_intrinsic_load_barycentric_model: {
7117       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
7118
7119       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7120       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
7121       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
7122       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
7123       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7124                  Operand(p1), Operand(p2), Operand(p3));
7125       emit_split_vector(ctx, dst, 3);
7126       break;
7127    }
7128    case nir_intrinsic_load_barycentric_at_sample: {
7129       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
7130       switch (ctx->options->key.fs.num_samples) {
7131          case 2: sample_pos_offset += 1 << 3; break;
7132          case 4: sample_pos_offset += 3 << 3; break;
7133          case 8: sample_pos_offset += 7 << 3; break;
7134          default: break;
7135       }
7136       Temp sample_pos;
7137       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
7138       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
7139       Temp private_segment_buffer = ctx->program->private_segment_buffer;
7140       if (addr.type() == RegType::sgpr) {
7141          Operand offset;
7142          if (const_addr) {
7143             sample_pos_offset += const_addr->u32 << 3;
7144             offset = Operand(sample_pos_offset);
7145          } else if (ctx->options->chip_class >= GFX9) {
7146             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7147          } else {
7148             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
7149             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7150          }
7151
7152          Operand off = bld.copy(bld.def(s1), Operand(offset));
7153          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
7154
7155       } else if (ctx->options->chip_class >= GFX9) {
7156          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7157          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
7158       } else if (ctx->options->chip_class >= GFX7) {
7159          /* addr += private_segment_buffer + sample_pos_offset */
7160          Temp tmp0 = bld.tmp(s1);
7161          Temp tmp1 = bld.tmp(s1);
7162          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
7163          Definition scc_tmp = bld.def(s1, scc);
7164          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
7165          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
7166          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7167          Temp pck0 = bld.tmp(v1);
7168          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
7169          tmp1 = as_vgpr(ctx, tmp1);
7170          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
7171          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
7172
7173          /* sample_pos = flat_load_dwordx2 addr */
7174          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
7175       } else {
7176          assert(ctx->options->chip_class == GFX6);
7177
7178          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
7179                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
7180          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
7181
7182          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7183          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
7184
7185          sample_pos = bld.tmp(v2);
7186
7187          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
7188          load->definitions[0] = Definition(sample_pos);
7189          load->operands[0] = Operand(rsrc);
7190          load->operands[1] = Operand(addr);
7191          load->operands[2] = Operand(0u);
7192          load->offset = sample_pos_offset;
7193          load->offen = 0;
7194          load->addr64 = true;
7195          load->glc = false;
7196          load->dlc = false;
7197          load->disable_wqm = false;
7198          load->barrier = barrier_none;
7199          load->can_reorder = true;
7200          ctx->block->instructions.emplace_back(std::move(load));
7201       }
7202
7203       /* sample_pos -= 0.5 */
7204       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
7205       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
7206       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
7207       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
7208       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
7209
7210       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7211       break;
7212    }
7213    case nir_intrinsic_load_barycentric_at_offset: {
7214       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
7215       RegClass rc = RegClass(offset.type(), 1);
7216       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
7217       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
7218       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7219       break;
7220    }
7221    case nir_intrinsic_load_front_face: {
7222       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7223                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
7224       break;
7225    }
7226    case nir_intrinsic_load_view_index: {
7227       if (ctx->stage & (sw_vs | sw_gs | sw_tcs | sw_tes)) {
7228          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7229          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
7230          break;
7231       }
7232
7233       /* fallthrough */
7234    }
7235    case nir_intrinsic_load_layer_id: {
7236       unsigned idx = nir_intrinsic_base(instr);
7237       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7238                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
7239       break;
7240    }
7241    case nir_intrinsic_load_frag_coord: {
7242       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
7243       break;
7244    }
7245    case nir_intrinsic_load_sample_pos: {
7246       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
7247       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
7248       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7249                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
7250                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
7251       break;
7252    }
7253    case nir_intrinsic_load_tess_coord:
7254       visit_load_tess_coord(ctx, instr);
7255       break;
7256    case nir_intrinsic_load_interpolated_input:
7257       visit_load_interpolated_input(ctx, instr);
7258       break;
7259    case nir_intrinsic_store_output:
7260       visit_store_output(ctx, instr);
7261       break;
7262    case nir_intrinsic_load_input:
7263    case nir_intrinsic_load_input_vertex:
7264       visit_load_input(ctx, instr);
7265       break;
7266    case nir_intrinsic_load_output:
7267       visit_load_output(ctx, instr);
7268       break;
7269    case nir_intrinsic_load_per_vertex_input:
7270       visit_load_per_vertex_input(ctx, instr);
7271       break;
7272    case nir_intrinsic_load_per_vertex_output:
7273       visit_load_per_vertex_output(ctx, instr);
7274       break;
7275    case nir_intrinsic_store_per_vertex_output:
7276       visit_store_per_vertex_output(ctx, instr);
7277       break;
7278    case nir_intrinsic_load_ubo:
7279       visit_load_ubo(ctx, instr);
7280       break;
7281    case nir_intrinsic_load_push_constant:
7282       visit_load_push_constant(ctx, instr);
7283       break;
7284    case nir_intrinsic_load_constant:
7285       visit_load_constant(ctx, instr);
7286       break;
7287    case nir_intrinsic_vulkan_resource_index:
7288       visit_load_resource(ctx, instr);
7289       break;
7290    case nir_intrinsic_discard:
7291       visit_discard(ctx, instr);
7292       break;
7293    case nir_intrinsic_discard_if:
7294       visit_discard_if(ctx, instr);
7295       break;
7296    case nir_intrinsic_load_shared:
7297       visit_load_shared(ctx, instr);
7298       break;
7299    case nir_intrinsic_store_shared:
7300       visit_store_shared(ctx, instr);
7301       break;
7302    case nir_intrinsic_shared_atomic_add:
7303    case nir_intrinsic_shared_atomic_imin:
7304    case nir_intrinsic_shared_atomic_umin:
7305    case nir_intrinsic_shared_atomic_imax:
7306    case nir_intrinsic_shared_atomic_umax:
7307    case nir_intrinsic_shared_atomic_and:
7308    case nir_intrinsic_shared_atomic_or:
7309    case nir_intrinsic_shared_atomic_xor:
7310    case nir_intrinsic_shared_atomic_exchange:
7311    case nir_intrinsic_shared_atomic_comp_swap:
7312       visit_shared_atomic(ctx, instr);
7313       break;
7314    case nir_intrinsic_image_deref_load:
7315       visit_image_load(ctx, instr);
7316       break;
7317    case nir_intrinsic_image_deref_store:
7318       visit_image_store(ctx, instr);
7319       break;
7320    case nir_intrinsic_image_deref_atomic_add:
7321    case nir_intrinsic_image_deref_atomic_umin:
7322    case nir_intrinsic_image_deref_atomic_imin:
7323    case nir_intrinsic_image_deref_atomic_umax:
7324    case nir_intrinsic_image_deref_atomic_imax:
7325    case nir_intrinsic_image_deref_atomic_and:
7326    case nir_intrinsic_image_deref_atomic_or:
7327    case nir_intrinsic_image_deref_atomic_xor:
7328    case nir_intrinsic_image_deref_atomic_exchange:
7329    case nir_intrinsic_image_deref_atomic_comp_swap:
7330       visit_image_atomic(ctx, instr);
7331       break;
7332    case nir_intrinsic_image_deref_size:
7333       visit_image_size(ctx, instr);
7334       break;
7335    case nir_intrinsic_load_ssbo:
7336       visit_load_ssbo(ctx, instr);
7337       break;
7338    case nir_intrinsic_store_ssbo:
7339       visit_store_ssbo(ctx, instr);
7340       break;
7341    case nir_intrinsic_load_global:
7342       visit_load_global(ctx, instr);
7343       break;
7344    case nir_intrinsic_store_global:
7345       visit_store_global(ctx, instr);
7346       break;
7347    case nir_intrinsic_global_atomic_add:
7348    case nir_intrinsic_global_atomic_imin:
7349    case nir_intrinsic_global_atomic_umin:
7350    case nir_intrinsic_global_atomic_imax:
7351    case nir_intrinsic_global_atomic_umax:
7352    case nir_intrinsic_global_atomic_and:
7353    case nir_intrinsic_global_atomic_or:
7354    case nir_intrinsic_global_atomic_xor:
7355    case nir_intrinsic_global_atomic_exchange:
7356    case nir_intrinsic_global_atomic_comp_swap:
7357       visit_global_atomic(ctx, instr);
7358       break;
7359    case nir_intrinsic_ssbo_atomic_add:
7360    case nir_intrinsic_ssbo_atomic_imin:
7361    case nir_intrinsic_ssbo_atomic_umin:
7362    case nir_intrinsic_ssbo_atomic_imax:
7363    case nir_intrinsic_ssbo_atomic_umax:
7364    case nir_intrinsic_ssbo_atomic_and:
7365    case nir_intrinsic_ssbo_atomic_or:
7366    case nir_intrinsic_ssbo_atomic_xor:
7367    case nir_intrinsic_ssbo_atomic_exchange:
7368    case nir_intrinsic_ssbo_atomic_comp_swap:
7369       visit_atomic_ssbo(ctx, instr);
7370       break;
7371    case nir_intrinsic_load_scratch:
7372       visit_load_scratch(ctx, instr);
7373       break;
7374    case nir_intrinsic_store_scratch:
7375       visit_store_scratch(ctx, instr);
7376       break;
7377    case nir_intrinsic_get_buffer_size:
7378       visit_get_buffer_size(ctx, instr);
7379       break;
7380    case nir_intrinsic_control_barrier: {
7381       if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7382          /* GFX6 only (thanks to a hw bug workaround):
7383           * The real barrier instruction isn’t needed, because an entire patch
7384           * always fits into a single wave.
7385           */
7386          break;
7387       }
7388
7389       if (ctx->program->workgroup_size > ctx->program->wave_size)
7390          bld.sopp(aco_opcode::s_barrier);
7391
7392       break;
7393    }
7394    case nir_intrinsic_memory_barrier_tcs_patch:
7395    case nir_intrinsic_group_memory_barrier:
7396    case nir_intrinsic_memory_barrier:
7397    case nir_intrinsic_memory_barrier_buffer:
7398    case nir_intrinsic_memory_barrier_image:
7399    case nir_intrinsic_memory_barrier_shared:
7400       emit_memory_barrier(ctx, instr);
7401       break;
7402    case nir_intrinsic_load_num_work_groups: {
7403       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7404       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
7405       emit_split_vector(ctx, dst, 3);
7406       break;
7407    }
7408    case nir_intrinsic_load_local_invocation_id: {
7409       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7410       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
7411       emit_split_vector(ctx, dst, 3);
7412       break;
7413    }
7414    case nir_intrinsic_load_work_group_id: {
7415       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7416       struct ac_arg *args = ctx->args->ac.workgroup_ids;
7417       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7418                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
7419                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
7420                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
7421       emit_split_vector(ctx, dst, 3);
7422       break;
7423    }
7424    case nir_intrinsic_load_local_invocation_index: {
7425       Temp id = emit_mbcnt(ctx, bld.def(v1));
7426
7427       /* The tg_size bits [6:11] contain the subgroup id,
7428        * we need this multiplied by the wave size, and then OR the thread id to it.
7429        */
7430       if (ctx->program->wave_size == 64) {
7431          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
7432          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
7433                                 get_arg(ctx, ctx->args->ac.tg_size));
7434          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
7435       } else {
7436          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
7437          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7438                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7439          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
7440       }
7441       break;
7442    }
7443    case nir_intrinsic_load_subgroup_id: {
7444       if (ctx->stage == compute_cs) {
7445          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
7446                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7447       } else {
7448          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
7449       }
7450       break;
7451    }
7452    case nir_intrinsic_load_subgroup_invocation: {
7453       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
7454       break;
7455    }
7456    case nir_intrinsic_load_num_subgroups: {
7457       if (ctx->stage == compute_cs)
7458          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
7459                   get_arg(ctx, ctx->args->ac.tg_size));
7460       else
7461          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
7462       break;
7463    }
7464    case nir_intrinsic_ballot: {
7465       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7466       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7467       Definition tmp = bld.def(dst.regClass());
7468       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
7469       if (instr->src[0].ssa->bit_size == 1) {
7470          assert(src.regClass() == bld.lm);
7471          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
7472       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
7473          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
7474       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
7475          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
7476       } else {
7477          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7478          nir_print_instr(&instr->instr, stderr);
7479          fprintf(stderr, "\n");
7480       }
7481       if (dst.size() != bld.lm.size()) {
7482          /* Wave32 with ballot size set to 64 */
7483          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
7484       }
7485       emit_wqm(ctx, tmp.getTemp(), dst);
7486       break;
7487    }
7488    case nir_intrinsic_shuffle:
7489    case nir_intrinsic_read_invocation: {
7490       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7491       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
7492          emit_uniform_subgroup(ctx, instr, src);
7493       } else {
7494          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
7495          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
7496             tid = bld.as_uniform(tid);
7497          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7498          if (src.regClass() == v1) {
7499             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
7500          } else if (src.regClass() == v2) {
7501             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7502             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7503             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
7504             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
7505             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7506             emit_split_vector(ctx, dst, 2);
7507          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
7508             assert(src.regClass() == bld.lm);
7509             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
7510             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7511          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
7512             assert(src.regClass() == bld.lm);
7513             Temp tmp;
7514             if (ctx->program->chip_class <= GFX7)
7515                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
7516             else if (ctx->program->wave_size == 64)
7517                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
7518             else
7519                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
7520             tmp = emit_extract_vector(ctx, tmp, 0, v1);
7521             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
7522             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
7523          } else {
7524             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7525             nir_print_instr(&instr->instr, stderr);
7526             fprintf(stderr, "\n");
7527          }
7528       }
7529       break;
7530    }
7531    case nir_intrinsic_load_sample_id: {
7532       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7533                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
7534       break;
7535    }
7536    case nir_intrinsic_load_sample_mask_in: {
7537       visit_load_sample_mask_in(ctx, instr);
7538       break;
7539    }
7540    case nir_intrinsic_read_first_invocation: {
7541       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7542       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7543       if (src.regClass() == v1) {
7544          emit_wqm(ctx,
7545                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
7546                   dst);
7547       } else if (src.regClass() == v2) {
7548          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7549          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7550          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
7551          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
7552          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7553          emit_split_vector(ctx, dst, 2);
7554       } else if (instr->dest.ssa.bit_size == 1) {
7555          assert(src.regClass() == bld.lm);
7556          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
7557                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
7558          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7559       } else if (src.regClass() == s1) {
7560          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
7561       } else if (src.regClass() == s2) {
7562          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
7563       } else {
7564          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7565          nir_print_instr(&instr->instr, stderr);
7566          fprintf(stderr, "\n");
7567       }
7568       break;
7569    }
7570    case nir_intrinsic_vote_all: {
7571       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7572       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7573       assert(src.regClass() == bld.lm);
7574       assert(dst.regClass() == bld.lm);
7575
7576       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
7577       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
7578       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
7579       break;
7580    }
7581    case nir_intrinsic_vote_any: {
7582       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7583       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7584       assert(src.regClass() == bld.lm);
7585       assert(dst.regClass() == bld.lm);
7586
7587       Temp tmp = bool_to_scalar_condition(ctx, src);
7588       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7589       break;
7590    }
7591    case nir_intrinsic_reduce:
7592    case nir_intrinsic_inclusive_scan:
7593    case nir_intrinsic_exclusive_scan: {
7594       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7595       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7596       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
7597       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
7598          nir_intrinsic_cluster_size(instr) : 0;
7599       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
7600
7601       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
7602          emit_uniform_subgroup(ctx, instr, src);
7603       } else if (instr->dest.ssa.bit_size == 1) {
7604          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
7605             op = nir_op_iand;
7606          else if (op == nir_op_iadd)
7607             op = nir_op_ixor;
7608          else if (op == nir_op_umax || op == nir_op_imax)
7609             op = nir_op_ior;
7610          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
7611
7612          switch (instr->intrinsic) {
7613          case nir_intrinsic_reduce:
7614             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
7615             break;
7616          case nir_intrinsic_exclusive_scan:
7617             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
7618             break;
7619          case nir_intrinsic_inclusive_scan:
7620             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
7621             break;
7622          default:
7623             assert(false);
7624          }
7625       } else if (cluster_size == 1) {
7626          bld.copy(Definition(dst), src);
7627       } else {
7628          src = as_vgpr(ctx, src);
7629
7630          ReduceOp reduce_op;
7631          switch (op) {
7632          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
7633             CASE(iadd)
7634             CASE(imul)
7635             CASE(fadd)
7636             CASE(fmul)
7637             CASE(imin)
7638             CASE(umin)
7639             CASE(fmin)
7640             CASE(imax)
7641             CASE(umax)
7642             CASE(fmax)
7643             CASE(iand)
7644             CASE(ior)
7645             CASE(ixor)
7646             default:
7647                unreachable("unknown reduction op");
7648          #undef CASE
7649          }
7650
7651          aco_opcode aco_op;
7652          switch (instr->intrinsic) {
7653             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
7654             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
7655             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
7656             default:
7657                unreachable("unknown reduce intrinsic");
7658          }
7659
7660          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
7661          reduce->operands[0] = Operand(src);
7662          // filled in by aco_reduce_assign.cpp, used internally as part of the
7663          // reduce sequence
7664          assert(dst.size() == 1 || dst.size() == 2);
7665          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
7666          reduce->operands[2] = Operand(v1.as_linear());
7667
7668          Temp tmp_dst = bld.tmp(dst.regClass());
7669          reduce->definitions[0] = Definition(tmp_dst);
7670          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
7671          reduce->definitions[2] = Definition();
7672          reduce->definitions[3] = Definition(scc, s1);
7673          reduce->definitions[4] = Definition();
7674          reduce->reduce_op = reduce_op;
7675          reduce->cluster_size = cluster_size;
7676          ctx->block->instructions.emplace_back(std::move(reduce));
7677
7678          emit_wqm(ctx, tmp_dst, dst);
7679       }
7680       break;
7681    }
7682    case nir_intrinsic_quad_broadcast: {
7683       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7684       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7685          emit_uniform_subgroup(ctx, instr, src);
7686       } else {
7687          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7688          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
7689          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
7690
7691          if (instr->dest.ssa.bit_size == 1) {
7692             assert(src.regClass() == bld.lm);
7693             assert(dst.regClass() == bld.lm);
7694             uint32_t half_mask = 0x11111111u << lane;
7695             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
7696             Temp tmp = bld.tmp(bld.lm);
7697             bld.sop1(Builder::s_wqm, Definition(tmp),
7698                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
7699                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
7700             emit_wqm(ctx, tmp, dst);
7701          } else if (instr->dest.ssa.bit_size == 32) {
7702             if (ctx->program->chip_class >= GFX8)
7703                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
7704             else
7705                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
7706          } else if (instr->dest.ssa.bit_size == 64) {
7707             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7708             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7709             if (ctx->program->chip_class >= GFX8) {
7710                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7711                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7712             } else {
7713                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
7714                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
7715             }
7716             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7717             emit_split_vector(ctx, dst, 2);
7718          } else {
7719             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7720             nir_print_instr(&instr->instr, stderr);
7721             fprintf(stderr, "\n");
7722          }
7723       }
7724       break;
7725    }
7726    case nir_intrinsic_quad_swap_horizontal:
7727    case nir_intrinsic_quad_swap_vertical:
7728    case nir_intrinsic_quad_swap_diagonal:
7729    case nir_intrinsic_quad_swizzle_amd: {
7730       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7731       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7732          emit_uniform_subgroup(ctx, instr, src);
7733          break;
7734       }
7735       uint16_t dpp_ctrl = 0;
7736       switch (instr->intrinsic) {
7737       case nir_intrinsic_quad_swap_horizontal:
7738          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
7739          break;
7740       case nir_intrinsic_quad_swap_vertical:
7741          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
7742          break;
7743       case nir_intrinsic_quad_swap_diagonal:
7744          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
7745          break;
7746       case nir_intrinsic_quad_swizzle_amd:
7747          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
7748          break;
7749       default:
7750          break;
7751       }
7752       if (ctx->program->chip_class < GFX8)
7753          dpp_ctrl |= (1 << 15);
7754
7755       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7756       if (instr->dest.ssa.bit_size == 1) {
7757          assert(src.regClass() == bld.lm);
7758          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
7759          if (ctx->program->chip_class >= GFX8)
7760             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7761          else
7762             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7763          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
7764          emit_wqm(ctx, tmp, dst);
7765       } else if (instr->dest.ssa.bit_size == 32) {
7766          Temp tmp;
7767          if (ctx->program->chip_class >= GFX8)
7768             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7769          else
7770             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7771          emit_wqm(ctx, tmp, dst);
7772       } else if (instr->dest.ssa.bit_size == 64) {
7773          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7774          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7775          if (ctx->program->chip_class >= GFX8) {
7776             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7777             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7778          } else {
7779             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
7780             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
7781          }
7782          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7783          emit_split_vector(ctx, dst, 2);
7784       } else {
7785          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7786          nir_print_instr(&instr->instr, stderr);
7787          fprintf(stderr, "\n");
7788       }
7789       break;
7790    }
7791    case nir_intrinsic_masked_swizzle_amd: {
7792       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7793       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7794          emit_uniform_subgroup(ctx, instr, src);
7795          break;
7796       }
7797       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7798       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
7799       if (dst.regClass() == v1) {
7800          emit_wqm(ctx,
7801                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
7802                   dst);
7803       } else if (dst.regClass() == v2) {
7804          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7805          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7806          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
7807          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
7808          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7809          emit_split_vector(ctx, dst, 2);
7810       } else {
7811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7812          nir_print_instr(&instr->instr, stderr);
7813          fprintf(stderr, "\n");
7814       }
7815       break;
7816    }
7817    case nir_intrinsic_write_invocation_amd: {
7818       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
7819       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
7820       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
7821       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7822       if (dst.regClass() == v1) {
7823          /* src2 is ignored for writelane. RA assigns the same reg for dst */
7824          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
7825       } else if (dst.regClass() == v2) {
7826          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
7827          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
7828          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
7829          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
7830          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
7831          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
7832          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7833          emit_split_vector(ctx, dst, 2);
7834       } else {
7835          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7836          nir_print_instr(&instr->instr, stderr);
7837          fprintf(stderr, "\n");
7838       }
7839       break;
7840    }
7841    case nir_intrinsic_mbcnt_amd: {
7842       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7843       RegClass rc = RegClass(src.type(), 1);
7844       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
7845       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
7846       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7847       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
7848       emit_wqm(ctx, wqm_tmp, dst);
7849       break;
7850    }
7851    case nir_intrinsic_load_helper_invocation: {
7852       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7853       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
7854       ctx->block->kind |= block_kind_needs_lowering;
7855       ctx->program->needs_exact = true;
7856       break;
7857    }
7858    case nir_intrinsic_is_helper_invocation: {
7859       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7860       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
7861       ctx->block->kind |= block_kind_needs_lowering;
7862       ctx->program->needs_exact = true;
7863       break;
7864    }
7865    case nir_intrinsic_demote:
7866       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
7867
7868       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7869          ctx->cf_info.exec_potentially_empty_discard = true;
7870       ctx->block->kind |= block_kind_uses_demote;
7871       ctx->program->needs_exact = true;
7872       break;
7873    case nir_intrinsic_demote_if: {
7874       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7875       assert(src.regClass() == bld.lm);
7876       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
7877       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
7878
7879       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7880          ctx->cf_info.exec_potentially_empty_discard = true;
7881       ctx->block->kind |= block_kind_uses_demote;
7882       ctx->program->needs_exact = true;
7883       break;
7884    }
7885    case nir_intrinsic_first_invocation: {
7886       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
7887                get_ssa_temp(ctx, &instr->dest.ssa));
7888       break;
7889    }
7890    case nir_intrinsic_shader_clock:
7891       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
7892       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
7893       break;
7894    case nir_intrinsic_load_vertex_id_zero_base: {
7895       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7896       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
7897       break;
7898    }
7899    case nir_intrinsic_load_first_vertex: {
7900       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7901       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
7902       break;
7903    }
7904    case nir_intrinsic_load_base_instance: {
7905       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7906       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
7907       break;
7908    }
7909    case nir_intrinsic_load_instance_id: {
7910       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7911       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
7912       break;
7913    }
7914    case nir_intrinsic_load_draw_id: {
7915       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7916       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
7917       break;
7918    }
7919    case nir_intrinsic_load_invocation_id: {
7920       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7921
7922       if (ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
7923          if (ctx->options->chip_class >= GFX10)
7924             bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7925          else
7926             bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7927       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7928          bld.vop3(aco_opcode::v_bfe_u32, Definition(dst),
7929                   get_arg(ctx, ctx->args->ac.tcs_rel_ids), Operand(8u), Operand(5u));
7930       } else {
7931          unreachable("Unsupported stage for load_invocation_id");
7932       }
7933
7934       break;
7935    }
7936    case nir_intrinsic_load_primitive_id: {
7937       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7938
7939       switch (ctx->shader->info.stage) {
7940       case MESA_SHADER_GEOMETRY:
7941          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
7942          break;
7943       case MESA_SHADER_TESS_CTRL:
7944          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tcs_patch_id));
7945          break;
7946       case MESA_SHADER_TESS_EVAL:
7947          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tes_patch_id));
7948          break;
7949       default:
7950          unreachable("Unimplemented shader stage for nir_intrinsic_load_primitive_id");
7951       }
7952
7953       break;
7954    }
7955    case nir_intrinsic_load_patch_vertices_in: {
7956       assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL ||
7957              ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
7958
7959       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7960       bld.copy(Definition(dst), Operand(ctx->args->options->key.tcs.input_vertices));
7961       break;
7962    }
7963    case nir_intrinsic_emit_vertex_with_counter: {
7964       visit_emit_vertex_with_counter(ctx, instr);
7965       break;
7966    }
7967    case nir_intrinsic_end_primitive_with_counter: {
7968       unsigned stream = nir_intrinsic_stream_id(instr);
7969       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
7970       break;
7971    }
7972    case nir_intrinsic_set_vertex_count: {
7973       /* unused, the HW keeps track of this for us */
7974       break;
7975    }
7976    default:
7977       fprintf(stderr, "Unimplemented intrinsic instr: ");
7978       nir_print_instr(&instr->instr, stderr);
7979       fprintf(stderr, "\n");
7980       abort();
7981
7982       break;
7983    }
7984 }
7985
7986
7987 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
7988                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
7989                     enum glsl_base_type *stype)
7990 {
7991    nir_deref_instr *texture_deref_instr = NULL;
7992    nir_deref_instr *sampler_deref_instr = NULL;
7993    int plane = -1;
7994
7995    for (unsigned i = 0; i < instr->num_srcs; i++) {
7996       switch (instr->src[i].src_type) {
7997       case nir_tex_src_texture_deref:
7998          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
7999          break;
8000       case nir_tex_src_sampler_deref:
8001          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
8002          break;
8003       case nir_tex_src_plane:
8004          plane = nir_src_as_int(instr->src[i].src);
8005          break;
8006       default:
8007          break;
8008       }
8009    }
8010
8011    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
8012
8013    if (!sampler_deref_instr)
8014       sampler_deref_instr = texture_deref_instr;
8015
8016    if (plane >= 0) {
8017       assert(instr->op != nir_texop_txf_ms &&
8018              instr->op != nir_texop_samples_identical);
8019       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
8020       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
8021    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
8022       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
8023    } else if (instr->op == nir_texop_fragment_mask_fetch) {
8024       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8025    } else {
8026       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
8027    }
8028    if (samp_ptr) {
8029       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
8030
8031       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
8032          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
8033          Builder bld(ctx->program, ctx->block);
8034
8035          /* to avoid unnecessary moves, we split and recombine sampler and image */
8036          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
8037                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8038          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8039          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
8040                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
8041                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
8042          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
8043                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
8044
8045          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
8046          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
8047                                img[0], img[1], img[2], img[3],
8048                                img[4], img[5], img[6], img[7]);
8049          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
8050                                 samp[0], samp[1], samp[2], samp[3]);
8051       }
8052    }
8053    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
8054                      instr->op == nir_texop_samples_identical))
8055       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8056 }
8057
8058 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
8059                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
8060 {
8061    Builder bld(ctx->program, ctx->block);
8062
8063    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
8064    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
8065    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
8066
8067    Operand neg_one(0xbf800000u);
8068    Operand one(0x3f800000u);
8069    Operand two(0x40000000u);
8070    Operand four(0x40800000u);
8071
8072    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
8073    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
8074    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
8075
8076    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
8077    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
8078    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
8079    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
8080
8081    // select sc
8082    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
8083    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
8084                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
8085                        one, is_ma_y);
8086    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8087
8088    // select tc
8089    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
8090    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
8091    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8092
8093    // select ma
8094    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8095                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
8096                   deriv_z, is_ma_z);
8097    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
8098    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
8099 }
8100
8101 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
8102 {
8103    Builder bld(ctx->program, ctx->block);
8104    Temp ma, tc, sc, id;
8105
8106    if (is_array) {
8107       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
8108
8109       // see comment in ac_prepare_cube_coords()
8110       if (ctx->options->chip_class <= GFX8)
8111          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
8112    }
8113
8114    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8115
8116    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
8117    vop3a->operands[0] = Operand(ma);
8118    vop3a->abs[0] = true;
8119    Temp invma = bld.tmp(v1);
8120    vop3a->definitions[0] = Definition(invma);
8121    ctx->block->instructions.emplace_back(std::move(vop3a));
8122
8123    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8124    if (!is_deriv)
8125       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
8126
8127    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8128    if (!is_deriv)
8129       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
8130
8131    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8132
8133    if (is_deriv) {
8134       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
8135       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
8136
8137       for (unsigned i = 0; i < 2; i++) {
8138          // see comment in ac_prepare_cube_coords()
8139          Temp deriv_ma;
8140          Temp deriv_sc, deriv_tc;
8141          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
8142                            &deriv_ma, &deriv_sc, &deriv_tc);
8143
8144          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
8145
8146          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8147                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
8148                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
8149          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8150                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
8151                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
8152          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
8153       }
8154
8155       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
8156       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
8157    }
8158
8159    if (is_array)
8160       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
8161    coords.resize(3);
8162    coords[0] = sc;
8163    coords[1] = tc;
8164    coords[2] = id;
8165 }
8166
8167 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
8168 {
8169    if (vec->parent_instr->type != nir_instr_type_alu)
8170       return;
8171    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
8172    if (vec_instr->op != nir_op_vec(vec->num_components))
8173       return;
8174
8175    for (unsigned i = 0; i < vec->num_components; i++) {
8176       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
8177               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
8178    }
8179 }
8180
8181 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
8182 {
8183    Builder bld(ctx->program, ctx->block);
8184    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
8185         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
8186    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
8187         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp();
8188    std::vector<Temp> coords;
8189    std::vector<Temp> derivs;
8190    nir_const_value *sample_index_cv = NULL;
8191    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
8192    enum glsl_base_type stype;
8193    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
8194
8195    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
8196                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
8197    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
8198                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
8199
8200    for (unsigned i = 0; i < instr->num_srcs; i++) {
8201       switch (instr->src[i].src_type) {
8202       case nir_tex_src_coord: {
8203          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
8204          for (unsigned i = 0; i < coord.size(); i++)
8205             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
8206          break;
8207       }
8208       case nir_tex_src_bias:
8209          if (instr->op == nir_texop_txb) {
8210             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
8211             has_bias = true;
8212          }
8213          break;
8214       case nir_tex_src_lod: {
8215          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
8216
8217          if (val && val->f32 <= 0.0) {
8218             level_zero = true;
8219          } else {
8220             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
8221             has_lod = true;
8222          }
8223          break;
8224       }
8225       case nir_tex_src_comparator:
8226          if (instr->is_shadow) {
8227             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
8228             has_compare = true;
8229          }
8230          break;
8231       case nir_tex_src_offset:
8232          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
8233          get_const_vec(instr->src[i].src.ssa, const_offset);
8234          has_offset = true;
8235          break;
8236       case nir_tex_src_ddx:
8237          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
8238          has_ddx = true;
8239          break;
8240       case nir_tex_src_ddy:
8241          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
8242          has_ddy = true;
8243          break;
8244       case nir_tex_src_ms_index:
8245          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
8246          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
8247          has_sample_index = true;
8248          break;
8249       case nir_tex_src_texture_offset:
8250       case nir_tex_src_sampler_offset:
8251       default:
8252          break;
8253       }
8254    }
8255
8256    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
8257       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
8258
8259    if (instr->op == nir_texop_texture_samples) {
8260       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
8261
8262       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
8263       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
8264       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
8265       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
8266
8267       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
8268                samples, Operand(1u), bld.scc(is_msaa));
8269       return;
8270    }
8271
8272    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
8273       aco_ptr<Instruction> tmp_instr;
8274       Temp acc, pack = Temp();
8275
8276       uint32_t pack_const = 0;
8277       for (unsigned i = 0; i < offset.size(); i++) {
8278          if (!const_offset[i])
8279             continue;
8280          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
8281       }
8282
8283       if (offset.type() == RegType::sgpr) {
8284          for (unsigned i = 0; i < offset.size(); i++) {
8285             if (const_offset[i])
8286                continue;
8287
8288             acc = emit_extract_vector(ctx, offset, i, s1);
8289             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
8290
8291             if (i) {
8292                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
8293             }
8294
8295             if (pack == Temp()) {
8296                pack = acc;
8297             } else {
8298                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
8299             }
8300          }
8301
8302          if (pack_const && pack != Temp())
8303             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
8304       } else {
8305          for (unsigned i = 0; i < offset.size(); i++) {
8306             if (const_offset[i])
8307                continue;
8308
8309             acc = emit_extract_vector(ctx, offset, i, v1);
8310             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
8311
8312             if (i) {
8313                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
8314             }
8315
8316             if (pack == Temp()) {
8317                pack = acc;
8318             } else {
8319                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
8320             }
8321          }
8322
8323          if (pack_const && pack != Temp())
8324             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
8325       }
8326       if (pack_const && pack == Temp())
8327          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
8328       else if (pack == Temp())
8329          has_offset = false;
8330       else
8331          offset = pack;
8332    }
8333
8334    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
8335       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
8336
8337    /* pack derivatives */
8338    if (has_ddx || has_ddy) {
8339       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
8340          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
8341          Temp zero = bld.copy(bld.def(v1), Operand(0u));
8342          derivs = {ddy, zero, ddy, zero};
8343       } else {
8344          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
8345             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
8346          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
8347             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
8348       }
8349       has_derivs = true;
8350    }
8351
8352    if (instr->coord_components > 1 &&
8353        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8354        instr->is_array &&
8355        instr->op != nir_texop_txf)
8356       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
8357
8358    if (instr->coord_components > 2 &&
8359       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
8360        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8361        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
8362        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8363        instr->is_array &&
8364        instr->op != nir_texop_txf &&
8365        instr->op != nir_texop_txf_ms &&
8366        instr->op != nir_texop_fragment_fetch &&
8367        instr->op != nir_texop_fragment_mask_fetch)
8368       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
8369
8370    if (ctx->options->chip_class == GFX9 &&
8371        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8372        instr->op != nir_texop_lod && instr->coord_components) {
8373       assert(coords.size() > 0 && coords.size() < 3);
8374
8375       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
8376                                                                      Operand((uint32_t) 0) :
8377                                                                      Operand((uint32_t) 0x3f000000)));
8378    }
8379
8380    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
8381
8382    if (instr->op == nir_texop_samples_identical)
8383       resource = fmask_ptr;
8384
8385    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8386              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8387             instr->op != nir_texop_txs &&
8388             instr->op != nir_texop_fragment_fetch &&
8389             instr->op != nir_texop_fragment_mask_fetch) {
8390       assert(has_sample_index);
8391       Operand op(sample_index);
8392       if (sample_index_cv)
8393          op = Operand(sample_index_cv->u32);
8394       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
8395    }
8396
8397    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
8398       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
8399          Temp off = emit_extract_vector(ctx, offset, i, v1);
8400          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
8401       }
8402       has_offset = false;
8403    }
8404
8405    /* Build tex instruction */
8406    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
8407    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
8408                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
8409                   : 0;
8410    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8411    Temp tmp_dst = dst;
8412
8413    /* gather4 selects the component by dmask and always returns vec4 */
8414    if (instr->op == nir_texop_tg4) {
8415       assert(instr->dest.ssa.num_components == 4);
8416       if (instr->is_shadow)
8417          dmask = 1;
8418       else
8419          dmask = 1 << instr->component;
8420       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
8421          tmp_dst = bld.tmp(v4);
8422    } else if (instr->op == nir_texop_samples_identical) {
8423       tmp_dst = bld.tmp(v1);
8424    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
8425       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
8426    }
8427
8428    aco_ptr<MIMG_instruction> tex;
8429    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
8430       if (!has_lod)
8431          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8432
8433       bool div_by_6 = instr->op == nir_texop_txs &&
8434                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
8435                       instr->is_array &&
8436                       (dmask & (1 << 2));
8437       if (tmp_dst.id() == dst.id() && div_by_6)
8438          tmp_dst = bld.tmp(tmp_dst.regClass());
8439
8440       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8441       tex->operands[0] = Operand(resource);
8442       tex->operands[1] = Operand(s4); /* no sampler */
8443       tex->operands[2] = Operand(as_vgpr(ctx,lod));
8444       if (ctx->options->chip_class == GFX9 &&
8445           instr->op == nir_texop_txs &&
8446           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8447           instr->is_array) {
8448          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
8449       } else if (instr->op == nir_texop_query_levels) {
8450          tex->dmask = 1 << 3;
8451       } else {
8452          tex->dmask = dmask;
8453       }
8454       tex->da = da;
8455       tex->definitions[0] = Definition(tmp_dst);
8456       tex->dim = dim;
8457       tex->can_reorder = true;
8458       ctx->block->instructions.emplace_back(std::move(tex));
8459
8460       if (div_by_6) {
8461          /* divide 3rd value by 6 by multiplying with magic number */
8462          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8463          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
8464          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
8465          assert(instr->dest.ssa.num_components == 3);
8466          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
8467          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8468                               emit_extract_vector(ctx, tmp_dst, 0, v1),
8469                               emit_extract_vector(ctx, tmp_dst, 1, v1),
8470                               by_6);
8471
8472       }
8473
8474       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8475       return;
8476    }
8477
8478    Temp tg4_compare_cube_wa64 = Temp();
8479
8480    if (tg4_integer_workarounds) {
8481       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8482       tex->operands[0] = Operand(resource);
8483       tex->operands[1] = Operand(s4); /* no sampler */
8484       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8485       tex->dim = dim;
8486       tex->dmask = 0x3;
8487       tex->da = da;
8488       Temp size = bld.tmp(v2);
8489       tex->definitions[0] = Definition(size);
8490       tex->can_reorder = true;
8491       ctx->block->instructions.emplace_back(std::move(tex));
8492       emit_split_vector(ctx, size, size.size());
8493
8494       Temp half_texel[2];
8495       for (unsigned i = 0; i < 2; i++) {
8496          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
8497          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
8498          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
8499          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
8500       }
8501
8502       Temp new_coords[2] = {
8503          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
8504          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
8505       };
8506
8507       if (tg4_integer_cube_workaround) {
8508          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
8509          Temp desc[resource.size()];
8510          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
8511                                                                            Format::PSEUDO, 1, resource.size())};
8512          split->operands[0] = Operand(resource);
8513          for (unsigned i = 0; i < resource.size(); i++) {
8514             desc[i] = bld.tmp(s1);
8515             split->definitions[i] = Definition(desc[i]);
8516          }
8517          ctx->block->instructions.emplace_back(std::move(split));
8518
8519          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
8520          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
8521                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
8522
8523          Temp nfmt;
8524          if (stype == GLSL_TYPE_UINT) {
8525             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8526                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
8527                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
8528                             bld.scc(compare_cube_wa));
8529          } else {
8530             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8531                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
8532                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
8533                             bld.scc(compare_cube_wa));
8534          }
8535          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
8536          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
8537
8538          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
8539
8540          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
8541                             Operand((uint32_t)C_008F14_NUM_FORMAT));
8542          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
8543
8544          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
8545                                                                          Format::PSEUDO, resource.size(), 1)};
8546          for (unsigned i = 0; i < resource.size(); i++)
8547             vec->operands[i] = Operand(desc[i]);
8548          resource = bld.tmp(resource.regClass());
8549          vec->definitions[0] = Definition(resource);
8550          ctx->block->instructions.emplace_back(std::move(vec));
8551
8552          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8553                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
8554          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8555                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
8556       }
8557       coords[0] = new_coords[0];
8558       coords[1] = new_coords[1];
8559    }
8560
8561    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
8562       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
8563
8564       assert(coords.size() == 1);
8565       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
8566       aco_opcode op;
8567       switch (last_bit) {
8568       case 1:
8569          op = aco_opcode::buffer_load_format_x; break;
8570       case 2:
8571          op = aco_opcode::buffer_load_format_xy; break;
8572       case 3:
8573          op = aco_opcode::buffer_load_format_xyz; break;
8574       case 4:
8575          op = aco_opcode::buffer_load_format_xyzw; break;
8576       default:
8577          unreachable("Tex instruction loads more than 4 components.");
8578       }
8579
8580       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
8581       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
8582          tmp_dst = dst;
8583       else
8584          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
8585
8586       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
8587       mubuf->operands[0] = Operand(resource);
8588       mubuf->operands[1] = Operand(coords[0]);
8589       mubuf->operands[2] = Operand((uint32_t) 0);
8590       mubuf->definitions[0] = Definition(tmp_dst);
8591       mubuf->idxen = true;
8592       mubuf->can_reorder = true;
8593       ctx->block->instructions.emplace_back(std::move(mubuf));
8594
8595       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
8596       return;
8597    }
8598
8599    /* gather MIMG address components */
8600    std::vector<Temp> args;
8601    if (has_offset)
8602       args.emplace_back(offset);
8603    if (has_bias)
8604       args.emplace_back(bias);
8605    if (has_compare)
8606       args.emplace_back(compare);
8607    if (has_derivs)
8608       args.insert(args.end(), derivs.begin(), derivs.end());
8609
8610    args.insert(args.end(), coords.begin(), coords.end());
8611    if (has_sample_index)
8612       args.emplace_back(sample_index);
8613    if (has_lod)
8614       args.emplace_back(lod);
8615
8616    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
8617    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
8618    vec->definitions[0] = Definition(arg);
8619    for (unsigned i = 0; i < args.size(); i++)
8620       vec->operands[i] = Operand(args[i]);
8621    ctx->block->instructions.emplace_back(std::move(vec));
8622
8623
8624    if (instr->op == nir_texop_txf ||
8625        instr->op == nir_texop_txf_ms ||
8626        instr->op == nir_texop_samples_identical ||
8627        instr->op == nir_texop_fragment_fetch ||
8628        instr->op == nir_texop_fragment_mask_fetch) {
8629       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
8630       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
8631       tex->operands[0] = Operand(resource);
8632       tex->operands[1] = Operand(s4); /* no sampler */
8633       tex->operands[2] = Operand(arg);
8634       tex->dim = dim;
8635       tex->dmask = dmask;
8636       tex->unrm = true;
8637       tex->da = da;
8638       tex->definitions[0] = Definition(tmp_dst);
8639       tex->can_reorder = true;
8640       ctx->block->instructions.emplace_back(std::move(tex));
8641
8642       if (instr->op == nir_texop_samples_identical) {
8643          assert(dmask == 1 && dst.regClass() == v1);
8644          assert(dst.id() != tmp_dst.id());
8645
8646          Temp tmp = bld.tmp(bld.lm);
8647          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
8648          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
8649
8650       } else {
8651          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8652       }
8653       return;
8654    }
8655
8656    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
8657    aco_opcode opcode = aco_opcode::image_sample;
8658    if (has_offset) { /* image_sample_*_o */
8659       if (has_compare) {
8660          opcode = aco_opcode::image_sample_c_o;
8661          if (has_derivs)
8662             opcode = aco_opcode::image_sample_c_d_o;
8663          if (has_bias)
8664             opcode = aco_opcode::image_sample_c_b_o;
8665          if (level_zero)
8666             opcode = aco_opcode::image_sample_c_lz_o;
8667          if (has_lod)
8668             opcode = aco_opcode::image_sample_c_l_o;
8669       } else {
8670          opcode = aco_opcode::image_sample_o;
8671          if (has_derivs)
8672             opcode = aco_opcode::image_sample_d_o;
8673          if (has_bias)
8674             opcode = aco_opcode::image_sample_b_o;
8675          if (level_zero)
8676             opcode = aco_opcode::image_sample_lz_o;
8677          if (has_lod)
8678             opcode = aco_opcode::image_sample_l_o;
8679       }
8680    } else { /* no offset */
8681       if (has_compare) {
8682          opcode = aco_opcode::image_sample_c;
8683          if (has_derivs)
8684             opcode = aco_opcode::image_sample_c_d;
8685          if (has_bias)
8686             opcode = aco_opcode::image_sample_c_b;
8687          if (level_zero)
8688             opcode = aco_opcode::image_sample_c_lz;
8689          if (has_lod)
8690             opcode = aco_opcode::image_sample_c_l;
8691       } else {
8692          opcode = aco_opcode::image_sample;
8693          if (has_derivs)
8694             opcode = aco_opcode::image_sample_d;
8695          if (has_bias)
8696             opcode = aco_opcode::image_sample_b;
8697          if (level_zero)
8698             opcode = aco_opcode::image_sample_lz;
8699          if (has_lod)
8700             opcode = aco_opcode::image_sample_l;
8701       }
8702    }
8703
8704    if (instr->op == nir_texop_tg4) {
8705       if (has_offset) {
8706          opcode = aco_opcode::image_gather4_lz_o;
8707          if (has_compare)
8708             opcode = aco_opcode::image_gather4_c_lz_o;
8709       } else {
8710          opcode = aco_opcode::image_gather4_lz;
8711          if (has_compare)
8712             opcode = aco_opcode::image_gather4_c_lz;
8713       }
8714    } else if (instr->op == nir_texop_lod) {
8715       opcode = aco_opcode::image_get_lod;
8716    }
8717
8718    /* we don't need the bias, sample index, compare value or offset to be
8719     * computed in WQM but if the p_create_vector copies the coordinates, then it
8720     * needs to be in WQM */
8721    if (ctx->stage == fragment_fs &&
8722        !has_derivs && !has_lod && !level_zero &&
8723        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
8724        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
8725       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
8726
8727    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
8728    tex->operands[0] = Operand(resource);
8729    tex->operands[1] = Operand(sampler);
8730    tex->operands[2] = Operand(arg);
8731    tex->dim = dim;
8732    tex->dmask = dmask;
8733    tex->da = da;
8734    tex->definitions[0] = Definition(tmp_dst);
8735    tex->can_reorder = true;
8736    ctx->block->instructions.emplace_back(std::move(tex));
8737
8738    if (tg4_integer_cube_workaround) {
8739       assert(tmp_dst.id() != dst.id());
8740       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
8741
8742       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8743       Temp val[4];
8744       for (unsigned i = 0; i < dst.size(); i++) {
8745          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
8746          Temp cvt_val;
8747          if (stype == GLSL_TYPE_UINT)
8748             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
8749          else
8750             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
8751          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
8752       }
8753       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
8754       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8755                            val[0], val[1], val[2], val[3]);
8756    }
8757    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
8758    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
8759
8760 }
8761
8762
8763 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
8764 {
8765    Temp tmp = get_ssa_temp(ctx, ssa);
8766    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
8767       return Operand(tmp.regClass());
8768    else
8769       return Operand(tmp);
8770 }
8771
8772 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
8773 {
8774    aco_ptr<Pseudo_instruction> phi;
8775    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8776    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
8777
8778    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
8779    logical |= ctx->block->kind & block_kind_merge;
8780    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
8781
8782    /* we want a sorted list of sources, since the predecessor list is also sorted */
8783    std::map<unsigned, nir_ssa_def*> phi_src;
8784    nir_foreach_phi_src(src, instr)
8785       phi_src[src->pred->index] = src->src.ssa;
8786
8787    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
8788    unsigned num_operands = 0;
8789    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size()) + 1];
8790    unsigned num_defined = 0;
8791    unsigned cur_pred_idx = 0;
8792    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
8793       if (cur_pred_idx < preds.size()) {
8794          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
8795          unsigned block = ctx->cf_info.nir_to_aco[src.first];
8796          unsigned skipped = 0;
8797          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
8798             skipped++;
8799          if (cur_pred_idx + skipped < preds.size()) {
8800             for (unsigned i = 0; i < skipped; i++)
8801                operands[num_operands++] = Operand(dst.regClass());
8802             cur_pred_idx += skipped;
8803          } else {
8804             continue;
8805          }
8806       }
8807       /* Handle missing predecessors at the end. This shouldn't happen with loop
8808        * headers and we can't ignore these sources for loop header phis. */
8809       if (!(ctx->block->kind & block_kind_loop_header) && cur_pred_idx >= preds.size())
8810          continue;
8811       cur_pred_idx++;
8812       Operand op = get_phi_operand(ctx, src.second);
8813       operands[num_operands++] = op;
8814       num_defined += !op.isUndefined();
8815    }
8816    /* handle block_kind_continue_or_break at loop exit blocks */
8817    while (cur_pred_idx++ < preds.size())
8818       operands[num_operands++] = Operand(dst.regClass());
8819
8820    /* If the loop ends with a break, still add a linear continue edge in case
8821     * that break is divergent or continue_or_break is used. We'll either remove
8822     * this operand later in visit_loop() if it's not necessary or replace the
8823     * undef with something correct. */
8824    if (!logical && ctx->block->kind & block_kind_loop_header) {
8825       nir_loop *loop = nir_cf_node_as_loop(instr->instr.block->cf_node.parent);
8826       nir_block *last = nir_loop_last_block(loop);
8827       if (last->successors[0] != instr->instr.block)
8828          operands[num_operands++] = Operand(RegClass());
8829    }
8830
8831    if (num_defined == 0) {
8832       Builder bld(ctx->program, ctx->block);
8833       if (dst.regClass() == s1) {
8834          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
8835       } else if (dst.regClass() == v1) {
8836          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
8837       } else {
8838          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8839          for (unsigned i = 0; i < dst.size(); i++)
8840             vec->operands[i] = Operand(0u);
8841          vec->definitions[0] = Definition(dst);
8842          ctx->block->instructions.emplace_back(std::move(vec));
8843       }
8844       return;
8845    }
8846
8847    /* we can use a linear phi in some cases if one src is undef */
8848    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
8849       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
8850
8851       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
8852       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
8853       assert(invert->kind & block_kind_invert);
8854
8855       unsigned then_block = invert->linear_preds[0];
8856
8857       Block* insert_block = NULL;
8858       for (unsigned i = 0; i < num_operands; i++) {
8859          Operand op = operands[i];
8860          if (op.isUndefined())
8861             continue;
8862          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
8863          phi->operands[0] = op;
8864          break;
8865       }
8866       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
8867       phi->operands[1] = Operand(dst.regClass());
8868       phi->definitions[0] = Definition(dst);
8869       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
8870       return;
8871    }
8872
8873    /* try to scalarize vector phis */
8874    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
8875       // TODO: scalarize linear phis on divergent ifs
8876       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
8877       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
8878       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
8879          Operand src = operands[i];
8880          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
8881             can_scalarize = false;
8882       }
8883       if (can_scalarize) {
8884          unsigned num_components = instr->dest.ssa.num_components;
8885          assert(dst.size() % num_components == 0);
8886          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
8887
8888          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
8889          for (unsigned k = 0; k < num_components; k++) {
8890             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8891             for (unsigned i = 0; i < num_operands; i++) {
8892                Operand src = operands[i];
8893                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
8894             }
8895             Temp phi_dst = {ctx->program->allocateId(), rc};
8896             phi->definitions[0] = Definition(phi_dst);
8897             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8898             new_vec[k] = phi_dst;
8899             vec->operands[k] = Operand(phi_dst);
8900          }
8901          vec->definitions[0] = Definition(dst);
8902          ctx->block->instructions.emplace_back(std::move(vec));
8903          ctx->allocated_vec.emplace(dst.id(), new_vec);
8904          return;
8905       }
8906    }
8907
8908    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8909    for (unsigned i = 0; i < num_operands; i++)
8910       phi->operands[i] = operands[i];
8911    phi->definitions[0] = Definition(dst);
8912    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8913 }
8914
8915
8916 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
8917 {
8918    Temp dst = get_ssa_temp(ctx, &instr->def);
8919
8920    assert(dst.type() == RegType::sgpr);
8921
8922    if (dst.size() == 1) {
8923       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
8924    } else {
8925       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8926       for (unsigned i = 0; i < dst.size(); i++)
8927          vec->operands[i] = Operand(0u);
8928       vec->definitions[0] = Definition(dst);
8929       ctx->block->instructions.emplace_back(std::move(vec));
8930    }
8931 }
8932
8933 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
8934 {
8935    Builder bld(ctx->program, ctx->block);
8936    Block *logical_target;
8937    append_logical_end(ctx->block);
8938    unsigned idx = ctx->block->index;
8939
8940    switch (instr->type) {
8941    case nir_jump_break:
8942       logical_target = ctx->cf_info.parent_loop.exit;
8943       add_logical_edge(idx, logical_target);
8944       ctx->block->kind |= block_kind_break;
8945
8946       if (!ctx->cf_info.parent_if.is_divergent &&
8947           !ctx->cf_info.parent_loop.has_divergent_continue) {
8948          /* uniform break - directly jump out of the loop */
8949          ctx->block->kind |= block_kind_uniform;
8950          ctx->cf_info.has_branch = true;
8951          bld.branch(aco_opcode::p_branch);
8952          add_linear_edge(idx, logical_target);
8953          return;
8954       }
8955       ctx->cf_info.parent_loop.has_divergent_branch = true;
8956       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8957       break;
8958    case nir_jump_continue:
8959       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8960       add_logical_edge(idx, logical_target);
8961       ctx->block->kind |= block_kind_continue;
8962
8963       if (ctx->cf_info.parent_if.is_divergent) {
8964          /* for potential uniform breaks after this continue,
8965             we must ensure that they are handled correctly */
8966          ctx->cf_info.parent_loop.has_divergent_continue = true;
8967          ctx->cf_info.parent_loop.has_divergent_branch = true;
8968          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8969       } else {
8970          /* uniform continue - directly jump to the loop header */
8971          ctx->block->kind |= block_kind_uniform;
8972          ctx->cf_info.has_branch = true;
8973          bld.branch(aco_opcode::p_branch);
8974          add_linear_edge(idx, logical_target);
8975          return;
8976       }
8977       break;
8978    default:
8979       fprintf(stderr, "Unknown NIR jump instr: ");
8980       nir_print_instr(&instr->instr, stderr);
8981       fprintf(stderr, "\n");
8982       abort();
8983    }
8984
8985    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
8986       ctx->cf_info.exec_potentially_empty_break = true;
8987       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
8988    }
8989
8990    /* remove critical edges from linear CFG */
8991    bld.branch(aco_opcode::p_branch);
8992    Block* break_block = ctx->program->create_and_insert_block();
8993    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8994    break_block->kind |= block_kind_uniform;
8995    add_linear_edge(idx, break_block);
8996    /* the loop_header pointer might be invalidated by this point */
8997    if (instr->type == nir_jump_continue)
8998       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8999    add_linear_edge(break_block->index, logical_target);
9000    bld.reset(break_block);
9001    bld.branch(aco_opcode::p_branch);
9002
9003    Block* continue_block = ctx->program->create_and_insert_block();
9004    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9005    add_linear_edge(idx, continue_block);
9006    append_logical_start(continue_block);
9007    ctx->block = continue_block;
9008    return;
9009 }
9010
9011 void visit_block(isel_context *ctx, nir_block *block)
9012 {
9013    nir_foreach_instr(instr, block) {
9014       switch (instr->type) {
9015       case nir_instr_type_alu:
9016          visit_alu_instr(ctx, nir_instr_as_alu(instr));
9017          break;
9018       case nir_instr_type_load_const:
9019          visit_load_const(ctx, nir_instr_as_load_const(instr));
9020          break;
9021       case nir_instr_type_intrinsic:
9022          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
9023          break;
9024       case nir_instr_type_tex:
9025          visit_tex(ctx, nir_instr_as_tex(instr));
9026          break;
9027       case nir_instr_type_phi:
9028          visit_phi(ctx, nir_instr_as_phi(instr));
9029          break;
9030       case nir_instr_type_ssa_undef:
9031          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
9032          break;
9033       case nir_instr_type_deref:
9034          break;
9035       case nir_instr_type_jump:
9036          visit_jump(ctx, nir_instr_as_jump(instr));
9037          break;
9038       default:
9039          fprintf(stderr, "Unknown NIR instr type: ");
9040          nir_print_instr(instr, stderr);
9041          fprintf(stderr, "\n");
9042          //abort();
9043       }
9044    }
9045
9046    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9047       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
9048 }
9049
9050
9051
9052 static Operand create_continue_phis(isel_context *ctx, unsigned first, unsigned last,
9053                                     aco_ptr<Instruction>& header_phi, Operand *vals)
9054 {
9055    vals[0] = Operand(header_phi->definitions[0].getTemp());
9056    RegClass rc = vals[0].regClass();
9057
9058    unsigned loop_nest_depth = ctx->program->blocks[first].loop_nest_depth;
9059
9060    unsigned next_pred = 1;
9061
9062    for (unsigned idx = first + 1; idx <= last; idx++) {
9063       Block& block = ctx->program->blocks[idx];
9064       if (block.loop_nest_depth != loop_nest_depth) {
9065          vals[idx - first] = vals[idx - 1 - first];
9066          continue;
9067       }
9068
9069       if (block.kind & block_kind_continue) {
9070          vals[idx - first] = header_phi->operands[next_pred];
9071          next_pred++;
9072          continue;
9073       }
9074
9075       bool all_same = true;
9076       for (unsigned i = 1; all_same && (i < block.linear_preds.size()); i++)
9077          all_same = vals[block.linear_preds[i] - first] == vals[block.linear_preds[0] - first];
9078
9079       Operand val;
9080       if (all_same) {
9081          val = vals[block.linear_preds[0] - first];
9082       } else {
9083          aco_ptr<Instruction> phi(create_instruction<Pseudo_instruction>(
9084             aco_opcode::p_linear_phi, Format::PSEUDO, block.linear_preds.size(), 1));
9085          for (unsigned i = 0; i < block.linear_preds.size(); i++)
9086             phi->operands[i] = vals[block.linear_preds[i] - first];
9087          val = Operand(Temp(ctx->program->allocateId(), rc));
9088          phi->definitions[0] = Definition(val.getTemp());
9089          block.instructions.emplace(block.instructions.begin(), std::move(phi));
9090       }
9091       vals[idx - first] = val;
9092    }
9093
9094    return vals[last - first];
9095 }
9096
9097 static void visit_loop(isel_context *ctx, nir_loop *loop)
9098 {
9099    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
9100    append_logical_end(ctx->block);
9101    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
9102    Builder bld(ctx->program, ctx->block);
9103    bld.branch(aco_opcode::p_branch);
9104    unsigned loop_preheader_idx = ctx->block->index;
9105
9106    Block loop_exit = Block();
9107    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9108    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
9109
9110    Block* loop_header = ctx->program->create_and_insert_block();
9111    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
9112    loop_header->kind |= block_kind_loop_header;
9113    add_edge(loop_preheader_idx, loop_header);
9114    ctx->block = loop_header;
9115
9116    /* emit loop body */
9117    unsigned loop_header_idx = loop_header->index;
9118    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
9119    append_logical_start(ctx->block);
9120    bool unreachable = visit_cf_list(ctx, &loop->body);
9121
9122    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
9123    if (!ctx->cf_info.has_branch) {
9124       append_logical_end(ctx->block);
9125       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
9126          /* Discards can result in code running with an empty exec mask.
9127           * This would result in divergent breaks not ever being taken. As a
9128           * workaround, break the loop when the loop mask is empty instead of
9129           * always continuing. */
9130          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
9131          unsigned block_idx = ctx->block->index;
9132
9133          /* create helper blocks to avoid critical edges */
9134          Block *break_block = ctx->program->create_and_insert_block();
9135          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9136          break_block->kind = block_kind_uniform;
9137          bld.reset(break_block);
9138          bld.branch(aco_opcode::p_branch);
9139          add_linear_edge(block_idx, break_block);
9140          add_linear_edge(break_block->index, &loop_exit);
9141
9142          Block *continue_block = ctx->program->create_and_insert_block();
9143          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9144          continue_block->kind = block_kind_uniform;
9145          bld.reset(continue_block);
9146          bld.branch(aco_opcode::p_branch);
9147          add_linear_edge(block_idx, continue_block);
9148          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
9149
9150          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9151             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
9152          ctx->block = &ctx->program->blocks[block_idx];
9153       } else {
9154          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
9155          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9156             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9157          else
9158             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9159       }
9160
9161       bld.reset(ctx->block);
9162       bld.branch(aco_opcode::p_branch);
9163    }
9164
9165    /* Fixup phis in loop header from unreachable blocks.
9166     * has_branch/has_divergent_branch also indicates if the loop ends with a
9167     * break/continue instruction, but we don't emit those if unreachable=true */
9168    if (unreachable) {
9169       assert(ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch);
9170       bool linear = ctx->cf_info.has_branch;
9171       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
9172       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9173          if ((logical && instr->opcode == aco_opcode::p_phi) ||
9174              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
9175             /* the last operand should be the one that needs to be removed */
9176             instr->operands.pop_back();
9177          } else if (!is_phi(instr)) {
9178             break;
9179          }
9180       }
9181    }
9182
9183    /* Fixup linear phis in loop header from expecting a continue. Both this fixup
9184     * and the previous one shouldn't both happen at once because a break in the
9185     * merge block would get CSE'd */
9186    if (nir_loop_last_block(loop)->successors[0] != nir_loop_first_block(loop)) {
9187       unsigned num_vals = ctx->cf_info.has_branch ? 1 : (ctx->block->index - loop_header_idx + 1);
9188       Operand vals[num_vals];
9189       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9190          if (instr->opcode == aco_opcode::p_linear_phi) {
9191             if (ctx->cf_info.has_branch)
9192                instr->operands.pop_back();
9193             else
9194                instr->operands.back() = create_continue_phis(ctx, loop_header_idx, ctx->block->index, instr, vals);
9195          } else if (!is_phi(instr)) {
9196             break;
9197          }
9198       }
9199    }
9200
9201    ctx->cf_info.has_branch = false;
9202
9203    // TODO: if the loop has not a single exit, we must add one °°
9204    /* emit loop successor block */
9205    ctx->block = ctx->program->insert_block(std::move(loop_exit));
9206    append_logical_start(ctx->block);
9207
9208    #if 0
9209    // TODO: check if it is beneficial to not branch on continues
9210    /* trim linear phis in loop header */
9211    for (auto&& instr : loop_entry->instructions) {
9212       if (instr->opcode == aco_opcode::p_linear_phi) {
9213          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
9214          new_phi->definitions[0] = instr->definitions[0];
9215          for (unsigned i = 0; i < new_phi->operands.size(); i++)
9216             new_phi->operands[i] = instr->operands[i];
9217          /* check that the remaining operands are all the same */
9218          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
9219             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
9220          instr.swap(new_phi);
9221       } else if (instr->opcode == aco_opcode::p_phi) {
9222          continue;
9223       } else {
9224          break;
9225       }
9226    }
9227    #endif
9228 }
9229
9230 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
9231 {
9232    ic->cond = cond;
9233
9234    append_logical_end(ctx->block);
9235    ctx->block->kind |= block_kind_branch;
9236
9237    /* branch to linear then block */
9238    assert(cond.regClass() == ctx->program->lane_mask);
9239    aco_ptr<Pseudo_branch_instruction> branch;
9240    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
9241    branch->operands[0] = Operand(cond);
9242    ctx->block->instructions.push_back(std::move(branch));
9243
9244    ic->BB_if_idx = ctx->block->index;
9245    ic->BB_invert = Block();
9246    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9247    /* Invert blocks are intentionally not marked as top level because they
9248     * are not part of the logical cfg. */
9249    ic->BB_invert.kind |= block_kind_invert;
9250    ic->BB_endif = Block();
9251    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9252    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
9253
9254    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
9255    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
9256    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
9257    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
9258    ctx->cf_info.parent_if.is_divergent = true;
9259
9260    /* divergent branches use cbranch_execz */
9261    ctx->cf_info.exec_potentially_empty_discard = false;
9262    ctx->cf_info.exec_potentially_empty_break = false;
9263    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9264
9265    /** emit logical then block */
9266    Block* BB_then_logical = ctx->program->create_and_insert_block();
9267    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9268    add_edge(ic->BB_if_idx, BB_then_logical);
9269    ctx->block = BB_then_logical;
9270    append_logical_start(BB_then_logical);
9271 }
9272
9273 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
9274 {
9275    Block *BB_then_logical = ctx->block;
9276    append_logical_end(BB_then_logical);
9277     /* branch from logical then block to invert block */
9278    aco_ptr<Pseudo_branch_instruction> branch;
9279    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9280    BB_then_logical->instructions.emplace_back(std::move(branch));
9281    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
9282    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9283       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
9284    BB_then_logical->kind |= block_kind_uniform;
9285    assert(!ctx->cf_info.has_branch);
9286    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9287    ctx->cf_info.parent_loop.has_divergent_branch = false;
9288
9289    /** emit linear then block */
9290    Block* BB_then_linear = ctx->program->create_and_insert_block();
9291    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9292    BB_then_linear->kind |= block_kind_uniform;
9293    add_linear_edge(ic->BB_if_idx, BB_then_linear);
9294    /* branch from linear then block to invert block */
9295    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9296    BB_then_linear->instructions.emplace_back(std::move(branch));
9297    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
9298
9299    /** emit invert merge block */
9300    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
9301    ic->invert_idx = ctx->block->index;
9302
9303    /* branch to linear else block (skip else) */
9304    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
9305    branch->operands[0] = Operand(ic->cond);
9306    ctx->block->instructions.push_back(std::move(branch));
9307
9308    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
9309    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
9310    ic->exec_potentially_empty_break_depth_old =
9311       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9312    /* divergent branches use cbranch_execz */
9313    ctx->cf_info.exec_potentially_empty_discard = false;
9314    ctx->cf_info.exec_potentially_empty_break = false;
9315    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9316
9317    /** emit logical else block */
9318    Block* BB_else_logical = ctx->program->create_and_insert_block();
9319    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9320    add_logical_edge(ic->BB_if_idx, BB_else_logical);
9321    add_linear_edge(ic->invert_idx, BB_else_logical);
9322    ctx->block = BB_else_logical;
9323    append_logical_start(BB_else_logical);
9324 }
9325
9326 static void end_divergent_if(isel_context *ctx, if_context *ic)
9327 {
9328    Block *BB_else_logical = ctx->block;
9329    append_logical_end(BB_else_logical);
9330
9331    /* branch from logical else block to endif block */
9332    aco_ptr<Pseudo_branch_instruction> branch;
9333    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9334    BB_else_logical->instructions.emplace_back(std::move(branch));
9335    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
9336    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9337       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
9338    BB_else_logical->kind |= block_kind_uniform;
9339
9340    assert(!ctx->cf_info.has_branch);
9341    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9342
9343
9344    /** emit linear else block */
9345    Block* BB_else_linear = ctx->program->create_and_insert_block();
9346    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9347    BB_else_linear->kind |= block_kind_uniform;
9348    add_linear_edge(ic->invert_idx, BB_else_linear);
9349
9350    /* branch from linear else block to endif block */
9351    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9352    BB_else_linear->instructions.emplace_back(std::move(branch));
9353    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
9354
9355
9356    /** emit endif merge block */
9357    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9358    append_logical_start(ctx->block);
9359
9360
9361    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
9362    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
9363    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
9364    ctx->cf_info.exec_potentially_empty_break_depth =
9365       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9366    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
9367        !ctx->cf_info.parent_if.is_divergent) {
9368       ctx->cf_info.exec_potentially_empty_break = false;
9369       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9370    }
9371    /* uniform control flow never has an empty exec-mask */
9372    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
9373       ctx->cf_info.exec_potentially_empty_discard = false;
9374       ctx->cf_info.exec_potentially_empty_break = false;
9375       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9376    }
9377 }
9378
9379 static void begin_uniform_if_then(isel_context *ctx, if_context *ic, Temp cond)
9380 {
9381    assert(cond.regClass() == s1);
9382
9383    append_logical_end(ctx->block);
9384    ctx->block->kind |= block_kind_uniform;
9385
9386    aco_ptr<Pseudo_branch_instruction> branch;
9387    aco_opcode branch_opcode = aco_opcode::p_cbranch_z;
9388    branch.reset(create_instruction<Pseudo_branch_instruction>(branch_opcode, Format::PSEUDO_BRANCH, 1, 0));
9389    branch->operands[0] = Operand(cond);
9390    branch->operands[0].setFixed(scc);
9391    ctx->block->instructions.emplace_back(std::move(branch));
9392
9393    ic->BB_if_idx = ctx->block->index;
9394    ic->BB_endif = Block();
9395    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9396    ic->BB_endif.kind |= ctx->block->kind & block_kind_top_level;
9397
9398    ctx->cf_info.has_branch = false;
9399    ctx->cf_info.parent_loop.has_divergent_branch = false;
9400
9401    /** emit then block */
9402    Block* BB_then = ctx->program->create_and_insert_block();
9403    BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9404    add_edge(ic->BB_if_idx, BB_then);
9405    append_logical_start(BB_then);
9406    ctx->block = BB_then;
9407 }
9408
9409 static void begin_uniform_if_else(isel_context *ctx, if_context *ic)
9410 {
9411    Block *BB_then = ctx->block;
9412
9413    ic->uniform_has_then_branch = ctx->cf_info.has_branch;
9414    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9415
9416    if (!ic->uniform_has_then_branch) {
9417       append_logical_end(BB_then);
9418       /* branch from then block to endif block */
9419       aco_ptr<Pseudo_branch_instruction> branch;
9420       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9421       BB_then->instructions.emplace_back(std::move(branch));
9422       add_linear_edge(BB_then->index, &ic->BB_endif);
9423       if (!ic->then_branch_divergent)
9424          add_logical_edge(BB_then->index, &ic->BB_endif);
9425       BB_then->kind |= block_kind_uniform;
9426    }
9427
9428    ctx->cf_info.has_branch = false;
9429    ctx->cf_info.parent_loop.has_divergent_branch = false;
9430
9431    /** emit else block */
9432    Block* BB_else = ctx->program->create_and_insert_block();
9433    BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9434    add_edge(ic->BB_if_idx, BB_else);
9435    append_logical_start(BB_else);
9436    ctx->block = BB_else;
9437 }
9438
9439 static void end_uniform_if(isel_context *ctx, if_context *ic)
9440 {
9441    Block *BB_else = ctx->block;
9442
9443    if (!ctx->cf_info.has_branch) {
9444       append_logical_end(BB_else);
9445       /* branch from then block to endif block */
9446       aco_ptr<Pseudo_branch_instruction> branch;
9447       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9448       BB_else->instructions.emplace_back(std::move(branch));
9449       add_linear_edge(BB_else->index, &ic->BB_endif);
9450       if (!ctx->cf_info.parent_loop.has_divergent_branch)
9451          add_logical_edge(BB_else->index, &ic->BB_endif);
9452       BB_else->kind |= block_kind_uniform;
9453    }
9454
9455    ctx->cf_info.has_branch &= ic->uniform_has_then_branch;
9456    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9457
9458    /** emit endif merge block */
9459    if (!ctx->cf_info.has_branch) {
9460       ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9461       append_logical_start(ctx->block);
9462    }
9463 }
9464
9465 static bool visit_if(isel_context *ctx, nir_if *if_stmt)
9466 {
9467    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
9468    Builder bld(ctx->program, ctx->block);
9469    aco_ptr<Pseudo_branch_instruction> branch;
9470    if_context ic;
9471
9472    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
9473       /**
9474        * Uniform conditionals are represented in the following way*) :
9475        *
9476        * The linear and logical CFG:
9477        *                        BB_IF
9478        *                        /    \
9479        *       BB_THEN (logical)      BB_ELSE (logical)
9480        *                        \    /
9481        *                        BB_ENDIF
9482        *
9483        * *) Exceptions may be due to break and continue statements within loops
9484        *    If a break/continue happens within uniform control flow, it branches
9485        *    to the loop exit/entry block. Otherwise, it branches to the next
9486        *    merge block.
9487        **/
9488
9489       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
9490       assert(cond.regClass() == ctx->program->lane_mask);
9491       cond = bool_to_scalar_condition(ctx, cond);
9492
9493       begin_uniform_if_then(ctx, &ic, cond);
9494       visit_cf_list(ctx, &if_stmt->then_list);
9495
9496       begin_uniform_if_else(ctx, &ic);
9497       visit_cf_list(ctx, &if_stmt->else_list);
9498
9499       end_uniform_if(ctx, &ic);
9500
9501       return !ctx->cf_info.has_branch;
9502    } else { /* non-uniform condition */
9503       /**
9504        * To maintain a logical and linear CFG without critical edges,
9505        * non-uniform conditionals are represented in the following way*) :
9506        *
9507        * The linear CFG:
9508        *                        BB_IF
9509        *                        /    \
9510        *       BB_THEN (logical)      BB_THEN (linear)
9511        *                        \    /
9512        *                        BB_INVERT (linear)
9513        *                        /    \
9514        *       BB_ELSE (logical)      BB_ELSE (linear)
9515        *                        \    /
9516        *                        BB_ENDIF
9517        *
9518        * The logical CFG:
9519        *                        BB_IF
9520        *                        /    \
9521        *       BB_THEN (logical)      BB_ELSE (logical)
9522        *                        \    /
9523        *                        BB_ENDIF
9524        *
9525        * *) Exceptions may be due to break and continue statements within loops
9526        **/
9527
9528       begin_divergent_if_then(ctx, &ic, cond);
9529       visit_cf_list(ctx, &if_stmt->then_list);
9530
9531       begin_divergent_if_else(ctx, &ic);
9532       visit_cf_list(ctx, &if_stmt->else_list);
9533
9534       end_divergent_if(ctx, &ic);
9535
9536       return true;
9537    }
9538 }
9539
9540 static bool visit_cf_list(isel_context *ctx,
9541                           struct exec_list *list)
9542 {
9543    foreach_list_typed(nir_cf_node, node, node, list) {
9544       switch (node->type) {
9545       case nir_cf_node_block:
9546          visit_block(ctx, nir_cf_node_as_block(node));
9547          break;
9548       case nir_cf_node_if:
9549          if (!visit_if(ctx, nir_cf_node_as_if(node)))
9550             return true;
9551          break;
9552       case nir_cf_node_loop:
9553          visit_loop(ctx, nir_cf_node_as_loop(node));
9554          break;
9555       default:
9556          unreachable("unimplemented cf list type");
9557       }
9558    }
9559    return false;
9560 }
9561
9562 static void create_null_export(isel_context *ctx)
9563 {
9564    /* Some shader stages always need to have exports.
9565     * So when there is none, we need to add a null export.
9566     */
9567
9568    unsigned dest = (ctx->program->stage & hw_fs) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS;
9569    bool vm = (ctx->program->stage & hw_fs) || ctx->program->chip_class >= GFX10;
9570    Builder bld(ctx->program, ctx->block);
9571    bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
9572            /* enabled_mask */ 0, dest, /* compr */ false, /* done */ true, vm);
9573 }
9574
9575 static bool export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
9576 {
9577    assert(ctx->stage == vertex_vs ||
9578           ctx->stage == tess_eval_vs ||
9579           ctx->stage == gs_copy_vs ||
9580           ctx->stage == ngg_vertex_gs ||
9581           ctx->stage == ngg_tess_eval_gs);
9582
9583    int offset = (ctx->stage & sw_tes)
9584                 ? ctx->program->info->tes.outinfo.vs_output_param_offset[slot]
9585                 : ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
9586    uint64_t mask = ctx->outputs.mask[slot];
9587    if (!is_pos && !mask)
9588       return false;
9589    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
9590       return false;
9591    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9592    exp->enabled_mask = mask;
9593    for (unsigned i = 0; i < 4; ++i) {
9594       if (mask & (1 << i))
9595          exp->operands[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9596       else
9597          exp->operands[i] = Operand(v1);
9598    }
9599    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
9600     * Setting valid_mask=1 prevents it and has no other effect.
9601     */
9602    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
9603    exp->done = false;
9604    exp->compressed = false;
9605    if (is_pos)
9606       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9607    else
9608       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
9609    ctx->block->instructions.emplace_back(std::move(exp));
9610
9611    return true;
9612 }
9613
9614 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
9615 {
9616    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9617    exp->enabled_mask = 0;
9618    for (unsigned i = 0; i < 4; ++i)
9619       exp->operands[i] = Operand(v1);
9620    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
9621       exp->operands[0] = Operand(ctx->outputs.temps[VARYING_SLOT_PSIZ * 4u]);
9622       exp->enabled_mask |= 0x1;
9623    }
9624    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
9625       exp->operands[2] = Operand(ctx->outputs.temps[VARYING_SLOT_LAYER * 4u]);
9626       exp->enabled_mask |= 0x4;
9627    }
9628    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
9629       if (ctx->options->chip_class < GFX9) {
9630          exp->operands[3] = Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]);
9631          exp->enabled_mask |= 0x8;
9632       } else {
9633          Builder bld(ctx->program, ctx->block);
9634
9635          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
9636                              Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]));
9637          if (exp->operands[2].isTemp())
9638             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
9639
9640          exp->operands[2] = Operand(out);
9641          exp->enabled_mask |= 0x4;
9642       }
9643    }
9644    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
9645    exp->done = false;
9646    exp->compressed = false;
9647    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9648    ctx->block->instructions.emplace_back(std::move(exp));
9649 }
9650
9651 static void create_export_phis(isel_context *ctx)
9652 {
9653    /* Used when exports are needed, but the output temps are defined in a preceding block.
9654     * This function will set up phis in order to access the outputs in the next block.
9655     */
9656
9657    assert(ctx->block->instructions.back()->opcode == aco_opcode::p_logical_start);
9658    aco_ptr<Instruction> logical_start = aco_ptr<Instruction>(ctx->block->instructions.back().release());
9659    ctx->block->instructions.pop_back();
9660
9661    Builder bld(ctx->program, ctx->block);
9662
9663    for (unsigned slot = 0; slot <= VARYING_SLOT_VAR31; ++slot) {
9664       uint64_t mask = ctx->outputs.mask[slot];
9665       for (unsigned i = 0; i < 4; ++i) {
9666          if (!(mask & (1 << i)))
9667             continue;
9668
9669          Temp old = ctx->outputs.temps[slot * 4 + i];
9670          Temp phi = bld.pseudo(aco_opcode::p_phi, bld.def(v1), old, Operand(v1));
9671          ctx->outputs.temps[slot * 4 + i] = phi;
9672       }
9673    }
9674
9675    bld.insert(std::move(logical_start));
9676 }
9677
9678 static void create_vs_exports(isel_context *ctx)
9679 {
9680    assert(ctx->stage == vertex_vs ||
9681           ctx->stage == tess_eval_vs ||
9682           ctx->stage == gs_copy_vs ||
9683           ctx->stage == ngg_vertex_gs ||
9684           ctx->stage == ngg_tess_eval_gs);
9685
9686    radv_vs_output_info *outinfo = (ctx->stage & sw_tes)
9687                                   ? &ctx->program->info->tes.outinfo
9688                                   : &ctx->program->info->vs.outinfo;
9689
9690    if (outinfo->export_prim_id && !(ctx->stage & hw_ngg_gs)) {
9691       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
9692       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = get_arg(ctx, ctx->args->vs_prim_id);
9693    }
9694
9695    if (ctx->options->key.has_multiview_view_index) {
9696       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
9697       ctx->outputs.temps[VARYING_SLOT_LAYER * 4u] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
9698    }
9699
9700    /* the order these position exports are created is important */
9701    int next_pos = 0;
9702    bool exported_pos = export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
9703    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
9704       export_vs_psiz_layer_viewport(ctx, &next_pos);
9705       exported_pos = true;
9706    }
9707    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9708       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
9709    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9710       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
9711
9712    if (ctx->export_clip_dists) {
9713       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9714          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
9715       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9716          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
9717    }
9718
9719    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9720       if (i < VARYING_SLOT_VAR0 &&
9721           i != VARYING_SLOT_LAYER &&
9722           i != VARYING_SLOT_PRIMITIVE_ID)
9723          continue;
9724
9725       export_vs_varying(ctx, i, false, NULL);
9726    }
9727
9728    if (!exported_pos)
9729       create_null_export(ctx);
9730 }
9731
9732 static bool export_fs_mrt_z(isel_context *ctx)
9733 {
9734    Builder bld(ctx->program, ctx->block);
9735    unsigned enabled_channels = 0;
9736    bool compr = false;
9737    Operand values[4];
9738
9739    for (unsigned i = 0; i < 4; ++i) {
9740       values[i] = Operand(v1);
9741    }
9742
9743    /* Both stencil and sample mask only need 16-bits. */
9744    if (!ctx->program->info->ps.writes_z &&
9745        (ctx->program->info->ps.writes_stencil ||
9746         ctx->program->info->ps.writes_sample_mask)) {
9747       compr = true; /* COMPR flag */
9748
9749       if (ctx->program->info->ps.writes_stencil) {
9750          /* Stencil should be in X[23:16]. */
9751          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9752          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
9753          enabled_channels |= 0x3;
9754       }
9755
9756       if (ctx->program->info->ps.writes_sample_mask) {
9757          /* SampleMask should be in Y[15:0]. */
9758          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9759          enabled_channels |= 0xc;
9760      }
9761    } else {
9762       if (ctx->program->info->ps.writes_z) {
9763          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]);
9764          enabled_channels |= 0x1;
9765       }
9766
9767       if (ctx->program->info->ps.writes_stencil) {
9768          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9769          enabled_channels |= 0x2;
9770       }
9771
9772       if (ctx->program->info->ps.writes_sample_mask) {
9773          values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9774          enabled_channels |= 0x4;
9775       }
9776    }
9777
9778    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
9779     * writemask component.
9780     */
9781    if (ctx->options->chip_class == GFX6 &&
9782        ctx->options->family != CHIP_OLAND &&
9783        ctx->options->family != CHIP_HAINAN) {
9784             enabled_channels |= 0x1;
9785    }
9786
9787    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9788            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
9789
9790    return true;
9791 }
9792
9793 static bool export_fs_mrt_color(isel_context *ctx, int slot)
9794 {
9795    Builder bld(ctx->program, ctx->block);
9796    unsigned write_mask = ctx->outputs.mask[slot];
9797    Operand values[4];
9798
9799    for (unsigned i = 0; i < 4; ++i) {
9800       if (write_mask & (1 << i)) {
9801          values[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9802       } else {
9803          values[i] = Operand(v1);
9804       }
9805    }
9806
9807    unsigned target, col_format;
9808    unsigned enabled_channels = 0;
9809    aco_opcode compr_op = (aco_opcode)0;
9810
9811    slot -= FRAG_RESULT_DATA0;
9812    target = V_008DFC_SQ_EXP_MRT + slot;
9813    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
9814
9815    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
9816    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
9817
9818    switch (col_format)
9819    {
9820    case V_028714_SPI_SHADER_ZERO:
9821       enabled_channels = 0; /* writemask */
9822       target = V_008DFC_SQ_EXP_NULL;
9823       break;
9824
9825    case V_028714_SPI_SHADER_32_R:
9826       enabled_channels = 1;
9827       break;
9828
9829    case V_028714_SPI_SHADER_32_GR:
9830       enabled_channels = 0x3;
9831       break;
9832
9833    case V_028714_SPI_SHADER_32_AR:
9834       if (ctx->options->chip_class >= GFX10) {
9835          /* Special case: on GFX10, the outputs are different for 32_AR */
9836          enabled_channels = 0x3;
9837          values[1] = values[3];
9838          values[3] = Operand(v1);
9839       } else {
9840          enabled_channels = 0x9;
9841       }
9842       break;
9843
9844    case V_028714_SPI_SHADER_FP16_ABGR:
9845       enabled_channels = 0x5;
9846       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
9847       break;
9848
9849    case V_028714_SPI_SHADER_UNORM16_ABGR:
9850       enabled_channels = 0x5;
9851       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
9852       break;
9853
9854    case V_028714_SPI_SHADER_SNORM16_ABGR:
9855       enabled_channels = 0x5;
9856       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
9857       break;
9858
9859    case V_028714_SPI_SHADER_UINT16_ABGR: {
9860       enabled_channels = 0x5;
9861       compr_op = aco_opcode::v_cvt_pk_u16_u32;
9862       if (is_int8 || is_int10) {
9863          /* clamp */
9864          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
9865          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9866
9867          for (unsigned i = 0; i < 4; i++) {
9868             if ((write_mask >> i) & 1) {
9869                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
9870                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
9871                                     values[i]);
9872             }
9873          }
9874       }
9875       break;
9876    }
9877
9878    case V_028714_SPI_SHADER_SINT16_ABGR:
9879       enabled_channels = 0x5;
9880       compr_op = aco_opcode::v_cvt_pk_i16_i32;
9881       if (is_int8 || is_int10) {
9882          /* clamp */
9883          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
9884          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
9885          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9886          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
9887
9888          for (unsigned i = 0; i < 4; i++) {
9889             if ((write_mask >> i) & 1) {
9890                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
9891                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
9892                                     values[i]);
9893                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
9894                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
9895                                     values[i]);
9896             }
9897          }
9898       }
9899       break;
9900
9901    case V_028714_SPI_SHADER_32_ABGR:
9902       enabled_channels = 0xF;
9903       break;
9904
9905    default:
9906       break;
9907    }
9908
9909    if (target == V_008DFC_SQ_EXP_NULL)
9910       return false;
9911
9912    if ((bool) compr_op) {
9913       for (int i = 0; i < 2; i++) {
9914          /* check if at least one of the values to be compressed is enabled */
9915          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
9916          if (enabled) {
9917             enabled_channels |= enabled << (i*2);
9918             values[i] = bld.vop3(compr_op, bld.def(v1),
9919                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
9920                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
9921          } else {
9922             values[i] = Operand(v1);
9923          }
9924       }
9925       values[2] = Operand(v1);
9926       values[3] = Operand(v1);
9927    } else {
9928       for (int i = 0; i < 4; i++)
9929          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
9930    }
9931
9932    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9933            enabled_channels, target, (bool) compr_op);
9934    return true;
9935 }
9936
9937 static void create_fs_exports(isel_context *ctx)
9938 {
9939    bool exported = false;
9940
9941    /* Export depth, stencil and sample mask. */
9942    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
9943        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
9944        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK])
9945       exported |= export_fs_mrt_z(ctx);
9946
9947    /* Export all color render targets. */
9948    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i)
9949       if (ctx->outputs.mask[i])
9950          exported |= export_fs_mrt_color(ctx, i);
9951
9952    if (!exported)
9953       create_null_export(ctx);
9954 }
9955
9956 static void write_tcs_tess_factors(isel_context *ctx)
9957 {
9958    unsigned outer_comps;
9959    unsigned inner_comps;
9960
9961    switch (ctx->args->options->key.tcs.primitive_mode) {
9962    case GL_ISOLINES:
9963       outer_comps = 2;
9964       inner_comps = 0;
9965       break;
9966    case GL_TRIANGLES:
9967       outer_comps = 3;
9968       inner_comps = 1;
9969       break;
9970    case GL_QUADS:
9971       outer_comps = 4;
9972       inner_comps = 2;
9973       break;
9974    default:
9975       return;
9976    }
9977
9978    Builder bld(ctx->program, ctx->block);
9979
9980    bld.barrier(aco_opcode::p_memory_barrier_shared);
9981    if (unlikely(ctx->program->chip_class != GFX6 && ctx->program->workgroup_size > ctx->program->wave_size))
9982       bld.sopp(aco_opcode::s_barrier);
9983
9984    Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids);
9985    Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u));
9986
9987    Temp invocation_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), invocation_id);
9988    if_context ic_invocation_id_is_zero;
9989    begin_divergent_if_then(ctx, &ic_invocation_id_is_zero, invocation_id_is_zero);
9990    bld.reset(ctx->block);
9991
9992    Temp hs_ring_tess_factor = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_FACTOR * 16u));
9993
9994    std::pair<Temp, unsigned> lds_base = get_tcs_output_lds_offset(ctx);
9995    unsigned stride = inner_comps + outer_comps;
9996    unsigned lds_align = calculate_lds_alignment(ctx, lds_base.second);
9997    Temp tf_inner_vec;
9998    Temp tf_outer_vec;
9999    Temp out[6];
10000    assert(stride <= (sizeof(out) / sizeof(Temp)));
10001
10002    if (ctx->args->options->key.tcs.primitive_mode == GL_ISOLINES) {
10003       // LINES reversal
10004       tf_outer_vec = load_lds(ctx, 4, bld.tmp(v2), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10005       out[1] = emit_extract_vector(ctx, tf_outer_vec, 0, v1);
10006       out[0] = emit_extract_vector(ctx, tf_outer_vec, 1, v1);
10007    } else {
10008       tf_outer_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, outer_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10009       tf_inner_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, inner_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_in_loc, lds_align);
10010
10011       for (unsigned i = 0; i < outer_comps; ++i)
10012          out[i] = emit_extract_vector(ctx, tf_outer_vec, i, v1);
10013       for (unsigned i = 0; i < inner_comps; ++i)
10014          out[outer_comps + i] = emit_extract_vector(ctx, tf_inner_vec, i, v1);
10015    }
10016
10017    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
10018    Temp tf_base = get_arg(ctx, ctx->args->tess_factor_offset);
10019    Temp byte_offset = bld.v_mul_imm(bld.def(v1), rel_patch_id, stride * 4u);
10020    unsigned tf_const_offset = 0;
10021
10022    if (ctx->program->chip_class <= GFX8) {
10023       Temp rel_patch_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), rel_patch_id);
10024       if_context ic_rel_patch_id_is_zero;
10025       begin_divergent_if_then(ctx, &ic_rel_patch_id_is_zero, rel_patch_id_is_zero);
10026       bld.reset(ctx->block);
10027
10028       /* Store the dynamic HS control word. */
10029       Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u));
10030       bld.mubuf(aco_opcode::buffer_store_dword,
10031                 /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word,
10032                 /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false,
10033                 /* disable_wqm */ false, /* glc */ true);
10034       tf_const_offset += 4;
10035
10036       begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero);
10037       end_divergent_if(ctx, &ic_rel_patch_id_is_zero);
10038       bld.reset(ctx->block);
10039    }
10040
10041    assert(stride == 2 || stride == 4 || stride == 6);
10042    Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr, 4u);
10043    store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false);
10044
10045    /* Store to offchip for TES to read - only if TES reads them */
10046    if (ctx->args->options->key.tcs.tes_reads_tess_factors) {
10047       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
10048       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
10049
10050       std::pair<Temp, unsigned> vmem_offs_outer = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_out_loc);
10051       store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, false);
10052
10053       if (likely(inner_comps)) {
10054          std::pair<Temp, unsigned> vmem_offs_inner = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_in_loc);
10055          store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, false);
10056       }
10057    }
10058
10059    begin_divergent_if_else(ctx, &ic_invocation_id_is_zero);
10060    end_divergent_if(ctx, &ic_invocation_id_is_zero);
10061 }
10062
10063 static void emit_stream_output(isel_context *ctx,
10064                                Temp const *so_buffers,
10065                                Temp const *so_write_offset,
10066                                const struct radv_stream_output *output)
10067 {
10068    unsigned num_comps = util_bitcount(output->component_mask);
10069    unsigned writemask = (1 << num_comps) - 1;
10070    unsigned loc = output->location;
10071    unsigned buf = output->buffer;
10072
10073    assert(num_comps && num_comps <= 4);
10074    if (!num_comps || num_comps > 4)
10075       return;
10076
10077    unsigned start = ffs(output->component_mask) - 1;
10078
10079    Temp out[4];
10080    bool all_undef = true;
10081    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
10082    for (unsigned i = 0; i < num_comps; i++) {
10083       out[i] = ctx->outputs.temps[loc * 4 + start + i];
10084       all_undef = all_undef && !out[i].id();
10085    }
10086    if (all_undef)
10087       return;
10088
10089    while (writemask) {
10090       int start, count;
10091       u_bit_scan_consecutive_range(&writemask, &start, &count);
10092       if (count == 3 && ctx->options->chip_class == GFX6) {
10093          /* GFX6 doesn't support storing vec3, split it. */
10094          writemask |= 1u << (start + 2);
10095          count = 2;
10096       }
10097
10098       unsigned offset = output->offset + start * 4;
10099
10100       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
10101       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
10102       for (int i = 0; i < count; ++i)
10103          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
10104       vec->definitions[0] = Definition(write_data);
10105       ctx->block->instructions.emplace_back(std::move(vec));
10106
10107       aco_opcode opcode;
10108       switch (count) {
10109       case 1:
10110          opcode = aco_opcode::buffer_store_dword;
10111          break;
10112       case 2:
10113          opcode = aco_opcode::buffer_store_dwordx2;
10114          break;
10115       case 3:
10116          opcode = aco_opcode::buffer_store_dwordx3;
10117          break;
10118       case 4:
10119          opcode = aco_opcode::buffer_store_dwordx4;
10120          break;
10121       default:
10122          unreachable("Unsupported dword count.");
10123       }
10124
10125       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
10126       store->operands[0] = Operand(so_buffers[buf]);
10127       store->operands[1] = Operand(so_write_offset[buf]);
10128       store->operands[2] = Operand((uint32_t) 0);
10129       store->operands[3] = Operand(write_data);
10130       if (offset > 4095) {
10131          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
10132          Builder bld(ctx->program, ctx->block);
10133          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
10134       } else {
10135          store->offset = offset;
10136       }
10137       store->offen = true;
10138       store->glc = true;
10139       store->dlc = false;
10140       store->slc = true;
10141       store->can_reorder = true;
10142       ctx->block->instructions.emplace_back(std::move(store));
10143    }
10144 }
10145
10146 static void emit_streamout(isel_context *ctx, unsigned stream)
10147 {
10148    Builder bld(ctx->program, ctx->block);
10149
10150    Temp so_buffers[4];
10151    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
10152    for (unsigned i = 0; i < 4; i++) {
10153       unsigned stride = ctx->program->info->so.strides[i];
10154       if (!stride)
10155          continue;
10156
10157       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
10158       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
10159    }
10160
10161    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10162                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
10163
10164    Temp tid = emit_mbcnt(ctx, bld.def(v1));
10165
10166    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
10167
10168    if_context ic;
10169    begin_divergent_if_then(ctx, &ic, can_emit);
10170
10171    bld.reset(ctx->block);
10172
10173    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
10174
10175    Temp so_write_offset[4];
10176
10177    for (unsigned i = 0; i < 4; i++) {
10178       unsigned stride = ctx->program->info->so.strides[i];
10179       if (!stride)
10180          continue;
10181
10182       if (stride == 1) {
10183          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
10184                                 get_arg(ctx, ctx->args->streamout_write_idx),
10185                                 get_arg(ctx, ctx->args->streamout_offset[i]));
10186          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
10187
10188          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
10189       } else {
10190          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
10191          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
10192                                  get_arg(ctx, ctx->args->streamout_offset[i]));
10193          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
10194       }
10195    }
10196
10197    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
10198       struct radv_stream_output *output =
10199          &ctx->program->info->so.outputs[i];
10200       if (stream != output->stream)
10201          continue;
10202
10203       emit_stream_output(ctx, so_buffers, so_write_offset, output);
10204    }
10205
10206    begin_divergent_if_else(ctx, &ic);
10207    end_divergent_if(ctx, &ic);
10208 }
10209
10210 } /* end namespace */
10211
10212 void fix_ls_vgpr_init_bug(isel_context *ctx, Pseudo_instruction *startpgm)
10213 {
10214    assert(ctx->shader->info.stage == MESA_SHADER_VERTEX);
10215    Builder bld(ctx->program, ctx->block);
10216    constexpr unsigned hs_idx = 1u;
10217    Builder::Result hs_thread_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10218                                               get_arg(ctx, ctx->args->merged_wave_info),
10219                                               Operand((8u << 16) | (hs_idx * 8u)));
10220    Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp());
10221
10222    /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */
10223
10224    Temp instance_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10225                                get_arg(ctx, ctx->args->rel_auto_id),
10226                                get_arg(ctx, ctx->args->ac.instance_id),
10227                                ls_has_nonzero_hs_threads);
10228    Temp rel_auto_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10229                                get_arg(ctx, ctx->args->ac.tcs_rel_ids),
10230                                get_arg(ctx, ctx->args->rel_auto_id),
10231                                ls_has_nonzero_hs_threads);
10232    Temp vertex_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10233                              get_arg(ctx, ctx->args->ac.tcs_patch_id),
10234                              get_arg(ctx, ctx->args->ac.vertex_id),
10235                              ls_has_nonzero_hs_threads);
10236
10237    ctx->arg_temps[ctx->args->ac.instance_id.arg_index] = instance_id;
10238    ctx->arg_temps[ctx->args->rel_auto_id.arg_index] = rel_auto_id;
10239    ctx->arg_temps[ctx->args->ac.vertex_id.arg_index] = vertex_id;
10240 }
10241
10242 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
10243 {
10244    /* Split all arguments except for the first (ring_offsets) and the last
10245     * (exec) so that the dead channels don't stay live throughout the program.
10246     */
10247    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
10248       if (startpgm->definitions[i].regClass().size() > 1) {
10249          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
10250                            startpgm->definitions[i].regClass().size());
10251       }
10252    }
10253 }
10254
10255 void handle_bc_optimize(isel_context *ctx)
10256 {
10257    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
10258    Builder bld(ctx->program, ctx->block);
10259    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
10260    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
10261    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
10262    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
10263    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
10264    if (uses_center && uses_centroid) {
10265       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
10266                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
10267
10268       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
10269          Temp new_coord[2];
10270          for (unsigned i = 0; i < 2; i++) {
10271             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
10272             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
10273             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10274                                     persp_centroid, persp_center, sel);
10275          }
10276          ctx->persp_centroid = bld.tmp(v2);
10277          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
10278                     Operand(new_coord[0]), Operand(new_coord[1]));
10279          emit_split_vector(ctx, ctx->persp_centroid, 2);
10280       }
10281
10282       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
10283          Temp new_coord[2];
10284          for (unsigned i = 0; i < 2; i++) {
10285             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
10286             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
10287             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10288                                     linear_centroid, linear_center, sel);
10289          }
10290          ctx->linear_centroid = bld.tmp(v2);
10291          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
10292                     Operand(new_coord[0]), Operand(new_coord[1]));
10293          emit_split_vector(ctx, ctx->linear_centroid, 2);
10294       }
10295    }
10296 }
10297
10298 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
10299 {
10300    Program *program = ctx->program;
10301
10302    unsigned float_controls = shader->info.float_controls_execution_mode;
10303
10304    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
10305       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
10306    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
10307       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
10308                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
10309
10310    program->next_fp_mode.must_flush_denorms32 =
10311       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
10312    program->next_fp_mode.must_flush_denorms16_64 =
10313       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
10314                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
10315
10316    program->next_fp_mode.care_about_round32 =
10317       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
10318
10319    program->next_fp_mode.care_about_round16_64 =
10320       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
10321                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
10322
10323    /* default to preserving fp16 and fp64 denorms, since it's free */
10324    if (program->next_fp_mode.must_flush_denorms16_64)
10325       program->next_fp_mode.denorm16_64 = 0;
10326    else
10327       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10328
10329    /* preserving fp32 denorms is expensive, so only do it if asked */
10330    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
10331       program->next_fp_mode.denorm32 = fp_denorm_keep;
10332    else
10333       program->next_fp_mode.denorm32 = 0;
10334
10335    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
10336       program->next_fp_mode.round32 = fp_round_tz;
10337    else
10338       program->next_fp_mode.round32 = fp_round_ne;
10339
10340    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
10341       program->next_fp_mode.round16_64 = fp_round_tz;
10342    else
10343       program->next_fp_mode.round16_64 = fp_round_ne;
10344
10345    ctx->block->fp_mode = program->next_fp_mode;
10346 }
10347
10348 void cleanup_cfg(Program *program)
10349 {
10350    /* create linear_succs/logical_succs */
10351    for (Block& BB : program->blocks) {
10352       for (unsigned idx : BB.linear_preds)
10353          program->blocks[idx].linear_succs.emplace_back(BB.index);
10354       for (unsigned idx : BB.logical_preds)
10355          program->blocks[idx].logical_succs.emplace_back(BB.index);
10356    }
10357 }
10358
10359 Temp merged_wave_info_to_mask(isel_context *ctx, unsigned i)
10360 {
10361    Builder bld(ctx->program, ctx->block);
10362
10363    /* The s_bfm only cares about s0.u[5:0] so we don't need either s_bfe nor s_and here */
10364    Temp count = i == 0
10365                 ? get_arg(ctx, ctx->args->merged_wave_info)
10366                 : bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc),
10367                            get_arg(ctx, ctx->args->merged_wave_info), Operand(i * 8u));
10368
10369    Temp mask = bld.sop2(aco_opcode::s_bfm_b64, bld.def(s2), count, Operand(0u));
10370    Temp cond;
10371
10372    if (ctx->program->wave_size == 64) {
10373       /* Special case for 64 active invocations, because 64 doesn't work with s_bfm */
10374       Temp active_64 = bld.sopc(aco_opcode::s_bitcmp1_b32, bld.def(s1, scc), count, Operand(6u /* log2(64) */));
10375       cond = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), mask, bld.scc(active_64));
10376    } else {
10377       /* We use s_bfm_b64 (not _b32) which works with 32, but we need to extract the lower half of the register */
10378       cond = emit_extract_vector(ctx, mask, 0, bld.lm);
10379    }
10380
10381    return cond;
10382 }
10383
10384 bool ngg_early_prim_export(isel_context *ctx)
10385 {
10386    /* TODO: Check edge flags, and if they are written, return false. (Needed for OpenGL, not for Vulkan.) */
10387    return true;
10388 }
10389
10390 void ngg_emit_sendmsg_gs_alloc_req(isel_context *ctx)
10391 {
10392    Builder bld(ctx->program, ctx->block);
10393
10394    /* It is recommended to do the GS_ALLOC_REQ as soon and as quickly as possible, so we set the maximum priority (3). */
10395    bld.sopp(aco_opcode::s_setprio, -1u, 0x3u);
10396
10397    /* Get the id of the current wave within the threadgroup (workgroup) */
10398    Builder::Result wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10399                                             get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10400
10401    /* Execute the following code only on the first wave (wave id 0),
10402     * use the SCC def to tell if the wave id is zero or not.
10403     */
10404    Temp cond = wave_id_in_tg.def(1).getTemp();
10405    if_context ic;
10406    begin_uniform_if_then(ctx, &ic, cond);
10407    begin_uniform_if_else(ctx, &ic);
10408    bld.reset(ctx->block);
10409
10410    /* Number of vertices output by VS/TES */
10411    Temp vtx_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10412                            get_arg(ctx, ctx->args->gs_tg_info), Operand(12u | (9u << 16u)));
10413    /* Number of primitives output by VS/TES */
10414    Temp prm_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10415                            get_arg(ctx, ctx->args->gs_tg_info), Operand(22u | (9u << 16u)));
10416
10417    /* Put the number of vertices and primitives into m0 for the GS_ALLOC_REQ */
10418    Temp tmp = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), prm_cnt, Operand(12u));
10419    tmp = bld.sop2(aco_opcode::s_or_b32, bld.m0(bld.def(s1)), bld.def(s1, scc), tmp, vtx_cnt);
10420
10421    /* Request the SPI to allocate space for the primitives and vertices that will be exported by the threadgroup. */
10422    bld.sopp(aco_opcode::s_sendmsg, bld.m0(tmp), -1, sendmsg_gs_alloc_req);
10423
10424    /* After the GS_ALLOC_REQ is done, reset priority to default (0). */
10425    bld.sopp(aco_opcode::s_setprio, -1u, 0x0u);
10426
10427    end_uniform_if(ctx, &ic);
10428 }
10429
10430 Temp ngg_get_prim_exp_arg(isel_context *ctx, unsigned num_vertices, const Temp vtxindex[])
10431 {
10432    Builder bld(ctx->program, ctx->block);
10433
10434    if (ctx->args->options->key.vs_common_out.as_ngg_passthrough) {
10435       return get_arg(ctx, ctx->args->gs_vtx_offset[0]);
10436    }
10437
10438    Temp gs_invocation_id = get_arg(ctx, ctx->args->ac.gs_invocation_id);
10439    Temp tmp;
10440
10441    for (unsigned i = 0; i < num_vertices; ++i) {
10442       assert(vtxindex[i].id());
10443
10444       if (i)
10445          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), vtxindex[i], Operand(10u * i), tmp);
10446       else
10447          tmp = vtxindex[i];
10448
10449       /* The initial edge flag is always false in tess eval shaders. */
10450       if (ctx->stage == ngg_vertex_gs) {
10451          Temp edgeflag = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), gs_invocation_id, Operand(8 + i), Operand(1u));
10452          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), edgeflag, Operand(10u * i + 9u), tmp);
10453       }
10454    }
10455
10456    /* TODO: Set isnull field in case of merged NGG VS+GS. */
10457
10458    return tmp;
10459 }
10460
10461 void ngg_emit_prim_export(isel_context *ctx, unsigned num_vertices_per_primitive, const Temp vtxindex[])
10462 {
10463    Builder bld(ctx->program, ctx->block);
10464    Temp prim_exp_arg = ngg_get_prim_exp_arg(ctx, num_vertices_per_primitive, vtxindex);
10465
10466    bld.exp(aco_opcode::exp, prim_exp_arg, Operand(v1), Operand(v1), Operand(v1),
10467         1 /* enabled mask */, V_008DFC_SQ_EXP_PRIM /* dest */,
10468         false /* compressed */, true/* done */, false /* valid mask */);
10469 }
10470
10471 void ngg_emit_nogs_gsthreads(isel_context *ctx)
10472 {
10473    /* Emit the things that NGG GS threads need to do, for shaders that don't have SW GS.
10474     * These must always come before VS exports.
10475     *
10476     * It is recommended to do these as early as possible. They can be at the beginning when
10477     * there is no SW GS and the shader doesn't write edge flags.
10478     */
10479
10480    if_context ic;
10481    Temp is_gs_thread = merged_wave_info_to_mask(ctx, 1);
10482    begin_divergent_if_then(ctx, &ic, is_gs_thread);
10483
10484    Builder bld(ctx->program, ctx->block);
10485    constexpr unsigned max_vertices_per_primitive = 3;
10486    unsigned num_vertices_per_primitive = max_vertices_per_primitive;
10487
10488    if (ctx->stage == ngg_vertex_gs) {
10489       /* TODO: optimize for points & lines */
10490    } else if (ctx->stage == ngg_tess_eval_gs) {
10491       if (ctx->shader->info.tess.point_mode)
10492          num_vertices_per_primitive = 1;
10493       else if (ctx->shader->info.tess.primitive_mode == GL_ISOLINES)
10494          num_vertices_per_primitive = 2;
10495    } else {
10496       unreachable("Unsupported NGG shader stage");
10497    }
10498
10499    Temp vtxindex[max_vertices_per_primitive];
10500    vtxindex[0] = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10501                           get_arg(ctx, ctx->args->gs_vtx_offset[0]));
10502    vtxindex[1] = num_vertices_per_primitive < 2 ? Temp(0, v1) :
10503                  bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
10504                           get_arg(ctx, ctx->args->gs_vtx_offset[0]), Operand(16u), Operand(16u));
10505    vtxindex[2] = num_vertices_per_primitive < 3 ? Temp(0, v1) :
10506                  bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10507                           get_arg(ctx, ctx->args->gs_vtx_offset[2]));
10508
10509    /* Export primitive data to the index buffer. */
10510    ngg_emit_prim_export(ctx, num_vertices_per_primitive, vtxindex);
10511
10512    /* Export primitive ID. */
10513    if (ctx->stage == ngg_vertex_gs && ctx->args->options->key.vs_common_out.export_prim_id) {
10514       /* Copy Primitive IDs from GS threads to the LDS address corresponding to the ES thread of the provoking vertex. */
10515       Temp prim_id = get_arg(ctx, ctx->args->ac.gs_prim_id);
10516       Temp provoking_vtx_index = vtxindex[0];
10517       Temp addr = bld.v_mul_imm(bld.def(v1), provoking_vtx_index, 4u);
10518
10519       store_lds(ctx, 4, prim_id, 0x1u, addr, 0u, 4u);
10520    }
10521
10522    begin_divergent_if_else(ctx, &ic);
10523    end_divergent_if(ctx, &ic);
10524 }
10525
10526 void ngg_emit_nogs_output(isel_context *ctx)
10527 {
10528    /* Emits NGG GS output, for stages that don't have SW GS. */
10529
10530    if_context ic;
10531    Builder bld(ctx->program, ctx->block);
10532    bool late_prim_export = !ngg_early_prim_export(ctx);
10533
10534    /* NGG streamout is currently disabled by default. */
10535    assert(!ctx->args->shader_info->so.num_outputs);
10536
10537    if (late_prim_export) {
10538       /* VS exports are output to registers in a predecessor block. Emit phis to get them into this block. */
10539       create_export_phis(ctx);
10540       /* Do what we need to do in the GS threads. */
10541       ngg_emit_nogs_gsthreads(ctx);
10542
10543       /* What comes next should be executed on ES threads. */
10544       Temp is_es_thread = merged_wave_info_to_mask(ctx, 0);
10545       begin_divergent_if_then(ctx, &ic, is_es_thread);
10546       bld.reset(ctx->block);
10547    }
10548
10549    /* Export VS outputs */
10550    ctx->block->kind |= block_kind_export_end;
10551    create_vs_exports(ctx);
10552
10553    /* Export primitive ID */
10554    if (ctx->args->options->key.vs_common_out.export_prim_id) {
10555       Temp prim_id;
10556
10557       if (ctx->stage == ngg_vertex_gs) {
10558          /* Wait for GS threads to store primitive ID in LDS. */
10559          bld.barrier(aco_opcode::p_memory_barrier_shared);
10560          bld.sopp(aco_opcode::s_barrier);
10561
10562          /* Calculate LDS address where the GS threads stored the primitive ID. */
10563          Temp wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10564                                        get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10565          Temp thread_id_in_wave = emit_mbcnt(ctx, bld.def(v1));
10566          Temp wave_id_mul = bld.v_mul_imm(bld.def(v1), as_vgpr(ctx, wave_id_in_tg), ctx->program->wave_size);
10567          Temp thread_id_in_tg = bld.vadd32(bld.def(v1), Operand(wave_id_mul), Operand(thread_id_in_wave));
10568          Temp addr = bld.v_mul_imm(bld.def(v1), thread_id_in_tg, 4u);
10569
10570          /* Load primitive ID from LDS. */
10571          prim_id = load_lds(ctx, 4, bld.tmp(v1), addr, 0u, 4u);
10572       } else if (ctx->stage == ngg_tess_eval_gs) {
10573          /* TES: Just use the patch ID as the primitive ID. */
10574          prim_id = get_arg(ctx, ctx->args->ac.tes_patch_id);
10575       } else {
10576          unreachable("unsupported NGG shader stage.");
10577       }
10578
10579       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
10580       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = prim_id;
10581
10582       export_vs_varying(ctx, VARYING_SLOT_PRIMITIVE_ID, false, nullptr);
10583    }
10584
10585    if (late_prim_export) {
10586       begin_divergent_if_else(ctx, &ic);
10587       end_divergent_if(ctx, &ic);
10588       bld.reset(ctx->block);
10589    }
10590 }
10591
10592 void select_program(Program *program,
10593                     unsigned shader_count,
10594                     struct nir_shader *const *shaders,
10595                     ac_shader_config* config,
10596                     struct radv_shader_args *args)
10597 {
10598    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
10599    if_context ic_merged_wave_info;
10600    bool ngg_no_gs = ctx.stage == ngg_vertex_gs || ctx.stage == ngg_tess_eval_gs;
10601
10602    for (unsigned i = 0; i < shader_count; i++) {
10603       nir_shader *nir = shaders[i];
10604       init_context(&ctx, nir);
10605
10606       setup_fp_mode(&ctx, nir);
10607
10608       if (!i) {
10609          /* needs to be after init_context() for FS */
10610          Pseudo_instruction *startpgm = add_startpgm(&ctx);
10611          append_logical_start(ctx.block);
10612
10613          if (unlikely(args->options->has_ls_vgpr_init_bug && ctx.stage == vertex_tess_control_hs))
10614             fix_ls_vgpr_init_bug(&ctx, startpgm);
10615
10616          split_arguments(&ctx, startpgm);
10617       }
10618
10619       if (ngg_no_gs) {
10620          ngg_emit_sendmsg_gs_alloc_req(&ctx);
10621
10622          if (ngg_early_prim_export(&ctx))
10623             ngg_emit_nogs_gsthreads(&ctx);
10624       }
10625
10626       /* In a merged VS+TCS HS, the VS implementation can be completely empty. */
10627       nir_function_impl *func = nir_shader_get_entrypoint(nir);
10628       bool empty_shader = nir_cf_list_is_empty_block(&func->body) &&
10629                           ((nir->info.stage == MESA_SHADER_VERTEX &&
10630                             (ctx.stage == vertex_tess_control_hs || ctx.stage == vertex_geometry_gs)) ||
10631                            (nir->info.stage == MESA_SHADER_TESS_EVAL &&
10632                             ctx.stage == tess_eval_geometry_gs));
10633
10634       bool check_merged_wave_info = ctx.tcs_in_out_eq ? i == 0 : ((shader_count >= 2 && !empty_shader) || ngg_no_gs);
10635       bool endif_merged_wave_info = ctx.tcs_in_out_eq ? i == 1 : check_merged_wave_info;
10636       if (check_merged_wave_info) {
10637          Temp cond = merged_wave_info_to_mask(&ctx, i);
10638          begin_divergent_if_then(&ctx, &ic_merged_wave_info, cond);
10639       }
10640
10641       if (i) {
10642          Builder bld(ctx.program, ctx.block);
10643
10644          bld.barrier(aco_opcode::p_memory_barrier_shared);
10645          bld.sopp(aco_opcode::s_barrier);
10646
10647          if (ctx.stage == vertex_geometry_gs || ctx.stage == tess_eval_geometry_gs) {
10648             ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
10649          }
10650       } else if (ctx.stage == geometry_gs)
10651          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
10652
10653       if (ctx.stage == fragment_fs)
10654          handle_bc_optimize(&ctx);
10655
10656       visit_cf_list(&ctx, &func->body);
10657
10658       if (ctx.program->info->so.num_outputs && (ctx.stage & hw_vs))
10659          emit_streamout(&ctx, 0);
10660
10661       if (ctx.stage & hw_vs) {
10662          create_vs_exports(&ctx);
10663          ctx.block->kind |= block_kind_export_end;
10664       } else if (ngg_no_gs && ngg_early_prim_export(&ctx)) {
10665          ngg_emit_nogs_output(&ctx);
10666       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
10667          Builder bld(ctx.program, ctx.block);
10668          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
10669          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
10670       } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
10671          write_tcs_tess_factors(&ctx);
10672       }
10673
10674       if (ctx.stage == fragment_fs) {
10675          create_fs_exports(&ctx);
10676          ctx.block->kind |= block_kind_export_end;
10677       }
10678
10679       if (endif_merged_wave_info) {
10680          begin_divergent_if_else(&ctx, &ic_merged_wave_info);
10681          end_divergent_if(&ctx, &ic_merged_wave_info);
10682       }
10683
10684       if (ngg_no_gs && !ngg_early_prim_export(&ctx))
10685          ngg_emit_nogs_output(&ctx);
10686
10687       ralloc_free(ctx.divergent_vals);
10688
10689       if (i == 0 && ctx.stage == vertex_tess_control_hs && ctx.tcs_in_out_eq) {
10690          /* Outputs of the previous stage are inputs to the next stage */
10691          ctx.inputs = ctx.outputs;
10692          ctx.outputs = shader_io_state();
10693       }
10694    }
10695
10696    program->config->float_mode = program->blocks[0].fp_mode.val;
10697
10698    append_logical_end(ctx.block);
10699    ctx.block->kind |= block_kind_uniform;
10700    Builder bld(ctx.program, ctx.block);
10701    if (ctx.program->wb_smem_l1_on_end)
10702       bld.smem(aco_opcode::s_dcache_wb, false);
10703    bld.sopp(aco_opcode::s_endpgm);
10704
10705    cleanup_cfg(program);
10706 }
10707
10708 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
10709                            ac_shader_config* config,
10710                            struct radv_shader_args *args)
10711 {
10712    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
10713
10714    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
10715    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
10716    program->next_fp_mode.must_flush_denorms32 = false;
10717    program->next_fp_mode.must_flush_denorms16_64 = false;
10718    program->next_fp_mode.care_about_round32 = false;
10719    program->next_fp_mode.care_about_round16_64 = false;
10720    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10721    program->next_fp_mode.denorm32 = 0;
10722    program->next_fp_mode.round32 = fp_round_ne;
10723    program->next_fp_mode.round16_64 = fp_round_ne;
10724    ctx.block->fp_mode = program->next_fp_mode;
10725
10726    add_startpgm(&ctx);
10727    append_logical_start(ctx.block);
10728
10729    Builder bld(ctx.program, ctx.block);
10730
10731    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
10732
10733    Operand stream_id(0u);
10734    if (args->shader_info->so.num_outputs)
10735       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10736                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
10737
10738    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
10739
10740    std::stack<Block> endif_blocks;
10741
10742    for (unsigned stream = 0; stream < 4; stream++) {
10743       if (stream_id.isConstant() && stream != stream_id.constantValue())
10744          continue;
10745
10746       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
10747       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
10748          continue;
10749
10750       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
10751
10752       unsigned BB_if_idx = ctx.block->index;
10753       Block BB_endif = Block();
10754       if (!stream_id.isConstant()) {
10755          /* begin IF */
10756          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
10757          append_logical_end(ctx.block);
10758          ctx.block->kind |= block_kind_uniform;
10759          bld.branch(aco_opcode::p_cbranch_z, cond);
10760
10761          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
10762
10763          ctx.block = ctx.program->create_and_insert_block();
10764          add_edge(BB_if_idx, ctx.block);
10765          bld.reset(ctx.block);
10766          append_logical_start(ctx.block);
10767       }
10768
10769       unsigned offset = 0;
10770       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
10771          if (args->shader_info->gs.output_streams[i] != stream)
10772             continue;
10773
10774          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
10775          unsigned length = util_last_bit(output_usage_mask);
10776          for (unsigned j = 0; j < length; ++j) {
10777             if (!(output_usage_mask & (1 << j)))
10778                continue;
10779
10780             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
10781             Temp voffset = vtx_offset;
10782             if (const_offset >= 4096u) {
10783                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
10784                const_offset %= 4096u;
10785             }
10786
10787             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
10788             mubuf->definitions[0] = bld.def(v1);
10789             mubuf->operands[0] = Operand(gsvs_ring);
10790             mubuf->operands[1] = Operand(voffset);
10791             mubuf->operands[2] = Operand(0u);
10792             mubuf->offen = true;
10793             mubuf->offset = const_offset;
10794             mubuf->glc = true;
10795             mubuf->slc = true;
10796             mubuf->dlc = args->options->chip_class >= GFX10;
10797             mubuf->barrier = barrier_none;
10798             mubuf->can_reorder = true;
10799
10800             ctx.outputs.mask[i] |= 1 << j;
10801             ctx.outputs.temps[i * 4u + j] = mubuf->definitions[0].getTemp();
10802
10803             bld.insert(std::move(mubuf));
10804
10805             offset++;
10806          }
10807       }
10808
10809       if (args->shader_info->so.num_outputs) {
10810          emit_streamout(&ctx, stream);
10811          bld.reset(ctx.block);
10812       }
10813
10814       if (stream == 0) {
10815          create_vs_exports(&ctx);
10816          ctx.block->kind |= block_kind_export_end;
10817       }
10818
10819       if (!stream_id.isConstant()) {
10820          append_logical_end(ctx.block);
10821
10822          /* branch from then block to endif block */
10823          bld.branch(aco_opcode::p_branch);
10824          add_edge(ctx.block->index, &BB_endif);
10825          ctx.block->kind |= block_kind_uniform;
10826
10827          /* emit else block */
10828          ctx.block = ctx.program->create_and_insert_block();
10829          add_edge(BB_if_idx, ctx.block);
10830          bld.reset(ctx.block);
10831          append_logical_start(ctx.block);
10832
10833          endif_blocks.push(std::move(BB_endif));
10834       }
10835    }
10836
10837    while (!endif_blocks.empty()) {
10838       Block BB_endif = std::move(endif_blocks.top());
10839       endif_blocks.pop();
10840
10841       Block *BB_else = ctx.block;
10842
10843       append_logical_end(BB_else);
10844       /* branch from else block to endif block */
10845       bld.branch(aco_opcode::p_branch);
10846       add_edge(BB_else->index, &BB_endif);
10847       BB_else->kind |= block_kind_uniform;
10848
10849       /** emit endif merge block */
10850       ctx.block = program->insert_block(std::move(BB_endif));
10851       bld.reset(ctx.block);
10852       append_logical_start(ctx.block);
10853    }
10854
10855    program->config->float_mode = program->blocks[0].fp_mode.val;
10856
10857    append_logical_end(ctx.block);
10858    ctx.block->kind |= block_kind_uniform;
10859    bld.sopp(aco_opcode::s_endpgm);
10860
10861    cleanup_cfg(program);
10862 }
10863 }