src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool uniform_has_then_branch;
  89    bool then_branch_divergent;
  90    Block BB_invert;
  91    Block BB_endif;
  92 };
  93
  94 static bool visit_cf_list(struct isel_context *ctx,
  95                           struct exec_list *list);
  96
  97 static void add_logical_edge(unsigned pred_idx, Block *succ)
  98 {
  99    succ->logical_preds.emplace_back(pred_idx);
 100 }
 101
 102
 103 static void add_linear_edge(unsigned pred_idx, Block *succ)
 104 {
 105    succ->linear_preds.emplace_back(pred_idx);
 106 }
 107
 108 static void add_edge(unsigned pred_idx, Block *succ)
 109 {
 110    add_logical_edge(pred_idx, succ);
 111    add_linear_edge(pred_idx, succ);
 112 }
 113
 114 static void append_logical_start(Block *b)
 115 {
 116    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 117 }
 118
 119 static void append_logical_end(Block *b)
 120 {
 121    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 122 }
 123
 124 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 125 {
 126    assert(ctx->allocated[def->index].id());
 127    return ctx->allocated[def->index];
 128 }
 129
 130 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 131                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 132 {
 133    Builder bld(ctx->program, ctx->block);
 134    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 135    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 136
 137    if (ctx->program->wave_size == 32) {
 138       return thread_id_lo;
 139    } else {
 140       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 141       return thread_id_hi;
 142    }
 143 }
 144
 145 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 146 {
 147    Builder bld(ctx->program, ctx->block);
 148
 149    if (!dst.id())
 150       dst = bld.tmp(src.regClass());
 151
 152    assert(src.size() == dst.size());
 153
 154    if (ctx->stage != fragment_fs) {
 155       if (!dst.id())
 156          return src;
 157
 158       bld.copy(Definition(dst), src);
 159       return dst;
 160    }
 161
 162    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 163    ctx->program->needs_wqm |= program_needs_wqm;
 164    return dst;
 165 }
 166
 167 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 168 {
 169    if (index.regClass() == s1)
 170       return bld.readlane(bld.def(s1), data, index);
 171
 172    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 173
 174    /* Currently not implemented on GFX6-7 */
 175    assert(ctx->options->chip_class >= GFX8);
 176
 177    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 178       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 179    }
 180
 181    /* GFX10, wave64 mode:
 182     * The bpermute instruction is limited to half-wave operation, which means that it can't
 183     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 184     * emulate it here.
 185     */
 186    if (!ctx->has_gfx10_wave64_bpermute) {
 187       ctx->has_gfx10_wave64_bpermute = true;
 188       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 189       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 190    }
 191
 192    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 193    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 194    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 195    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 196
 197    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 198                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 199 }
 200
 201 Temp as_vgpr(isel_context *ctx, Temp val)
 202 {
 203    if (val.type() == RegType::sgpr) {
 204       Builder bld(ctx->program, ctx->block);
 205       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 206    }
 207    assert(val.type() == RegType::vgpr);
 208    return val;
 209 }
 210
 211 //assumes a != 0xffffffff
 212 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 213 {
 214    assert(b != 0);
 215    Builder bld(ctx->program, ctx->block);
 216
 217    if (util_is_power_of_two_or_zero(b)) {
 218       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 219       return;
 220    }
 221
 222    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 223
 224    assert(info.multiplier <= 0xffffffff);
 225
 226    bool pre_shift = info.pre_shift != 0;
 227    bool increment = info.increment != 0;
 228    bool multiply = true;
 229    bool post_shift = info.post_shift != 0;
 230
 231    if (!pre_shift && !increment && !multiply && !post_shift) {
 232       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 233       return;
 234    }
 235
 236    Temp pre_shift_dst = a;
 237    if (pre_shift) {
 238       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 239       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 240    }
 241
 242    Temp increment_dst = pre_shift_dst;
 243    if (increment) {
 244       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 245       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 246    }
 247
 248    Temp multiply_dst = increment_dst;
 249    if (multiply) {
 250       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 251       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 252                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 253    }
 254
 255    if (post_shift) {
 256       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 257    }
 258 }
 259
 260 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 261 {
 262    Builder bld(ctx->program, ctx->block);
 263    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 264 }
 265
 266
 267 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 268 {
 269    /* no need to extract the whole vector */
 270    if (src.regClass() == dst_rc) {
 271       assert(idx == 0);
 272       return src;
 273    }
 274
 275    assert(src.bytes() > (idx * dst_rc.bytes()));
 276    Builder bld(ctx->program, ctx->block);
 277    auto it = ctx->allocated_vec.find(src.id());
 278    if (it != ctx->allocated_vec.end() && dst_rc.bytes() == it->second[idx].regClass().bytes()) {
 279       if (it->second[idx].regClass() == dst_rc) {
 280          return it->second[idx];
 281       } else {
 282          assert(!dst_rc.is_subdword());
 283          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 284          return bld.copy(bld.def(dst_rc), it->second[idx]);
 285       }
 286    }
 287
 288    if (dst_rc.is_subdword())
 289       src = as_vgpr(ctx, src);
 290
 291    if (src.bytes() == dst_rc.bytes()) {
 292       assert(idx == 0);
 293       return bld.copy(bld.def(dst_rc), src);
 294    } else {
 295       Temp dst = bld.tmp(dst_rc);
 296       emit_extract_vector(ctx, src, idx, dst);
 297       return dst;
 298    }
 299 }
 300
 301 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 302 {
 303    if (num_components == 1)
 304       return;
 305    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 306       return;
 307    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 308    split->operands[0] = Operand(vec_src);
 309    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 310    RegClass rc;
 311    if (num_components > vec_src.size()) {
 312       if (vec_src.type() == RegType::sgpr)
 313          return;
 314
 315       /* sub-dword split */
 316       assert(vec_src.type() == RegType::vgpr);
 317       rc = RegClass(RegType::vgpr, vec_src.bytes() / num_components).as_subdword();
 318    } else {
 319       rc = RegClass(vec_src.type(), vec_src.size() / num_components);
 320    }
 321    for (unsigned i = 0; i < num_components; i++) {
 322       elems[i] = {ctx->program->allocateId(), rc};
 323       split->definitions[i] = Definition(elems[i]);
 324    }
 325    ctx->block->instructions.emplace_back(std::move(split));
 326    ctx->allocated_vec.emplace(vec_src.id(), elems);
 327 }
 328
 329 /* This vector expansion uses a mask to determine which elements in the new vector
 330  * come from the original vector. The other elements are undefined. */
 331 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 332 {
 333    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 334
 335    if (vec_src == dst)
 336       return;
 337
 338    Builder bld(ctx->program, ctx->block);
 339    if (num_components == 1) {
 340       if (dst.type() == RegType::sgpr)
 341          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 342       else
 343          bld.copy(Definition(dst), vec_src);
 344       return;
 345    }
 346
 347    unsigned component_size = dst.size() / num_components;
 348    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 349
 350    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 351    vec->definitions[0] = Definition(dst);
 352    unsigned k = 0;
 353    for (unsigned i = 0; i < num_components; i++) {
 354       if (mask & (1 << i)) {
 355          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 356          if (dst.type() == RegType::sgpr)
 357             src = bld.as_uniform(src);
 358          vec->operands[i] = Operand(src);
 359       } else {
 360          vec->operands[i] = Operand(0u);
 361       }
 362       elems[i] = vec->operands[i].getTemp();
 363    }
 364    ctx->block->instructions.emplace_back(std::move(vec));
 365    ctx->allocated_vec.emplace(dst.id(), elems);
 366 }
 367
 368 /* adjust misaligned small bit size loads */
 369 void byte_align_scalar(isel_context *ctx, Temp vec, Operand offset, Temp dst)
 370 {
 371    Builder bld(ctx->program, ctx->block);
 372    Operand shift;
 373    Temp select = Temp();
 374    if (offset.isConstant()) {
 375       assert(offset.constantValue() && offset.constantValue() < 4);
 376       shift = Operand(offset.constantValue() * 8);
 377    } else {
 378       /* bit_offset = 8 * (offset & 0x3) */
 379       Temp tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), offset, Operand(3u));
 380       select = bld.tmp(s1);
 381       shift = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.scc(Definition(select)), tmp, Operand(3u));
 382    }
 383
 384    if (vec.size() == 1) {
 385       bld.sop2(aco_opcode::s_lshr_b32, Definition(dst), bld.def(s1, scc), vec, shift);
 386    } else if (vec.size() == 2) {
 387       Temp tmp = dst.size() == 2 ? dst : bld.tmp(s2);
 388       bld.sop2(aco_opcode::s_lshr_b64, Definition(tmp), bld.def(s1, scc), vec, shift);
 389       if (tmp == dst)
 390          emit_split_vector(ctx, dst, 2);
 391       else
 392          emit_extract_vector(ctx, tmp, 0, dst);
 393    } else if (vec.size() == 4) {
 394       Temp lo = bld.tmp(s2), hi = bld.tmp(s2);
 395       bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), vec);
 396       hi = bld.pseudo(aco_opcode::p_extract_vector, bld.def(s1), hi, Operand(0u));
 397       if (select != Temp())
 398          hi = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), hi, Operand(0u), select);
 399       lo = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), lo, shift);
 400       Temp mid = bld.tmp(s1);
 401       lo = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), Definition(mid), lo);
 402       hi = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), hi, shift);
 403       mid = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), hi, mid);
 404       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, mid);
 405       emit_split_vector(ctx, dst, 2);
 406    }
 407 }
 408
 409 /* this function trims subdword vectors:
 410  * if dst is vgpr - split the src and create a shrunk version according to the mask.
 411  * if dst is sgpr - split the src, but move the original to sgpr. */
 412 void trim_subdword_vector(isel_context *ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 413 {
 414    assert(vec_src.type() == RegType::vgpr);
 415    emit_split_vector(ctx, vec_src, num_components);
 416
 417    Builder bld(ctx->program, ctx->block);
 418    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 419    unsigned component_size = vec_src.bytes() / num_components;
 420    RegClass rc = RegClass(RegType::vgpr, component_size).as_subdword();
 421
 422    unsigned k = 0;
 423    for (unsigned i = 0; i < num_components; i++) {
 424       if (mask & (1 << i))
 425          elems[k++] = emit_extract_vector(ctx, vec_src, i, rc);
 426    }
 427
 428    if (dst.type() == RegType::vgpr) {
 429       assert(dst.bytes() == k * component_size);
 430       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, k, 1)};
 431       for (unsigned i = 0; i < k; i++)
 432          vec->operands[i] = Operand(elems[i]);
 433       vec->definitions[0] = Definition(dst);
 434       bld.insert(std::move(vec));
 435    } else {
 436       // TODO: alignbyte if mask doesn't start with 1?
 437       assert(mask & 1);
 438       assert(dst.size() == vec_src.size());
 439       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 440    }
 441    ctx->allocated_vec.emplace(dst.id(), elems);
 442 }
 443
 444 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 445 {
 446    Builder bld(ctx->program, ctx->block);
 447    if (!dst.id())
 448       dst = bld.tmp(bld.lm);
 449
 450    assert(val.regClass() == s1);
 451    assert(dst.regClass() == bld.lm);
 452
 453    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 454 }
 455
 456 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 457 {
 458    Builder bld(ctx->program, ctx->block);
 459    if (!dst.id())
 460       dst = bld.tmp(s1);
 461
 462    assert(val.regClass() == bld.lm);
 463    assert(dst.regClass() == s1);
 464
 465    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 466    Temp tmp = bld.tmp(s1);
 467    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 468    return emit_wqm(ctx, tmp, dst);
 469 }
 470
 471 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 472 {
 473    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 474       return get_ssa_temp(ctx, src.src.ssa);
 475
 476    if (src.src.ssa->num_components == size) {
 477       bool identity_swizzle = true;
 478       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 479          if (src.swizzle[i] != i)
 480             identity_swizzle = false;
 481       }
 482       if (identity_swizzle)
 483          return get_ssa_temp(ctx, src.src.ssa);
 484    }
 485
 486    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 487    unsigned elem_size = vec.bytes() / src.src.ssa->num_components;
 488    assert(elem_size > 0);
 489    assert(vec.bytes() % elem_size == 0);
 490
 491    if (elem_size < 4 && vec.type() == RegType::sgpr) {
 492       assert(src.src.ssa->bit_size == 8 || src.src.ssa->bit_size == 16);
 493       assert(size == 1);
 494       unsigned swizzle = src.swizzle[0];
 495       if (vec.size() > 1) {
 496          assert(src.src.ssa->bit_size == 16);
 497          vec = emit_extract_vector(ctx, vec, swizzle / 2, s1);
 498          swizzle = swizzle & 1;
 499       }
 500       if (swizzle == 0)
 501          return vec;
 502
 503       Temp dst{ctx->program->allocateId(), s1};
 504       aco_ptr<SOP2_instruction> bfe{create_instruction<SOP2_instruction>(aco_opcode::s_bfe_u32, Format::SOP2, 2, 1)};
 505       bfe->operands[0] = Operand(vec);
 506       bfe->operands[1] = Operand(uint32_t((src.src.ssa->bit_size << 16) | (src.src.ssa->bit_size * swizzle)));
 507       bfe->definitions[0] = Definition(dst);
 508       ctx->block->instructions.emplace_back(std::move(bfe));
 509       return dst;
 510    }
 511
 512    RegClass elem_rc = elem_size < 4 ? RegClass(vec.type(), elem_size).as_subdword() : RegClass(vec.type(), elem_size / 4);
 513    if (size == 1) {
 514       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 515    } else {
 516       assert(size <= 4);
 517       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 518       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 519       for (unsigned i = 0; i < size; ++i) {
 520          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 521          vec_instr->operands[i] = Operand{elems[i]};
 522       }
 523       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size / 4)};
 524       vec_instr->definitions[0] = Definition(dst);
 525       ctx->block->instructions.emplace_back(std::move(vec_instr));
 526       ctx->allocated_vec.emplace(dst.id(), elems);
 527       return dst;
 528    }
 529 }
 530
 531 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 532 {
 533    if (ptr.size() == 2)
 534       return ptr;
 535    Builder bld(ctx->program, ctx->block);
 536    if (ptr.type() == RegType::vgpr)
 537       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 538    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 539                      ptr, Operand((unsigned)ctx->options->address32_hi));
 540 }
 541
 542 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 543 {
 544    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 545    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 546    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 547    sop2->definitions[0] = Definition(dst);
 548    if (writes_scc)
 549       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 550    ctx->block->instructions.emplace_back(std::move(sop2));
 551 }
 552
 553 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 554                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 555 {
 556    Builder bld(ctx->program, ctx->block);
 557    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 558    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 559    if (src1.type() == RegType::sgpr) {
 560       if (commutative && src0.type() == RegType::vgpr) {
 561          Temp t = src0;
 562          src0 = src1;
 563          src1 = t;
 564       } else {
 565          src1 = as_vgpr(ctx, src1);
 566       }
 567    }
 568
 569    if (flush_denorms && ctx->program->chip_class < GFX9) {
 570       assert(dst.size() == 1);
 571       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 572       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 573    } else {
 574       bld.vop2(op, Definition(dst), src0, src1);
 575    }
 576 }
 577
 578 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 579                             bool flush_denorms = false)
 580 {
 581    Temp src0 = get_alu_src(ctx, instr->src[0]);
 582    Temp src1 = get_alu_src(ctx, instr->src[1]);
 583    Temp src2 = get_alu_src(ctx, instr->src[2]);
 584
 585    /* ensure that the instruction has at most 1 sgpr operand
 586     * The optimizer will inline constants for us */
 587    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 588       src0 = as_vgpr(ctx, src0);
 589    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 590       src1 = as_vgpr(ctx, src1);
 591    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 592       src2 = as_vgpr(ctx, src2);
 593
 594    Builder bld(ctx->program, ctx->block);
 595    if (flush_denorms && ctx->program->chip_class < GFX9) {
 596       assert(dst.size() == 1);
 597       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 598       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 599    } else {
 600       bld.vop3(op, Definition(dst), src0, src1, src2);
 601    }
 602 }
 603
 604 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 605 {
 606    Builder bld(ctx->program, ctx->block);
 607    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 608 }
 609
 610 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 611 {
 612    Temp src0 = get_alu_src(ctx, instr->src[0]);
 613    Temp src1 = get_alu_src(ctx, instr->src[1]);
 614    assert(src0.size() == src1.size());
 615
 616    aco_ptr<Instruction> vopc;
 617    if (src1.type() == RegType::sgpr) {
 618       if (src0.type() == RegType::vgpr) {
 619          /* to swap the operands, we might also have to change the opcode */
 620          switch (op) {
 621             case aco_opcode::v_cmp_lt_f16:
 622                op = aco_opcode::v_cmp_gt_f16;
 623                break;
 624             case aco_opcode::v_cmp_ge_f16:
 625                op = aco_opcode::v_cmp_le_f16;
 626                break;
 627             case aco_opcode::v_cmp_lt_i16:
 628                op = aco_opcode::v_cmp_gt_i16;
 629                break;
 630             case aco_opcode::v_cmp_ge_i16:
 631                op = aco_opcode::v_cmp_le_i16;
 632                break;
 633             case aco_opcode::v_cmp_lt_u16:
 634                op = aco_opcode::v_cmp_gt_u16;
 635                break;
 636             case aco_opcode::v_cmp_ge_u16:
 637                op = aco_opcode::v_cmp_le_u16;
 638                break;
 639             case aco_opcode::v_cmp_lt_f32:
 640                op = aco_opcode::v_cmp_gt_f32;
 641                break;
 642             case aco_opcode::v_cmp_ge_f32:
 643                op = aco_opcode::v_cmp_le_f32;
 644                break;
 645             case aco_opcode::v_cmp_lt_i32:
 646                op = aco_opcode::v_cmp_gt_i32;
 647                break;
 648             case aco_opcode::v_cmp_ge_i32:
 649                op = aco_opcode::v_cmp_le_i32;
 650                break;
 651             case aco_opcode::v_cmp_lt_u32:
 652                op = aco_opcode::v_cmp_gt_u32;
 653                break;
 654             case aco_opcode::v_cmp_ge_u32:
 655                op = aco_opcode::v_cmp_le_u32;
 656                break;
 657             case aco_opcode::v_cmp_lt_f64:
 658                op = aco_opcode::v_cmp_gt_f64;
 659                break;
 660             case aco_opcode::v_cmp_ge_f64:
 661                op = aco_opcode::v_cmp_le_f64;
 662                break;
 663             case aco_opcode::v_cmp_lt_i64:
 664                op = aco_opcode::v_cmp_gt_i64;
 665                break;
 666             case aco_opcode::v_cmp_ge_i64:
 667                op = aco_opcode::v_cmp_le_i64;
 668                break;
 669             case aco_opcode::v_cmp_lt_u64:
 670                op = aco_opcode::v_cmp_gt_u64;
 671                break;
 672             case aco_opcode::v_cmp_ge_u64:
 673                op = aco_opcode::v_cmp_le_u64;
 674                break;
 675             default: /* eq and ne are commutative */
 676                break;
 677          }
 678          Temp t = src0;
 679          src0 = src1;
 680          src1 = t;
 681       } else {
 682          src1 = as_vgpr(ctx, src1);
 683       }
 684    }
 685
 686    Builder bld(ctx->program, ctx->block);
 687    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 688 }
 689
 690 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 691 {
 692    Temp src0 = get_alu_src(ctx, instr->src[0]);
 693    Temp src1 = get_alu_src(ctx, instr->src[1]);
 694    Builder bld(ctx->program, ctx->block);
 695
 696    assert(dst.regClass() == bld.lm);
 697    assert(src0.type() == RegType::sgpr);
 698    assert(src1.type() == RegType::sgpr);
 699    assert(src0.regClass() == src1.regClass());
 700
 701    /* Emit the SALU comparison instruction */
 702    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 703    /* Turn the result into a per-lane bool */
 704    bool_to_vector_condition(ctx, cmp, dst);
 705 }
 706
 707 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 708                      aco_opcode v16_op, aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 709 {
 710    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : instr->src[0].src.ssa->bit_size == 32 ? s32_op : aco_opcode::num_opcodes;
 711    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : instr->src[0].src.ssa->bit_size == 32 ? v32_op : v16_op;
 712    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 713    bool use_valu = s_op == aco_opcode::num_opcodes ||
 714                    divergent_vals ||
 715                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 716                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 717    aco_opcode op = use_valu ? v_op : s_op;
 718    assert(op != aco_opcode::num_opcodes);
 719    assert(dst.regClass() == ctx->program->lane_mask);
 720
 721    if (use_valu)
 722       emit_vopc_instruction(ctx, instr, op, dst);
 723    else
 724       emit_sopc_instruction(ctx, instr, op, dst);
 725 }
 726
 727 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 728 {
 729    Builder bld(ctx->program, ctx->block);
 730    Temp src0 = get_alu_src(ctx, instr->src[0]);
 731    Temp src1 = get_alu_src(ctx, instr->src[1]);
 732
 733    assert(dst.regClass() == bld.lm);
 734    assert(src0.regClass() == bld.lm);
 735    assert(src1.regClass() == bld.lm);
 736
 737    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 738 }
 739
 740 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 741 {
 742    Builder bld(ctx->program, ctx->block);
 743    Temp cond = get_alu_src(ctx, instr->src[0]);
 744    Temp then = get_alu_src(ctx, instr->src[1]);
 745    Temp els = get_alu_src(ctx, instr->src[2]);
 746
 747    assert(cond.regClass() == bld.lm);
 748
 749    if (dst.type() == RegType::vgpr) {
 750       aco_ptr<Instruction> bcsel;
 751       if (dst.regClass() == v2b) {
 752          then = as_vgpr(ctx, then);
 753          els = as_vgpr(ctx, els);
 754
 755          Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), els, then, cond);
 756          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
 757       } else if (dst.regClass() == v1) {
 758          then = as_vgpr(ctx, then);
 759          els = as_vgpr(ctx, els);
 760
 761          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 762       } else if (dst.regClass() == v2) {
 763          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 764          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 765          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 766          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 767
 768          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 769          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 770
 771          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 772       } else {
 773          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 774          nir_print_instr(&instr->instr, stderr);
 775          fprintf(stderr, "\n");
 776       }
 777       return;
 778    }
 779
 780    if (instr->dest.dest.ssa.bit_size == 1) {
 781       assert(dst.regClass() == bld.lm);
 782       assert(then.regClass() == bld.lm);
 783       assert(els.regClass() == bld.lm);
 784    }
 785
 786    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 787       if (dst.regClass() == s1 || dst.regClass() == s2) {
 788          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 789          assert(dst.size() == then.size());
 790          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 791          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 792       } else {
 793          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 794          nir_print_instr(&instr->instr, stderr);
 795          fprintf(stderr, "\n");
 796       }
 797       return;
 798    }
 799
 800    /* divergent boolean bcsel
 801     * this implements bcsel on bools: dst = s0 ? s1 : s2
 802     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 803    assert(instr->dest.dest.ssa.bit_size == 1);
 804
 805    if (cond.id() != then.id())
 806       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 807
 808    if (cond.id() == els.id())
 809       bld.sop1(Builder::s_mov, Definition(dst), then);
 810    else
 811       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 812                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 813 }
 814
 815 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 816                     aco_opcode op, uint32_t undo)
 817 {
 818    /* multiply by 16777216 to handle denormals */
 819    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 820                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 821    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 822    scaled = bld.vop1(op, bld.def(v1), scaled);
 823    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 824
 825    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 826
 827    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 828 }
 829
 830 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 831 {
 832    if (ctx->block->fp_mode.denorm32 == 0) {
 833       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 834       return;
 835    }
 836
 837    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 838 }
 839
 840 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 841 {
 842    if (ctx->block->fp_mode.denorm32 == 0) {
 843       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 844       return;
 845    }
 846
 847    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 848 }
 849
 850 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 851 {
 852    if (ctx->block->fp_mode.denorm32 == 0) {
 853       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 854       return;
 855    }
 856
 857    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 858 }
 859
 860 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 861 {
 862    if (ctx->block->fp_mode.denorm32 == 0) {
 863       bld.vop1(aco_opcode::v_log_f32, dst, val);
 864       return;
 865    }
 866
 867    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 868 }
 869
 870 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 871 {
 872    if (ctx->options->chip_class >= GFX7)
 873       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 874
 875    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 876    /* TODO: create more efficient code! */
 877    if (val.type() == RegType::sgpr)
 878       val = as_vgpr(ctx, val);
 879
 880    /* Split the input value. */
 881    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 882    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 883
 884    /* Extract the exponent and compute the unbiased value. */
 885    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 886
 887    /* Extract the fractional part. */
 888    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 889    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 890
 891    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 892    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 893
 894    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 895    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 896    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 897    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 898    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 899
 900    /* Get the sign bit. */
 901    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 902
 903    /* Decide the operation to apply depending on the unbiased exponent. */
 904    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 905    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 906    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 907    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 908    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 909    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 910
 911    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 912 }
 913
 914 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 915 {
 916    if (ctx->options->chip_class >= GFX7)
 917       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 918
 919    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 920    Temp src0 = as_vgpr(ctx, val);
 921
 922    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 923    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 924
 925    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 926    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 927    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 928
 929    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 930    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 931    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 932    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 933
 934    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 935    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 936
 937    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 938
 939    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 940    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 941
 942    return add->definitions[0].getTemp();
 943 }
 944
 945 Temp convert_int(Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, bool is_signed, Temp dst=Temp()) {
 946    if (!dst.id()) {
 947       if (dst_bits % 32 == 0 || src.type() == RegType::sgpr)
 948          dst = bld.tmp(src.type(), DIV_ROUND_UP(dst_bits, 32u));
 949       else
 950          dst = bld.tmp(RegClass(RegType::vgpr, dst_bits / 8u).as_subdword());
 951    }
 952
 953    if (dst.bytes() == src.bytes() && dst_bits < src_bits)
 954       return bld.copy(Definition(dst), src);
 955    else if (dst.bytes() < src.bytes())
 956       return bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
 957
 958    Temp tmp = dst;
 959    if (dst_bits == 64)
 960       tmp = src_bits == 32 ? src : bld.tmp(src.type(), 1);
 961
 962    if (tmp == src) {
 963    } else if (src.regClass() == s1) {
 964       if (is_signed)
 965          bld.sop1(src_bits == 8 ? aco_opcode::s_sext_i32_i8 : aco_opcode::s_sext_i32_i16, Definition(tmp), src);
 966       else
 967          bld.sop2(aco_opcode::s_and_b32, Definition(tmp), bld.def(s1, scc), Operand(src_bits == 8 ? 0xFFu : 0xFFFFu), src);
 968    } else {
 969       assert(src_bits != 8 || src.regClass() == v1b);
 970       assert(src_bits != 16 || src.regClass() == v2b);
 971       aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
 972       sdwa->operands[0] = Operand(src);
 973       sdwa->definitions[0] = Definition(tmp);
 974       if (is_signed)
 975          sdwa->sel[0] = src_bits == 8 ? sdwa_sbyte : sdwa_sword;
 976       else
 977          sdwa->sel[0] = src_bits == 8 ? sdwa_ubyte : sdwa_uword;
 978       sdwa->dst_sel = tmp.bytes() == 2 ? sdwa_uword : sdwa_udword;
 979       bld.insert(std::move(sdwa));
 980    }
 981
 982    if (dst_bits == 64) {
 983       if (is_signed && dst.regClass() == s2) {
 984          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), tmp, Operand(31u));
 985          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, high);
 986       } else if (is_signed && dst.regClass() == v2) {
 987          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), tmp);
 988          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, high);
 989       } else {
 990          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, Operand(0u));
 991       }
 992    }
 993
 994    return dst;
 995 }
 996
 997 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 998 {
 999    if (!instr->dest.dest.is_ssa) {
1000       fprintf(stderr, "nir alu dst not in ssa: ");
1001       nir_print_instr(&instr->instr, stderr);
1002       fprintf(stderr, "\n");
1003       abort();
1004    }
1005    Builder bld(ctx->program, ctx->block);
1006    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
1007    switch(instr->op) {
1008    case nir_op_vec2:
1009    case nir_op_vec3:
1010    case nir_op_vec4: {
1011       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
1012       unsigned num = instr->dest.dest.ssa.num_components;
1013       for (unsigned i = 0; i < num; ++i)
1014          elems[i] = get_alu_src(ctx, instr->src[i]);
1015
1016       if (instr->dest.dest.ssa.bit_size >= 32 || dst.type() == RegType::vgpr) {
1017          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
1018          for (unsigned i = 0; i < num; ++i)
1019             vec->operands[i] = Operand{elems[i]};
1020          vec->definitions[0] = Definition(dst);
1021          ctx->block->instructions.emplace_back(std::move(vec));
1022          ctx->allocated_vec.emplace(dst.id(), elems);
1023       } else {
1024          // TODO: that is a bit suboptimal..
1025          Temp mask = bld.copy(bld.def(s1), Operand((1u << instr->dest.dest.ssa.bit_size) - 1));
1026          for (unsigned i = 0; i < num - 1; ++i)
1027             if (((i+1) * instr->dest.dest.ssa.bit_size) % 32)
1028                elems[i] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), elems[i], mask);
1029          for (unsigned i = 0; i < num; ++i) {
1030             unsigned bit = i * instr->dest.dest.ssa.bit_size;
1031             if (bit % 32 == 0) {
1032                elems[bit / 32] = elems[i];
1033             } else {
1034                elems[i] = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc),
1035                                    elems[i], Operand((i * instr->dest.dest.ssa.bit_size) % 32));
1036                elems[bit / 32] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), elems[bit / 32], elems[i]);
1037             }
1038          }
1039          if (dst.size() == 1)
1040             bld.copy(Definition(dst), elems[0]);
1041          else
1042             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), elems[0], elems[1]);
1043       }
1044       break;
1045    }
1046    case nir_op_mov: {
1047       Temp src = get_alu_src(ctx, instr->src[0]);
1048       aco_ptr<Instruction> mov;
1049       if (dst.type() == RegType::sgpr) {
1050          if (src.type() == RegType::vgpr)
1051             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
1052          else if (src.regClass() == s1)
1053             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
1054          else if (src.regClass() == s2)
1055             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
1056          else
1057             unreachable("wrong src register class for nir_op_imov");
1058       } else if (dst.regClass() == v1) {
1059          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
1060       } else if (dst.regClass() == v2) {
1061          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
1062       } else {
1063          nir_print_instr(&instr->instr, stderr);
1064          unreachable("Should have been lowered to scalar.");
1065       }
1066       break;
1067    }
1068    case nir_op_inot: {
1069       Temp src = get_alu_src(ctx, instr->src[0]);
1070       if (instr->dest.dest.ssa.bit_size == 1) {
1071          assert(src.regClass() == bld.lm);
1072          assert(dst.regClass() == bld.lm);
1073          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
1074          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
1075          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
1076       } else if (dst.regClass() == v1) {
1077          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
1078       } else if (dst.type() == RegType::sgpr) {
1079          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
1080          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
1081       } else {
1082          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1083          nir_print_instr(&instr->instr, stderr);
1084          fprintf(stderr, "\n");
1085       }
1086       break;
1087    }
1088    case nir_op_ineg: {
1089       Temp src = get_alu_src(ctx, instr->src[0]);
1090       if (dst.regClass() == v1) {
1091          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
1092       } else if (dst.regClass() == s1) {
1093          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
1094       } else if (dst.size() == 2) {
1095          Temp src0 = bld.tmp(dst.type(), 1);
1096          Temp src1 = bld.tmp(dst.type(), 1);
1097          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
1098
1099          if (dst.regClass() == s2) {
1100             Temp carry = bld.tmp(s1);
1101             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
1102             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
1103             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1104          } else {
1105             Temp lower = bld.tmp(v1);
1106             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
1107             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
1108             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1109          }
1110       } else {
1111          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1112          nir_print_instr(&instr->instr, stderr);
1113          fprintf(stderr, "\n");
1114       }
1115       break;
1116    }
1117    case nir_op_iabs: {
1118       if (dst.regClass() == s1) {
1119          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
1120       } else if (dst.regClass() == v1) {
1121          Temp src = get_alu_src(ctx, instr->src[0]);
1122          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_isign: {
1131       Temp src = get_alu_src(ctx, instr->src[0]);
1132       if (dst.regClass() == s1) {
1133          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1134          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
1135          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
1136       } else if (dst.regClass() == s2) {
1137          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
1138          Temp neqz;
1139          if (ctx->program->chip_class >= GFX8)
1140             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
1141          else
1142             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
1143          /* SCC gets zero-extended to 64 bit */
1144          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
1145       } else if (dst.regClass() == v1) {
1146          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1147          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1148          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
1149       } else if (dst.regClass() == v2) {
1150          Temp upper = emit_extract_vector(ctx, src, 1, v1);
1151          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
1152          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1153          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
1154          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
1155          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1156       } else {
1157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1158          nir_print_instr(&instr->instr, stderr);
1159          fprintf(stderr, "\n");
1160       }
1161       break;
1162    }
1163    case nir_op_imax: {
1164       if (dst.regClass() == v1) {
1165          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
1166       } else if (dst.regClass() == s1) {
1167          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
1168       } else {
1169          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1170          nir_print_instr(&instr->instr, stderr);
1171          fprintf(stderr, "\n");
1172       }
1173       break;
1174    }
1175    case nir_op_umax: {
1176       if (dst.regClass() == v1) {
1177          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
1178       } else if (dst.regClass() == s1) {
1179          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
1180       } else {
1181          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1182          nir_print_instr(&instr->instr, stderr);
1183          fprintf(stderr, "\n");
1184       }
1185       break;
1186    }
1187    case nir_op_imin: {
1188       if (dst.regClass() == v1) {
1189          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
1190       } else if (dst.regClass() == s1) {
1191          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
1192       } else {
1193          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1194          nir_print_instr(&instr->instr, stderr);
1195          fprintf(stderr, "\n");
1196       }
1197       break;
1198    }
1199    case nir_op_umin: {
1200       if (dst.regClass() == v1) {
1201          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
1202       } else if (dst.regClass() == s1) {
1203          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1204       } else {
1205          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1206          nir_print_instr(&instr->instr, stderr);
1207          fprintf(stderr, "\n");
1208       }
1209       break;
1210    }
1211    case nir_op_ior: {
1212       if (instr->dest.dest.ssa.bit_size == 1) {
1213          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1214       } else if (dst.regClass() == v1) {
1215          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1216       } else if (dst.regClass() == s1) {
1217          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1218       } else if (dst.regClass() == s2) {
1219          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1220       } else {
1221          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1222          nir_print_instr(&instr->instr, stderr);
1223          fprintf(stderr, "\n");
1224       }
1225       break;
1226    }
1227    case nir_op_iand: {
1228       if (instr->dest.dest.ssa.bit_size == 1) {
1229          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1230       } else if (dst.regClass() == v1) {
1231          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1232       } else if (dst.regClass() == s1) {
1233          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1234       } else if (dst.regClass() == s2) {
1235          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1236       } else {
1237          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1238          nir_print_instr(&instr->instr, stderr);
1239          fprintf(stderr, "\n");
1240       }
1241       break;
1242    }
1243    case nir_op_ixor: {
1244       if (instr->dest.dest.ssa.bit_size == 1) {
1245          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1246       } else if (dst.regClass() == v1) {
1247          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1248       } else if (dst.regClass() == s1) {
1249          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1250       } else if (dst.regClass() == s2) {
1251          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1252       } else {
1253          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1254          nir_print_instr(&instr->instr, stderr);
1255          fprintf(stderr, "\n");
1256       }
1257       break;
1258    }
1259    case nir_op_ushr: {
1260       if (dst.regClass() == v1) {
1261          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1262       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1263          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1264                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1265       } else if (dst.regClass() == v2) {
1266          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1267                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1268       } else if (dst.regClass() == s2) {
1269          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1270       } else if (dst.regClass() == s1) {
1271          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1272       } else {
1273          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1274          nir_print_instr(&instr->instr, stderr);
1275          fprintf(stderr, "\n");
1276       }
1277       break;
1278    }
1279    case nir_op_ishl: {
1280       if (dst.regClass() == v1) {
1281          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1282       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1283          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1284                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1285       } else if (dst.regClass() == v2) {
1286          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1287                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1288       } else if (dst.regClass() == s1) {
1289          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1290       } else if (dst.regClass() == s2) {
1291          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1292       } else {
1293          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1294          nir_print_instr(&instr->instr, stderr);
1295          fprintf(stderr, "\n");
1296       }
1297       break;
1298    }
1299    case nir_op_ishr: {
1300       if (dst.regClass() == v1) {
1301          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1302       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1303          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1304                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1305       } else if (dst.regClass() == v2) {
1306          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1307                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1308       } else if (dst.regClass() == s1) {
1309          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1310       } else if (dst.regClass() == s2) {
1311          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1312       } else {
1313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1314          nir_print_instr(&instr->instr, stderr);
1315          fprintf(stderr, "\n");
1316       }
1317       break;
1318    }
1319    case nir_op_find_lsb: {
1320       Temp src = get_alu_src(ctx, instr->src[0]);
1321       if (src.regClass() == s1) {
1322          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1323       } else if (src.regClass() == v1) {
1324          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1325       } else if (src.regClass() == s2) {
1326          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1327       } else {
1328          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1329          nir_print_instr(&instr->instr, stderr);
1330          fprintf(stderr, "\n");
1331       }
1332       break;
1333    }
1334    case nir_op_ufind_msb:
1335    case nir_op_ifind_msb: {
1336       Temp src = get_alu_src(ctx, instr->src[0]);
1337       if (src.regClass() == s1 || src.regClass() == s2) {
1338          aco_opcode op = src.regClass() == s2 ?
1339                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1340                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1341          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1342
1343          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1344                                         Operand(src.size() * 32u - 1u), msb_rev);
1345          Temp msb = sub.def(0).getTemp();
1346          Temp carry = sub.def(1).getTemp();
1347
1348          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1349       } else if (src.regClass() == v1) {
1350          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1351          Temp msb_rev = bld.tmp(v1);
1352          emit_vop1_instruction(ctx, instr, op, msb_rev);
1353          Temp msb = bld.tmp(v1);
1354          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1355          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1356       } else {
1357          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1358          nir_print_instr(&instr->instr, stderr);
1359          fprintf(stderr, "\n");
1360       }
1361       break;
1362    }
1363    case nir_op_bitfield_reverse: {
1364       if (dst.regClass() == s1) {
1365          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1366       } else if (dst.regClass() == v1) {
1367          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1368       } else {
1369          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1370          nir_print_instr(&instr->instr, stderr);
1371          fprintf(stderr, "\n");
1372       }
1373       break;
1374    }
1375    case nir_op_iadd: {
1376       if (dst.regClass() == s1) {
1377          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1378          break;
1379       }
1380
1381       Temp src0 = get_alu_src(ctx, instr->src[0]);
1382       Temp src1 = get_alu_src(ctx, instr->src[1]);
1383       if (dst.regClass() == v1) {
1384          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1385          break;
1386       }
1387
1388       assert(src0.size() == 2 && src1.size() == 2);
1389       Temp src00 = bld.tmp(src0.type(), 1);
1390       Temp src01 = bld.tmp(dst.type(), 1);
1391       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1392       Temp src10 = bld.tmp(src1.type(), 1);
1393       Temp src11 = bld.tmp(dst.type(), 1);
1394       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1395
1396       if (dst.regClass() == s2) {
1397          Temp carry = bld.tmp(s1);
1398          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1399          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1400          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1401       } else if (dst.regClass() == v2) {
1402          Temp dst0 = bld.tmp(v1);
1403          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1404          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1405          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1406       } else {
1407          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1408          nir_print_instr(&instr->instr, stderr);
1409          fprintf(stderr, "\n");
1410       }
1411       break;
1412    }
1413    case nir_op_uadd_sat: {
1414       Temp src0 = get_alu_src(ctx, instr->src[0]);
1415       Temp src1 = get_alu_src(ctx, instr->src[1]);
1416       if (dst.regClass() == s1) {
1417          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1418          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1419                   src0, src1);
1420          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1421       } else if (dst.regClass() == v1) {
1422          if (ctx->options->chip_class >= GFX9) {
1423             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1424             add->operands[0] = Operand(src0);
1425             add->operands[1] = Operand(src1);
1426             add->definitions[0] = Definition(dst);
1427             add->clamp = 1;
1428             ctx->block->instructions.emplace_back(std::move(add));
1429          } else {
1430             if (src1.regClass() != v1)
1431                std::swap(src0, src1);
1432             assert(src1.regClass() == v1);
1433             Temp tmp = bld.tmp(v1);
1434             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1435             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1436          }
1437       } else {
1438          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1439          nir_print_instr(&instr->instr, stderr);
1440          fprintf(stderr, "\n");
1441       }
1442       break;
1443    }
1444    case nir_op_uadd_carry: {
1445       Temp src0 = get_alu_src(ctx, instr->src[0]);
1446       Temp src1 = get_alu_src(ctx, instr->src[1]);
1447       if (dst.regClass() == s1) {
1448          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1449          break;
1450       }
1451       if (dst.regClass() == v1) {
1452          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1453          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1454          break;
1455       }
1456
1457       Temp src00 = bld.tmp(src0.type(), 1);
1458       Temp src01 = bld.tmp(dst.type(), 1);
1459       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1460       Temp src10 = bld.tmp(src1.type(), 1);
1461       Temp src11 = bld.tmp(dst.type(), 1);
1462       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1463       if (dst.regClass() == s2) {
1464          Temp carry = bld.tmp(s1);
1465          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1466          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1467          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1468       } else if (dst.regClass() == v2) {
1469          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1470          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1471          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1472          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1473       } else {
1474          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1475          nir_print_instr(&instr->instr, stderr);
1476          fprintf(stderr, "\n");
1477       }
1478       break;
1479    }
1480    case nir_op_isub: {
1481       if (dst.regClass() == s1) {
1482          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1483          break;
1484       }
1485
1486       Temp src0 = get_alu_src(ctx, instr->src[0]);
1487       Temp src1 = get_alu_src(ctx, instr->src[1]);
1488       if (dst.regClass() == v1) {
1489          bld.vsub32(Definition(dst), src0, src1);
1490          break;
1491       }
1492
1493       Temp src00 = bld.tmp(src0.type(), 1);
1494       Temp src01 = bld.tmp(dst.type(), 1);
1495       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1496       Temp src10 = bld.tmp(src1.type(), 1);
1497       Temp src11 = bld.tmp(dst.type(), 1);
1498       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1499       if (dst.regClass() == s2) {
1500          Temp carry = bld.tmp(s1);
1501          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1502          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1503          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1504       } else if (dst.regClass() == v2) {
1505          Temp lower = bld.tmp(v1);
1506          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1507          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1508          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1509       } else {
1510          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1511          nir_print_instr(&instr->instr, stderr);
1512          fprintf(stderr, "\n");
1513       }
1514       break;
1515    }
1516    case nir_op_usub_borrow: {
1517       Temp src0 = get_alu_src(ctx, instr->src[0]);
1518       Temp src1 = get_alu_src(ctx, instr->src[1]);
1519       if (dst.regClass() == s1) {
1520          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1521          break;
1522       } else if (dst.regClass() == v1) {
1523          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1524          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1525          break;
1526       }
1527
1528       Temp src00 = bld.tmp(src0.type(), 1);
1529       Temp src01 = bld.tmp(dst.type(), 1);
1530       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1531       Temp src10 = bld.tmp(src1.type(), 1);
1532       Temp src11 = bld.tmp(dst.type(), 1);
1533       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1534       if (dst.regClass() == s2) {
1535          Temp borrow = bld.tmp(s1);
1536          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1537          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1538          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1539       } else if (dst.regClass() == v2) {
1540          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1541          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1542          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1543          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_imul: {
1552       if (dst.regClass() == v1) {
1553          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1554                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1555       } else if (dst.regClass() == s1) {
1556          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1557       } else {
1558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1559          nir_print_instr(&instr->instr, stderr);
1560          fprintf(stderr, "\n");
1561       }
1562       break;
1563    }
1564    case nir_op_umul_high: {
1565       if (dst.regClass() == v1) {
1566          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1567       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1568          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1569       } else if (dst.regClass() == s1) {
1570          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1571                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1572          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1573       } else {
1574          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1575          nir_print_instr(&instr->instr, stderr);
1576          fprintf(stderr, "\n");
1577       }
1578       break;
1579    }
1580    case nir_op_imul_high: {
1581       if (dst.regClass() == v1) {
1582          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1583       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1584          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1585       } else if (dst.regClass() == s1) {
1586          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1587                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1588          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_fmul: {
1597       Temp src0 = get_alu_src(ctx, instr->src[0]);
1598       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1599       if (dst.regClass() == v2b) {
1600          Temp tmp = bld.tmp(v1);
1601          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f16, tmp, true);
1602          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1603       } else if (dst.regClass() == v1) {
1604          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1605       } else if (dst.regClass() == v2) {
1606          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), src0, src1);
1607       } else {
1608          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1609          nir_print_instr(&instr->instr, stderr);
1610          fprintf(stderr, "\n");
1611       }
1612       break;
1613    }
1614    case nir_op_fadd: {
1615       Temp src0 = get_alu_src(ctx, instr->src[0]);
1616       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1617       if (dst.regClass() == v2b) {
1618          Temp tmp = bld.tmp(v1);
1619          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f16, tmp, true);
1620          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1621       } else if (dst.regClass() == v1) {
1622          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1623       } else if (dst.regClass() == v2) {
1624          bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, src1);
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_fsub: {
1633       Temp src0 = get_alu_src(ctx, instr->src[0]);
1634       Temp src1 = get_alu_src(ctx, instr->src[1]);
1635       if (dst.regClass() == v2b) {
1636          Temp tmp = bld.tmp(v1);
1637          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1638             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f16, tmp, false);
1639          else
1640             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f16, tmp, true);
1641          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1642       } else if (dst.regClass() == v1) {
1643          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1644             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1645          else
1646             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1647       } else if (dst.regClass() == v2) {
1648          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1649                                      as_vgpr(ctx, src0), as_vgpr(ctx, src1));
1650          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1651          sub->neg[1] = true;
1652       } else {
1653          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1654          nir_print_instr(&instr->instr, stderr);
1655          fprintf(stderr, "\n");
1656       }
1657       break;
1658    }
1659    case nir_op_fmax: {
1660       Temp src0 = get_alu_src(ctx, instr->src[0]);
1661       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1662       if (dst.regClass() == v2b) {
1663          // TODO: check fp_mode.must_flush_denorms16_64
1664          Temp tmp = bld.tmp(v1);
1665          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f16, tmp, true);
1666          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1667       } else if (dst.regClass() == v1) {
1668          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1669       } else if (dst.regClass() == v2) {
1670          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1671             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2), src0, src1);
1672             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1673          } else {
1674             bld.vop3(aco_opcode::v_max_f64, Definition(dst), src0, src1);
1675          }
1676       } else {
1677          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1678          nir_print_instr(&instr->instr, stderr);
1679          fprintf(stderr, "\n");
1680       }
1681       break;
1682    }
1683    case nir_op_fmin: {
1684       Temp src0 = get_alu_src(ctx, instr->src[0]);
1685       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1686       if (dst.regClass() == v2b) {
1687          // TODO: check fp_mode.must_flush_denorms16_64
1688          Temp tmp = bld.tmp(v1);
1689          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f16, tmp, true);
1690          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1691       } else if (dst.regClass() == v1) {
1692          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1693       } else if (dst.regClass() == v2) {
1694          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1695             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), src0, src1);
1696             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1697          } else {
1698             bld.vop3(aco_opcode::v_min_f64, Definition(dst), src0, src1);
1699          }
1700       } else {
1701          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1702          nir_print_instr(&instr->instr, stderr);
1703          fprintf(stderr, "\n");
1704       }
1705       break;
1706    }
1707    case nir_op_fmax3: {
1708       if (dst.regClass() == v2b) {
1709          Temp tmp = bld.tmp(v1);
1710          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f16, tmp, false);
1711          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1712       } else if (dst.regClass() == v1) {
1713          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1714       } else {
1715          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1716          nir_print_instr(&instr->instr, stderr);
1717          fprintf(stderr, "\n");
1718       }
1719       break;
1720    }
1721    case nir_op_fmin3: {
1722       if (dst.regClass() == v2b) {
1723          Temp tmp = bld.tmp(v1);
1724          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f16, tmp, false);
1725          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1726       } else if (dst.regClass() == v1) {
1727          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1728       } else {
1729          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1730          nir_print_instr(&instr->instr, stderr);
1731          fprintf(stderr, "\n");
1732       }
1733       break;
1734    }
1735    case nir_op_fmed3: {
1736       if (dst.regClass() == v2b) {
1737          Temp tmp = bld.tmp(v1);
1738          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f16, tmp, false);
1739          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1740       } else if (dst.regClass() == v1) {
1741          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1742       } else {
1743          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1744          nir_print_instr(&instr->instr, stderr);
1745          fprintf(stderr, "\n");
1746       }
1747       break;
1748    }
1749    case nir_op_umax3: {
1750       if (dst.size() == 1) {
1751          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1752       } else {
1753          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1754          nir_print_instr(&instr->instr, stderr);
1755          fprintf(stderr, "\n");
1756       }
1757       break;
1758    }
1759    case nir_op_umin3: {
1760       if (dst.size() == 1) {
1761          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1762       } else {
1763          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1764          nir_print_instr(&instr->instr, stderr);
1765          fprintf(stderr, "\n");
1766       }
1767       break;
1768    }
1769    case nir_op_umed3: {
1770       if (dst.size() == 1) {
1771          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1772       } else {
1773          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1774          nir_print_instr(&instr->instr, stderr);
1775          fprintf(stderr, "\n");
1776       }
1777       break;
1778    }
1779    case nir_op_imax3: {
1780       if (dst.size() == 1) {
1781          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1782       } else {
1783          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1784          nir_print_instr(&instr->instr, stderr);
1785          fprintf(stderr, "\n");
1786       }
1787       break;
1788    }
1789    case nir_op_imin3: {
1790       if (dst.size() == 1) {
1791          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1792       } else {
1793          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1794          nir_print_instr(&instr->instr, stderr);
1795          fprintf(stderr, "\n");
1796       }
1797       break;
1798    }
1799    case nir_op_imed3: {
1800       if (dst.size() == 1) {
1801          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1802       } else {
1803          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1804          nir_print_instr(&instr->instr, stderr);
1805          fprintf(stderr, "\n");
1806       }
1807       break;
1808    }
1809    case nir_op_cube_face_coord: {
1810       Temp in = get_alu_src(ctx, instr->src[0], 3);
1811       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1812                       emit_extract_vector(ctx, in, 1, v1),
1813                       emit_extract_vector(ctx, in, 2, v1) };
1814       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1815       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1816       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1817       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1818       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1819       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1820       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1821       break;
1822    }
1823    case nir_op_cube_face_index: {
1824       Temp in = get_alu_src(ctx, instr->src[0], 3);
1825       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1826                       emit_extract_vector(ctx, in, 1, v1),
1827                       emit_extract_vector(ctx, in, 2, v1) };
1828       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1829       break;
1830    }
1831    case nir_op_bcsel: {
1832       emit_bcsel(ctx, instr, dst);
1833       break;
1834    }
1835    case nir_op_frsq: {
1836       Temp src = get_alu_src(ctx, instr->src[0]);
1837       if (dst.regClass() == v2b) {
1838          Temp tmp = bld.vop1(aco_opcode::v_rsq_f16, bld.def(v1), src);
1839          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1840       } else if (dst.regClass() == v1) {
1841          emit_rsq(ctx, bld, Definition(dst), src);
1842       } else if (dst.regClass() == v2) {
1843          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1844       } else {
1845          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1846          nir_print_instr(&instr->instr, stderr);
1847          fprintf(stderr, "\n");
1848       }
1849       break;
1850    }
1851    case nir_op_fneg: {
1852       Temp src = get_alu_src(ctx, instr->src[0]);
1853       if (dst.regClass() == v2b) {
1854          Temp tmp = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x8000u), as_vgpr(ctx, src));
1855          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1856       } else if (dst.regClass() == v1) {
1857          if (ctx->block->fp_mode.must_flush_denorms32)
1858             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1859          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1860       } else if (dst.regClass() == v2) {
1861          if (ctx->block->fp_mode.must_flush_denorms16_64)
1862             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1863          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1864          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1865          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1866          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1867       } else {
1868          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1869          nir_print_instr(&instr->instr, stderr);
1870          fprintf(stderr, "\n");
1871       }
1872       break;
1873    }
1874    case nir_op_fabs: {
1875       Temp src = get_alu_src(ctx, instr->src[0]);
1876       if (dst.regClass() == v2b) {
1877          Temp tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFu), as_vgpr(ctx, src));
1878          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1879       } else if (dst.regClass() == v1) {
1880          if (ctx->block->fp_mode.must_flush_denorms32)
1881             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1882          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1883       } else if (dst.regClass() == v2) {
1884          if (ctx->block->fp_mode.must_flush_denorms16_64)
1885             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1886          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1887          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1888          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1889          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1890       } else {
1891          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1892          nir_print_instr(&instr->instr, stderr);
1893          fprintf(stderr, "\n");
1894       }
1895       break;
1896    }
1897    case nir_op_fsat: {
1898       Temp src = get_alu_src(ctx, instr->src[0]);
1899       if (dst.regClass() == v2b) {
1900          Temp tmp = bld.vop3(aco_opcode::v_med3_f16, bld.def(v1), Operand(0u), Operand(0x3f800000u), src);
1901          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1902       } else if (dst.regClass() == v1) {
1903          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1904          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1905          // TODO: confirm that this holds under any circumstances
1906       } else if (dst.regClass() == v2) {
1907          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1908          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1909          vop3->clamp = true;
1910       } else {
1911          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1912          nir_print_instr(&instr->instr, stderr);
1913          fprintf(stderr, "\n");
1914       }
1915       break;
1916    }
1917    case nir_op_flog2: {
1918       Temp src = get_alu_src(ctx, instr->src[0]);
1919       if (dst.regClass() == v2b) {
1920          Temp tmp = bld.vop1(aco_opcode::v_log_f16, bld.def(v1), src);
1921          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1922       } else if (dst.regClass() == v1) {
1923          emit_log2(ctx, bld, Definition(dst), src);
1924       } else {
1925          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1926          nir_print_instr(&instr->instr, stderr);
1927          fprintf(stderr, "\n");
1928       }
1929       break;
1930    }
1931    case nir_op_frcp: {
1932       Temp src = get_alu_src(ctx, instr->src[0]);
1933       if (dst.regClass() == v2b) {
1934          Temp tmp = bld.vop1(aco_opcode::v_rcp_f16, bld.def(v1), src);
1935          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1936       } else if (dst.regClass() == v1) {
1937          emit_rcp(ctx, bld, Definition(dst), src);
1938       } else if (dst.regClass() == v2) {
1939          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1940       } else {
1941          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1942          nir_print_instr(&instr->instr, stderr);
1943          fprintf(stderr, "\n");
1944       }
1945       break;
1946    }
1947    case nir_op_fexp2: {
1948       if (dst.regClass() == v2b) {
1949          Temp src = get_alu_src(ctx, instr->src[0]);
1950          Temp tmp = bld.vop1(aco_opcode::v_exp_f16, bld.def(v1), src);
1951          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1952       } else if (dst.regClass() == v1) {
1953          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1954       } else {
1955          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1956          nir_print_instr(&instr->instr, stderr);
1957          fprintf(stderr, "\n");
1958       }
1959       break;
1960    }
1961    case nir_op_fsqrt: {
1962       Temp src = get_alu_src(ctx, instr->src[0]);
1963       if (dst.regClass() == v2b) {
1964          Temp tmp = bld.vop1(aco_opcode::v_sqrt_f16, bld.def(v1), src);
1965          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1966       } else if (dst.regClass() == v1) {
1967          emit_sqrt(ctx, bld, Definition(dst), src);
1968       } else if (dst.regClass() == v2) {
1969          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1970       } else {
1971          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1972          nir_print_instr(&instr->instr, stderr);
1973          fprintf(stderr, "\n");
1974       }
1975       break;
1976    }
1977    case nir_op_ffract: {
1978       if (dst.regClass() == v2b) {
1979          Temp src = get_alu_src(ctx, instr->src[0]);
1980          Temp tmp = bld.vop1(aco_opcode::v_fract_f16, bld.def(v1), src);
1981          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1982       } else if (dst.regClass() == v1) {
1983          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1984       } else if (dst.regClass() == v2) {
1985          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_ffloor: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       if (dst.regClass() == v2b) {
1996          Temp tmp = bld.vop1(aco_opcode::v_floor_f16, bld.def(v1), src);
1997          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
1998       } else if (dst.regClass() == v1) {
1999          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
2000       } else if (dst.regClass() == v2) {
2001          emit_floor_f64(ctx, bld, Definition(dst), src);
2002       } else {
2003          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2004          nir_print_instr(&instr->instr, stderr);
2005          fprintf(stderr, "\n");
2006       }
2007       break;
2008    }
2009    case nir_op_fceil: {
2010       Temp src0 = get_alu_src(ctx, instr->src[0]);
2011       if (dst.regClass() == v2b) {
2012          Temp tmp = bld.vop1(aco_opcode::v_ceil_f16, bld.def(v1), src0);
2013          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2014       } else if (dst.regClass() == v1) {
2015          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
2016       } else if (dst.regClass() == v2) {
2017          if (ctx->options->chip_class >= GFX7) {
2018             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
2019          } else {
2020             /* GFX6 doesn't support V_CEIL_F64, lower it. */
2021             /* trunc = trunc(src0)
2022              * if (src0 > 0.0 && src0 != trunc)
2023              *    trunc += 1.0
2024              */
2025             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
2026             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
2027             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
2028             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
2029             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
2030             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
2031             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
2032          }
2033       } else {
2034          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2035          nir_print_instr(&instr->instr, stderr);
2036          fprintf(stderr, "\n");
2037       }
2038       break;
2039    }
2040    case nir_op_ftrunc: {
2041       Temp src = get_alu_src(ctx, instr->src[0]);
2042       if (dst.regClass() == v2b) {
2043          Temp tmp = bld.vop1(aco_opcode::v_trunc_f16, bld.def(v1), src);
2044          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2045       } else if (dst.regClass() == v1) {
2046          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
2047       } else if (dst.regClass() == v2) {
2048          emit_trunc_f64(ctx, bld, Definition(dst), src);
2049       } else {
2050          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2051          nir_print_instr(&instr->instr, stderr);
2052          fprintf(stderr, "\n");
2053       }
2054       break;
2055    }
2056    case nir_op_fround_even: {
2057       Temp src0 = get_alu_src(ctx, instr->src[0]);
2058       if (dst.regClass() == v2b) {
2059          Temp tmp = bld.vop1(aco_opcode::v_rndne_f16, bld.def(v1), src0);
2060          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2061       } else if (dst.regClass() == v1) {
2062          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
2063       } else if (dst.regClass() == v2) {
2064          if (ctx->options->chip_class >= GFX7) {
2065             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
2066          } else {
2067             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
2068             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
2069             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
2070
2071             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
2072             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
2073             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
2074             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
2075             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
2076             tmp = sub->definitions[0].getTemp();
2077
2078             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
2079             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
2080             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2081             Temp cond = vop3->definitions[0].getTemp();
2082
2083             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
2084             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
2085             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
2086             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
2087
2088             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
2089          }
2090       } else {
2091          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2092          nir_print_instr(&instr->instr, stderr);
2093          fprintf(stderr, "\n");
2094       }
2095       break;
2096    }
2097    case nir_op_fsin:
2098    case nir_op_fcos: {
2099       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2100       aco_ptr<Instruction> norm;
2101       Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
2102       if (dst.regClass() == v2b) {
2103          Temp tmp = bld.vop2(aco_opcode::v_mul_f16, bld.def(v1), half_pi, src);
2104          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f16 : aco_opcode::v_cos_f16;
2105          tmp = bld.vop1(opcode, bld.def(v1), tmp);
2106          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2107       } else if (dst.regClass() == v1) {
2108          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
2109
2110          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
2111          if (ctx->options->chip_class < GFX9)
2112             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
2113
2114          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
2115          bld.vop1(opcode, Definition(dst), tmp);
2116       } else {
2117          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2118          nir_print_instr(&instr->instr, stderr);
2119          fprintf(stderr, "\n");
2120       }
2121       break;
2122    }
2123    case nir_op_ldexp: {
2124       Temp src0 = get_alu_src(ctx, instr->src[0]);
2125       Temp src1 = get_alu_src(ctx, instr->src[1]);
2126       if (dst.regClass() == v2b) {
2127          Temp tmp = bld.tmp(v1);
2128          emit_vop2_instruction(ctx, instr, aco_opcode::v_ldexp_f16, tmp, false);
2129          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2130       } else if (dst.regClass() == v1) {
2131          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst), as_vgpr(ctx, src0), src1);
2132       } else if (dst.regClass() == v2) {
2133          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst), as_vgpr(ctx, src0), src1);
2134       } else {
2135          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2136          nir_print_instr(&instr->instr, stderr);
2137          fprintf(stderr, "\n");
2138       }
2139       break;
2140    }
2141    case nir_op_frexp_sig: {
2142       Temp src = get_alu_src(ctx, instr->src[0]);
2143       if (dst.regClass() == v2b) {
2144          Temp tmp = bld.vop1(aco_opcode::v_frexp_mant_f16, bld.def(v1), src);
2145          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2146       } else if (dst.regClass() == v1) {
2147          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst), src);
2148       } else if (dst.regClass() == v2) {
2149          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst), src);
2150       } else {
2151          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2152          nir_print_instr(&instr->instr, stderr);
2153          fprintf(stderr, "\n");
2154       }
2155       break;
2156    }
2157    case nir_op_frexp_exp: {
2158       Temp src = get_alu_src(ctx, instr->src[0]);
2159       if (instr->src[0].src.ssa->bit_size == 16) {
2160          Temp tmp = bld.vop1(aco_opcode::v_frexp_exp_i16_f16, bld.def(v1), src);
2161          tmp = bld.pseudo(aco_opcode::p_extract_vector, bld.def(v1b), tmp, Operand(0u));
2162          convert_int(bld, tmp, 8, 32, true, dst);
2163       } else if (instr->src[0].src.ssa->bit_size == 32) {
2164          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst), src);
2165       } else if (instr->src[0].src.ssa->bit_size == 64) {
2166          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst), src);
2167       } else {
2168          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2169          nir_print_instr(&instr->instr, stderr);
2170          fprintf(stderr, "\n");
2171       }
2172       break;
2173    }
2174    case nir_op_fsign: {
2175       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2176       if (dst.regClass() == v2b) {
2177          Temp one = bld.copy(bld.def(v1), Operand(0x3c00u));
2178          Temp minus_one = bld.copy(bld.def(v1), Operand(0xbc00u));
2179          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f16, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2180          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), one, src, cond);
2181          cond = bld.vopc(aco_opcode::v_cmp_le_f16, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2182          Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), minus_one, src, cond);
2183          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2184       } else if (dst.regClass() == v1) {
2185          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2186          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
2187          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2188          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
2189       } else if (dst.regClass() == v2) {
2190          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2191          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
2192          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
2193
2194          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2195          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
2196          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
2197
2198          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2199       } else {
2200          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2201          nir_print_instr(&instr->instr, stderr);
2202          fprintf(stderr, "\n");
2203       }
2204       break;
2205    }
2206    case nir_op_f2f16:
2207    case nir_op_f2f16_rtne: {
2208       Temp src = get_alu_src(ctx, instr->src[0]);
2209       if (instr->src[0].src.ssa->bit_size == 64)
2210          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2211       src = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2212       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2213       break;
2214    }
2215    case nir_op_f2f16_rtz: {
2216       Temp src = get_alu_src(ctx, instr->src[0]);
2217       if (instr->src[0].src.ssa->bit_size == 64)
2218          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2219       src = bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, bld.def(v1), src, Operand(0u));
2220       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), src);
2221       break;
2222    }
2223    case nir_op_f2f32: {
2224       if (instr->src[0].src.ssa->bit_size == 16) {
2225          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f16, dst);
2226       } else if (instr->src[0].src.ssa->bit_size == 64) {
2227          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
2228       } else {
2229          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2230          nir_print_instr(&instr->instr, stderr);
2231          fprintf(stderr, "\n");
2232       }
2233       break;
2234    }
2235    case nir_op_f2f64: {
2236       Temp src = get_alu_src(ctx, instr->src[0]);
2237       if (instr->src[0].src.ssa->bit_size == 16)
2238          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2239       bld.vop1(aco_opcode::v_cvt_f64_f32, Definition(dst), src);
2240       break;
2241    }
2242    case nir_op_i2f16: {
2243       assert(dst.regClass() == v2b);
2244       Temp src = get_alu_src(ctx, instr->src[0]);
2245       if (instr->src[0].src.ssa->bit_size == 8)
2246          src = convert_int(bld, src, 8, 16, true);
2247       Temp tmp = bld.vop1(aco_opcode::v_cvt_f16_i16, bld.def(v1), src);
2248       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2249       break;
2250    }
2251    case nir_op_i2f32: {
2252       assert(dst.size() == 1);
2253       Temp src = get_alu_src(ctx, instr->src[0]);
2254       if (instr->src[0].src.ssa->bit_size <= 16)
2255          src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2256       bld.vop1(aco_opcode::v_cvt_f32_i32, Definition(dst), src);
2257       break;
2258    }
2259    case nir_op_i2f64: {
2260       if (instr->src[0].src.ssa->bit_size <= 32) {
2261          Temp src = get_alu_src(ctx, instr->src[0]);
2262          if (instr->src[0].src.ssa->bit_size <= 16)
2263             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2264          bld.vop1(aco_opcode::v_cvt_f64_i32, Definition(dst), src);
2265       } else if (instr->src[0].src.ssa->bit_size == 64) {
2266          Temp src = get_alu_src(ctx, instr->src[0]);
2267          RegClass rc = RegClass(src.type(), 1);
2268          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2269          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2270          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2271          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
2272          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2273          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2274
2275       } else {
2276          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2277          nir_print_instr(&instr->instr, stderr);
2278          fprintf(stderr, "\n");
2279       }
2280       break;
2281    }
2282    case nir_op_u2f16: {
2283       assert(dst.regClass() == v2b);
2284       Temp src = get_alu_src(ctx, instr->src[0]);
2285       if (instr->src[0].src.ssa->bit_size == 8)
2286          src = convert_int(bld, src, 8, 16, false);
2287       Temp tmp = bld.vop1(aco_opcode::v_cvt_f16_u16, bld.def(v1), src);
2288       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2289       break;
2290    }
2291    case nir_op_u2f32: {
2292       assert(dst.size() == 1);
2293       Temp src = get_alu_src(ctx, instr->src[0]);
2294       if (instr->src[0].src.ssa->bit_size == 8) {
2295          //TODO: we should use v_cvt_f32_ubyte1/v_cvt_f32_ubyte2/etc depending on the register assignment
2296          bld.vop1(aco_opcode::v_cvt_f32_ubyte0, Definition(dst), src);
2297       } else {
2298          if (instr->src[0].src.ssa->bit_size == 16)
2299             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2300          bld.vop1(aco_opcode::v_cvt_f32_u32, Definition(dst), src);
2301       }
2302       break;
2303    }
2304    case nir_op_u2f64: {
2305       if (instr->src[0].src.ssa->bit_size <= 32) {
2306          Temp src = get_alu_src(ctx, instr->src[0]);
2307          if (instr->src[0].src.ssa->bit_size <= 16)
2308             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, false);
2309          bld.vop1(aco_opcode::v_cvt_f64_u32, Definition(dst), src);
2310       } else if (instr->src[0].src.ssa->bit_size == 64) {
2311          Temp src = get_alu_src(ctx, instr->src[0]);
2312          RegClass rc = RegClass(src.type(), 1);
2313          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2314          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2315          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2316          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
2317          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2318          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2319       } else {
2320          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2321          nir_print_instr(&instr->instr, stderr);
2322          fprintf(stderr, "\n");
2323       }
2324       break;
2325    }
2326    case nir_op_f2i8:
2327    case nir_op_f2i16: {
2328       Temp src = get_alu_src(ctx, instr->src[0]);
2329       if (instr->src[0].src.ssa->bit_size == 16)
2330          src = bld.vop1(aco_opcode::v_cvt_i16_f16, bld.def(v1), src);
2331       else if (instr->src[0].src.ssa->bit_size == 32)
2332          src = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src);
2333       else
2334          src = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src);
2335
2336       if (dst.type() == RegType::vgpr)
2337          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
2338       else
2339          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2340       break;
2341    }
2342    case nir_op_f2u8:
2343    case nir_op_f2u16: {
2344       Temp src = get_alu_src(ctx, instr->src[0]);
2345       if (instr->src[0].src.ssa->bit_size == 16)
2346          src = bld.vop1(aco_opcode::v_cvt_u16_f16, bld.def(v1), src);
2347       else if (instr->src[0].src.ssa->bit_size == 32)
2348          src = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src);
2349       else
2350          src = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src);
2351
2352       if (dst.type() == RegType::vgpr)
2353          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
2354       else
2355          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2356       break;
2357    }
2358    case nir_op_f2i32: {
2359       Temp src = get_alu_src(ctx, instr->src[0]);
2360       if (instr->src[0].src.ssa->bit_size == 16) {
2361          Temp tmp = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2362          if (dst.type() == RegType::vgpr) {
2363             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), tmp);
2364          } else {
2365             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2366                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), tmp));
2367          }
2368       } else if (instr->src[0].src.ssa->bit_size == 32) {
2369          if (dst.type() == RegType::vgpr)
2370             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
2371          else
2372             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2373                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
2374
2375       } else if (instr->src[0].src.ssa->bit_size == 64) {
2376          if (dst.type() == RegType::vgpr)
2377             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
2378          else
2379             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2380                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
2381
2382       } else {
2383          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2384          nir_print_instr(&instr->instr, stderr);
2385          fprintf(stderr, "\n");
2386       }
2387       break;
2388    }
2389    case nir_op_f2u32: {
2390       Temp src = get_alu_src(ctx, instr->src[0]);
2391       if (instr->src[0].src.ssa->bit_size == 16) {
2392          Temp tmp = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2393          if (dst.type() == RegType::vgpr) {
2394             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), tmp);
2395          } else {
2396             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2397                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), tmp));
2398          }
2399       } else if (instr->src[0].src.ssa->bit_size == 32) {
2400          if (dst.type() == RegType::vgpr)
2401             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2402          else
2403             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2404                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2405
2406       } else if (instr->src[0].src.ssa->bit_size == 64) {
2407          if (dst.type() == RegType::vgpr)
2408             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2409          else
2410             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2411                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2412
2413       } else {
2414          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2415          nir_print_instr(&instr->instr, stderr);
2416          fprintf(stderr, "\n");
2417       }
2418       break;
2419    }
2420    case nir_op_f2i64: {
2421       Temp src = get_alu_src(ctx, instr->src[0]);
2422       if (instr->src[0].src.ssa->bit_size == 16)
2423          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2424
2425       if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::vgpr) {
2426          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2427          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2428          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2429          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2430          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2431          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2432          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2433          Temp new_exponent = bld.tmp(v1);
2434          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2435          if (ctx->program->chip_class >= GFX8)
2436             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2437          else
2438             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2439          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2440          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2441          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2442          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2443          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2444          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2445          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2446          Temp new_lower = bld.tmp(v1);
2447          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2448          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2449          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2450
2451       } else if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::sgpr) {
2452          if (src.type() == RegType::vgpr)
2453             src = bld.as_uniform(src);
2454          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2455          exponent = bld.sop2(aco_opcode::s_sub_i32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2456          exponent = bld.sop2(aco_opcode::s_max_i32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2457          exponent = bld.sop2(aco_opcode::s_min_i32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2458          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2459          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2460          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2461          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2462          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2463          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2464          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2465          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2466          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2467          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2468          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2469          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2470          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2471          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2472          Temp borrow = bld.tmp(s1);
2473          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2474          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2475          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2476
2477       } else if (instr->src[0].src.ssa->bit_size == 64) {
2478          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2479          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2480          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2481          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2482          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2483          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2484          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2485          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2486          if (dst.type() == RegType::sgpr) {
2487             lower = bld.as_uniform(lower);
2488             upper = bld.as_uniform(upper);
2489          }
2490          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2491
2492       } else {
2493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2494          nir_print_instr(&instr->instr, stderr);
2495          fprintf(stderr, "\n");
2496       }
2497       break;
2498    }
2499    case nir_op_f2u64: {
2500       Temp src = get_alu_src(ctx, instr->src[0]);
2501       if (instr->src[0].src.ssa->bit_size == 16)
2502          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2503
2504       if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::vgpr) {
2505          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2506          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2507          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2508          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2509          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2510          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2511          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2512          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2513          Temp new_exponent = bld.tmp(v1);
2514          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2515          if (ctx->program->chip_class >= GFX8)
2516             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2517          else
2518             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2519          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2520          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2521          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2522          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2523          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2524          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2525          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2526
2527       } else if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::sgpr) {
2528          if (src.type() == RegType::vgpr)
2529             src = bld.as_uniform(src);
2530          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2531          exponent = bld.sop2(aco_opcode::s_sub_i32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2532          exponent = bld.sop2(aco_opcode::s_max_i32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2533          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2534          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2535          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2536          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2537          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2538          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2539          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2540          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2541          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2542          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2543          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2544          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2545          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2546          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2547          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2548
2549       } else if (instr->src[0].src.ssa->bit_size == 64) {
2550          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2551          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2552          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2553          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2554          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2555          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2556          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2557          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2558          if (dst.type() == RegType::sgpr) {
2559             lower = bld.as_uniform(lower);
2560             upper = bld.as_uniform(upper);
2561          }
2562          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2563
2564       } else {
2565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2566          nir_print_instr(&instr->instr, stderr);
2567          fprintf(stderr, "\n");
2568       }
2569       break;
2570    }
2571    case nir_op_b2f16: {
2572       Temp src = get_alu_src(ctx, instr->src[0]);
2573       assert(src.regClass() == bld.lm);
2574
2575       if (dst.regClass() == s1) {
2576          src = bool_to_scalar_condition(ctx, src);
2577          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3c00u), src);
2578       } else if (dst.regClass() == v2b) {
2579          Temp one = bld.copy(bld.def(v1), Operand(0x3c00u));
2580          Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2581          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp);
2582       } else {
2583          unreachable("Wrong destination register class for nir_op_b2f16.");
2584       }
2585       break;
2586    }
2587    case nir_op_b2f32: {
2588       Temp src = get_alu_src(ctx, instr->src[0]);
2589       assert(src.regClass() == bld.lm);
2590
2591       if (dst.regClass() == s1) {
2592          src = bool_to_scalar_condition(ctx, src);
2593          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2594       } else if (dst.regClass() == v1) {
2595          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2596       } else {
2597          unreachable("Wrong destination register class for nir_op_b2f32.");
2598       }
2599       break;
2600    }
2601    case nir_op_b2f64: {
2602       Temp src = get_alu_src(ctx, instr->src[0]);
2603       assert(src.regClass() == bld.lm);
2604
2605       if (dst.regClass() == s2) {
2606          src = bool_to_scalar_condition(ctx, src);
2607          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2608       } else if (dst.regClass() == v2) {
2609          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2610          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2611          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2612       } else {
2613          unreachable("Wrong destination register class for nir_op_b2f64.");
2614       }
2615       break;
2616    }
2617    case nir_op_i2i8:
2618    case nir_op_i2i16:
2619    case nir_op_i2i32:
2620    case nir_op_i2i64: {
2621       convert_int(bld, get_alu_src(ctx, instr->src[0]),
2622                   instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, true, dst);
2623       break;
2624    }
2625    case nir_op_u2u8:
2626    case nir_op_u2u16:
2627    case nir_op_u2u32:
2628    case nir_op_u2u64: {
2629       convert_int(bld, get_alu_src(ctx, instr->src[0]),
2630                   instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, false, dst);
2631       break;
2632    }
2633    case nir_op_b2b32:
2634    case nir_op_b2i32: {
2635       Temp src = get_alu_src(ctx, instr->src[0]);
2636       assert(src.regClass() == bld.lm);
2637
2638       if (dst.regClass() == s1) {
2639          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2640          bool_to_scalar_condition(ctx, src, dst);
2641       } else if (dst.regClass() == v1) {
2642          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2643       } else {
2644          unreachable("Invalid register class for b2i32");
2645       }
2646       break;
2647    }
2648    case nir_op_b2b1:
2649    case nir_op_i2b1: {
2650       Temp src = get_alu_src(ctx, instr->src[0]);
2651       assert(dst.regClass() == bld.lm);
2652
2653       if (src.type() == RegType::vgpr) {
2654          assert(src.regClass() == v1 || src.regClass() == v2);
2655          assert(dst.regClass() == bld.lm);
2656          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2657                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2658       } else {
2659          assert(src.regClass() == s1 || src.regClass() == s2);
2660          Temp tmp;
2661          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2662             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2663          } else {
2664             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2665                            bld.scc(bld.def(s1)), Operand(0u), src);
2666          }
2667          bool_to_vector_condition(ctx, tmp, dst);
2668       }
2669       break;
2670    }
2671    case nir_op_pack_64_2x32_split: {
2672       Temp src0 = get_alu_src(ctx, instr->src[0]);
2673       Temp src1 = get_alu_src(ctx, instr->src[1]);
2674
2675       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2676       break;
2677    }
2678    case nir_op_unpack_64_2x32_split_x:
2679       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2680       break;
2681    case nir_op_unpack_64_2x32_split_y:
2682       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2683       break;
2684    case nir_op_unpack_32_2x16_split_x:
2685       if (dst.type() == RegType::vgpr) {
2686          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2687       } else {
2688          bld.copy(Definition(dst), get_alu_src(ctx, instr->src[0]));
2689       }
2690       break;
2691    case nir_op_unpack_32_2x16_split_y:
2692       if (dst.type() == RegType::vgpr) {
2693          bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2694       } else {
2695          bld.sop2(aco_opcode::s_bfe_u32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]), Operand(uint32_t(16 << 16 | 16)));
2696       }
2697       break;
2698    case nir_op_pack_32_2x16_split: {
2699       Temp src0 = get_alu_src(ctx, instr->src[0]);
2700       Temp src1 = get_alu_src(ctx, instr->src[1]);
2701       if (dst.regClass() == v1) {
2702          src0 = emit_extract_vector(ctx, src0, 0, v2b);
2703          src1 = emit_extract_vector(ctx, src1, 0, v2b);
2704          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2705       } else {
2706          src0 = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), src0, Operand(0xFFFFu));
2707          src1 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), src1, Operand(16u));
2708          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), src0, src1);
2709       }
2710       break;
2711    }
2712    case nir_op_pack_half_2x16: {
2713       Temp src = get_alu_src(ctx, instr->src[0], 2);
2714
2715       if (dst.regClass() == v1) {
2716          Temp src0 = bld.tmp(v1);
2717          Temp src1 = bld.tmp(v1);
2718          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2719          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2720             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2721          else
2722             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2723                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2724                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2725       } else {
2726          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2727          nir_print_instr(&instr->instr, stderr);
2728          fprintf(stderr, "\n");
2729       }
2730       break;
2731    }
2732    case nir_op_unpack_half_2x16_split_x: {
2733       if (dst.regClass() == v1) {
2734          Builder bld(ctx->program, ctx->block);
2735          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2736       } else {
2737          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2738          nir_print_instr(&instr->instr, stderr);
2739          fprintf(stderr, "\n");
2740       }
2741       break;
2742    }
2743    case nir_op_unpack_half_2x16_split_y: {
2744       if (dst.regClass() == v1) {
2745          Builder bld(ctx->program, ctx->block);
2746          /* TODO: use SDWA here */
2747          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2748                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2749       } else {
2750          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2751          nir_print_instr(&instr->instr, stderr);
2752          fprintf(stderr, "\n");
2753       }
2754       break;
2755    }
2756    case nir_op_fquantize2f16: {
2757       Temp src = get_alu_src(ctx, instr->src[0]);
2758       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2759       Temp f32, cmp_res;
2760
2761       if (ctx->program->chip_class >= GFX8) {
2762          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2763          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2764          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2765       } else {
2766          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2767           * so compare the result and flush to 0 if it's smaller.
2768           */
2769          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2770          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2771          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2772          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2773          cmp_res = vop3->definitions[0].getTemp();
2774       }
2775
2776       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2777          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2778          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2779       } else {
2780          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2781       }
2782       break;
2783    }
2784    case nir_op_bfm: {
2785       Temp bits = get_alu_src(ctx, instr->src[0]);
2786       Temp offset = get_alu_src(ctx, instr->src[1]);
2787
2788       if (dst.regClass() == s1) {
2789          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2790       } else if (dst.regClass() == v1) {
2791          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2792       } else {
2793          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2794          nir_print_instr(&instr->instr, stderr);
2795          fprintf(stderr, "\n");
2796       }
2797       break;
2798    }
2799    case nir_op_bitfield_select: {
2800       /* (mask & insert) | (~mask & base) */
2801       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2802       Temp insert = get_alu_src(ctx, instr->src[1]);
2803       Temp base = get_alu_src(ctx, instr->src[2]);
2804
2805       /* dst = (insert & bitmask) | (base & ~bitmask) */
2806       if (dst.regClass() == s1) {
2807          aco_ptr<Instruction> sop2;
2808          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2809          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2810          Operand lhs;
2811          if (const_insert && const_bitmask) {
2812             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2813          } else {
2814             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2815             lhs = Operand(insert);
2816          }
2817
2818          Operand rhs;
2819          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2820          if (const_base && const_bitmask) {
2821             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2822          } else {
2823             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2824             rhs = Operand(base);
2825          }
2826
2827          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2828
2829       } else if (dst.regClass() == v1) {
2830          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2831             base = as_vgpr(ctx, base);
2832          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2833             insert = as_vgpr(ctx, insert);
2834
2835          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2836
2837       } else {
2838          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2839          nir_print_instr(&instr->instr, stderr);
2840          fprintf(stderr, "\n");
2841       }
2842       break;
2843    }
2844    case nir_op_ubfe:
2845    case nir_op_ibfe: {
2846       Temp base = get_alu_src(ctx, instr->src[0]);
2847       Temp offset = get_alu_src(ctx, instr->src[1]);
2848       Temp bits = get_alu_src(ctx, instr->src[2]);
2849
2850       if (dst.type() == RegType::sgpr) {
2851          Operand extract;
2852          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2853          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2854          if (const_offset && const_bits) {
2855             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2856             extract = Operand(const_extract);
2857          } else {
2858             Operand width;
2859             if (const_bits) {
2860                width = Operand(const_bits->u32 << 16);
2861             } else {
2862                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2863             }
2864             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2865          }
2866
2867          aco_opcode opcode;
2868          if (dst.regClass() == s1) {
2869             if (instr->op == nir_op_ubfe)
2870                opcode = aco_opcode::s_bfe_u32;
2871             else
2872                opcode = aco_opcode::s_bfe_i32;
2873          } else if (dst.regClass() == s2) {
2874             if (instr->op == nir_op_ubfe)
2875                opcode = aco_opcode::s_bfe_u64;
2876             else
2877                opcode = aco_opcode::s_bfe_i64;
2878          } else {
2879             unreachable("Unsupported BFE bit size");
2880          }
2881
2882          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2883
2884       } else {
2885          aco_opcode opcode;
2886          if (dst.regClass() == v1) {
2887             if (instr->op == nir_op_ubfe)
2888                opcode = aco_opcode::v_bfe_u32;
2889             else
2890                opcode = aco_opcode::v_bfe_i32;
2891          } else {
2892             unreachable("Unsupported BFE bit size");
2893          }
2894
2895          emit_vop3a_instruction(ctx, instr, opcode, dst);
2896       }
2897       break;
2898    }
2899    case nir_op_bit_count: {
2900       Temp src = get_alu_src(ctx, instr->src[0]);
2901       if (src.regClass() == s1) {
2902          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2903       } else if (src.regClass() == v1) {
2904          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2905       } else if (src.regClass() == v2) {
2906          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2907                   emit_extract_vector(ctx, src, 1, v1),
2908                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2909                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2910       } else if (src.regClass() == s2) {
2911          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2912       } else {
2913          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2914          nir_print_instr(&instr->instr, stderr);
2915          fprintf(stderr, "\n");
2916       }
2917       break;
2918    }
2919    case nir_op_flt: {
2920       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f16, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2921       break;
2922    }
2923    case nir_op_fge: {
2924       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f16, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2925       break;
2926    }
2927    case nir_op_feq: {
2928       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f16, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2929       break;
2930    }
2931    case nir_op_fne: {
2932       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f16, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2933       break;
2934    }
2935    case nir_op_ilt: {
2936       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i16, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2937       break;
2938    }
2939    case nir_op_ige: {
2940       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i16, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2941       break;
2942    }
2943    case nir_op_ieq: {
2944       if (instr->src[0].src.ssa->bit_size == 1)
2945          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2946       else
2947          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i16, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2948                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2949       break;
2950    }
2951    case nir_op_ine: {
2952       if (instr->src[0].src.ssa->bit_size == 1)
2953          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2954       else
2955          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i16, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2956                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2957       break;
2958    }
2959    case nir_op_ult: {
2960       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u16, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2961       break;
2962    }
2963    case nir_op_uge: {
2964       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u16, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2965       break;
2966    }
2967    case nir_op_fddx:
2968    case nir_op_fddy:
2969    case nir_op_fddx_fine:
2970    case nir_op_fddy_fine:
2971    case nir_op_fddx_coarse:
2972    case nir_op_fddy_coarse: {
2973       Temp src = get_alu_src(ctx, instr->src[0]);
2974       uint16_t dpp_ctrl1, dpp_ctrl2;
2975       if (instr->op == nir_op_fddx_fine) {
2976          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2977          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2978       } else if (instr->op == nir_op_fddy_fine) {
2979          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2980          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2981       } else {
2982          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2983          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2984             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2985          else
2986             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2987       }
2988
2989       Temp tmp;
2990       if (ctx->program->chip_class >= GFX8) {
2991          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2992          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2993       } else {
2994          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2995          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2996          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2997       }
2998       emit_wqm(ctx, tmp, dst, true);
2999       break;
3000    }
3001    default:
3002       fprintf(stderr, "Unknown NIR ALU instr: ");
3003       nir_print_instr(&instr->instr, stderr);
3004       fprintf(stderr, "\n");
3005    }
3006 }
3007
3008 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
3009 {
3010    Temp dst = get_ssa_temp(ctx, &instr->def);
3011
3012    // TODO: we really want to have the resulting type as this would allow for 64bit literals
3013    // which get truncated the lsb if double and msb if int
3014    // for now, we only use s_mov_b64 with 64bit inline constants
3015    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
3016    assert(dst.type() == RegType::sgpr);
3017
3018    Builder bld(ctx->program, ctx->block);
3019
3020    if (instr->def.bit_size == 1) {
3021       assert(dst.regClass() == bld.lm);
3022       int val = instr->value[0].b ? -1 : 0;
3023       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
3024       bld.sop1(Builder::s_mov, Definition(dst), op);
3025    } else if (instr->def.bit_size == 8) {
3026       /* ensure that the value is correctly represented in the low byte of the register */
3027       bld.sopk(aco_opcode::s_movk_i32, Definition(dst), instr->value[0].u8);
3028    } else if (instr->def.bit_size == 16) {
3029       /* ensure that the value is correctly represented in the low half of the register */
3030       bld.sopk(aco_opcode::s_movk_i32, Definition(dst), instr->value[0].u16);
3031    } else if (dst.size() == 1) {
3032       bld.copy(Definition(dst), Operand(instr->value[0].u32));
3033    } else {
3034       assert(dst.size() != 1);
3035       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3036       if (instr->def.bit_size == 64)
3037          for (unsigned i = 0; i < dst.size(); i++)
3038             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
3039       else {
3040          for (unsigned i = 0; i < dst.size(); i++)
3041             vec->operands[i] = Operand{instr->value[i].u32};
3042       }
3043       vec->definitions[0] = Definition(dst);
3044       ctx->block->instructions.emplace_back(std::move(vec));
3045    }
3046 }
3047
3048 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
3049 {
3050    uint32_t new_mask = 0;
3051    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
3052       if (mask & (1u << i))
3053          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
3054    return new_mask;
3055 }
3056
3057 Operand load_lds_size_m0(isel_context *ctx)
3058 {
3059    /* TODO: m0 does not need to be initialized on GFX9+ */
3060    Builder bld(ctx->program, ctx->block);
3061    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
3062 }
3063
3064 Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
3065               Temp address, unsigned base_offset, unsigned align)
3066 {
3067    assert(util_is_power_of_two_nonzero(align) && align >= 4);
3068
3069    Builder bld(ctx->program, ctx->block);
3070
3071    Operand m = load_lds_size_m0(ctx);
3072
3073    unsigned num_components = dst.size() * 4u / elem_size_bytes;
3074    unsigned bytes_read = 0;
3075    unsigned result_size = 0;
3076    unsigned total_bytes = num_components * elem_size_bytes;
3077    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
3078    bool large_ds_read = ctx->options->chip_class >= GFX7;
3079    bool usable_read2 = ctx->options->chip_class >= GFX7;
3080
3081    while (bytes_read < total_bytes) {
3082       unsigned todo = total_bytes - bytes_read;
3083       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
3084       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
3085
3086       aco_opcode op = aco_opcode::last_opcode;
3087       bool read2 = false;
3088       if (todo >= 16 && aligned16 && large_ds_read) {
3089          op = aco_opcode::ds_read_b128;
3090          todo = 16;
3091       } else if (todo >= 16 && aligned8 && usable_read2) {
3092          op = aco_opcode::ds_read2_b64;
3093          read2 = true;
3094          todo = 16;
3095       } else if (todo >= 12 && aligned16 && large_ds_read) {
3096          op = aco_opcode::ds_read_b96;
3097          todo = 12;
3098       } else if (todo >= 8 && aligned8) {
3099          op = aco_opcode::ds_read_b64;
3100          todo = 8;
3101       } else if (todo >= 8 && usable_read2) {
3102          op = aco_opcode::ds_read2_b32;
3103          read2 = true;
3104          todo = 8;
3105       } else if (todo >= 4) {
3106          op = aco_opcode::ds_read_b32;
3107          todo = 4;
3108       } else {
3109          assert(false);
3110       }
3111       assert(todo % elem_size_bytes == 0);
3112       unsigned num_elements = todo / elem_size_bytes;
3113       unsigned offset = base_offset + bytes_read;
3114       unsigned max_offset = read2 ? 1019 : 65535;
3115
3116       Temp address_offset = address;
3117       if (offset > max_offset) {
3118          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3119          offset = bytes_read;
3120       }
3121       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
3122
3123       Temp res;
3124       if (num_components == 1 && dst.type() == RegType::vgpr)
3125          res = dst;
3126       else
3127          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
3128
3129       if (read2)
3130          res = bld.ds(op, Definition(res), address_offset, m, offset / (todo / 2), (offset / (todo / 2)) + 1);
3131       else
3132          res = bld.ds(op, Definition(res), address_offset, m, offset);
3133
3134       if (num_components == 1) {
3135          assert(todo == total_bytes);
3136          if (dst.type() == RegType::sgpr)
3137             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
3138          return dst;
3139       }
3140
3141       if (dst.type() == RegType::sgpr) {
3142          Temp new_res = bld.tmp(RegType::sgpr, res.size());
3143          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
3144          res = new_res;
3145       }
3146
3147       if (num_elements == 1) {
3148          result[result_size++] = res;
3149       } else {
3150          assert(res != dst && res.size() % num_elements == 0);
3151          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
3152          split->operands[0] = Operand(res);
3153          for (unsigned i = 0; i < num_elements; i++)
3154             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
3155          ctx->block->instructions.emplace_back(std::move(split));
3156       }
3157
3158       bytes_read += todo;
3159    }
3160
3161    assert(result_size == num_components && result_size > 1);
3162    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
3163    for (unsigned i = 0; i < result_size; i++)
3164       vec->operands[i] = Operand(result[i]);
3165    vec->definitions[0] = Definition(dst);
3166    ctx->block->instructions.emplace_back(std::move(vec));
3167    ctx->allocated_vec.emplace(dst.id(), result);
3168
3169    return dst;
3170 }
3171
3172 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
3173 {
3174    if (start == 0 && size == data.size())
3175       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
3176
3177    unsigned size_hint = 1;
3178    auto it = ctx->allocated_vec.find(data.id());
3179    if (it != ctx->allocated_vec.end())
3180       size_hint = it->second[0].size();
3181    if (size % size_hint || start % size_hint)
3182       size_hint = 1;
3183
3184    start /= size_hint;
3185    size /= size_hint;
3186
3187    Temp elems[size];
3188    for (unsigned i = 0; i < size; i++)
3189       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
3190
3191    if (size == 1)
3192       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
3193
3194    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
3195    for (unsigned i = 0; i < size; i++)
3196       vec->operands[i] = Operand(elems[i]);
3197    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
3198    vec->definitions[0] = Definition(res);
3199    ctx->block->instructions.emplace_back(std::move(vec));
3200    return res;
3201 }
3202
3203 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
3204 {
3205    Builder bld(ctx->program, ctx->block);
3206    unsigned bytes_written = 0;
3207    bool large_ds_write = ctx->options->chip_class >= GFX7;
3208    bool usable_write2 = ctx->options->chip_class >= GFX7;
3209
3210    while (bytes_written < total_size * 4) {
3211       unsigned todo = total_size * 4 - bytes_written;
3212       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
3213       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
3214
3215       aco_opcode op = aco_opcode::last_opcode;
3216       bool write2 = false;
3217       unsigned size = 0;
3218       if (todo >= 16 && aligned16 && large_ds_write) {
3219          op = aco_opcode::ds_write_b128;
3220          size = 4;
3221       } else if (todo >= 16 && aligned8 && usable_write2) {
3222          op = aco_opcode::ds_write2_b64;
3223          write2 = true;
3224          size = 4;
3225       } else if (todo >= 12 && aligned16 && large_ds_write) {
3226          op = aco_opcode::ds_write_b96;
3227          size = 3;
3228       } else if (todo >= 8 && aligned8) {
3229          op = aco_opcode::ds_write_b64;
3230          size = 2;
3231       } else if (todo >= 8 && usable_write2) {
3232          op = aco_opcode::ds_write2_b32;
3233          write2 = true;
3234          size = 2;
3235       } else if (todo >= 4) {
3236          op = aco_opcode::ds_write_b32;
3237          size = 1;
3238       } else {
3239          assert(false);
3240       }
3241
3242       unsigned offset = offset0 + offset1 + bytes_written;
3243       unsigned max_offset = write2 ? 1020 : 65535;
3244       Temp address_offset = address;
3245       if (offset > max_offset) {
3246          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
3247          offset = offset1 + bytes_written;
3248       }
3249       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
3250
3251       if (write2) {
3252          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
3253          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
3254          bld.ds(op, address_offset, val0, val1, m, offset / size / 2, (offset / size / 2) + 1);
3255       } else {
3256          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
3257          bld.ds(op, address_offset, val, m, offset);
3258       }
3259
3260       bytes_written += size * 4;
3261    }
3262 }
3263
3264 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
3265                Temp address, unsigned base_offset, unsigned align)
3266 {
3267    assert(util_is_power_of_two_nonzero(align) && align >= 4);
3268    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3269
3270    Operand m = load_lds_size_m0(ctx);
3271
3272    /* we need at most two stores, assuming that the writemask is at most 4 bits wide */
3273    assert(wrmask <= 0x0f);
3274    int start[2], count[2];
3275    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
3276    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
3277    assert(wrmask == 0);
3278
3279    /* one combined store is sufficient */
3280    if (count[0] == count[1] && (align % elem_size_bytes) == 0 && (base_offset % elem_size_bytes) == 0) {
3281       Builder bld(ctx->program, ctx->block);
3282
3283       Temp address_offset = address;
3284       if ((base_offset / elem_size_bytes) + start[1] > 255) {
3285          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3286          base_offset = 0;
3287       }
3288
3289       assert(count[0] == 1);
3290       RegClass xtract_rc(RegType::vgpr, elem_size_bytes / 4);
3291
3292       Temp val0 = emit_extract_vector(ctx, data, start[0], xtract_rc);
3293       Temp val1 = emit_extract_vector(ctx, data, start[1], xtract_rc);
3294       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
3295       base_offset = base_offset / elem_size_bytes;
3296       bld.ds(op, address_offset, val0, val1, m,
3297              base_offset + start[0], base_offset + start[1]);
3298       return;
3299    }
3300
3301    for (unsigned i = 0; i < 2; i++) {
3302       if (count[i] == 0)
3303          continue;
3304
3305       unsigned elem_size_words = elem_size_bytes / 4;
3306       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
3307                       base_offset, start[i] * elem_size_bytes, align);
3308    }
3309    return;
3310 }
3311
3312 unsigned calculate_lds_alignment(isel_context *ctx, unsigned const_offset)
3313 {
3314    unsigned align = 16;
3315    if (const_offset)
3316       align = std::min(align, 1u << (ffs(const_offset) - 1));
3317
3318    return align;
3319 }
3320
3321
3322 Temp create_vec_from_array(isel_context *ctx, Temp arr[], unsigned cnt, RegType reg_type, unsigned elem_size_bytes,
3323                            unsigned split_cnt = 0u, Temp dst = Temp())
3324 {
3325    Builder bld(ctx->program, ctx->block);
3326    unsigned dword_size = elem_size_bytes / 4;
3327
3328    if (!dst.id())
3329       dst = bld.tmp(RegClass(reg_type, cnt * dword_size));
3330
3331    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
3332    aco_ptr<Pseudo_instruction> instr {create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, cnt, 1)};
3333    instr->definitions[0] = Definition(dst);
3334
3335    for (unsigned i = 0; i < cnt; ++i) {
3336       if (arr[i].id()) {
3337          assert(arr[i].size() == dword_size);
3338          allocated_vec[i] = arr[i];
3339          instr->operands[i] = Operand(arr[i]);
3340       } else {
3341          Temp zero = bld.copy(bld.def(RegClass(reg_type, dword_size)), Operand(0u, dword_size == 2));
3342          allocated_vec[i] = zero;
3343          instr->operands[i] = Operand(zero);
3344       }
3345    }
3346
3347    bld.insert(std::move(instr));
3348
3349    if (split_cnt)
3350       emit_split_vector(ctx, dst, split_cnt);
3351    else
3352       ctx->allocated_vec.emplace(dst.id(), allocated_vec); /* emit_split_vector already does this */
3353
3354    return dst;
3355 }
3356
3357 inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, unsigned const_offset)
3358 {
3359    if (const_offset >= 4096) {
3360       unsigned excess_const_offset = const_offset / 4096u * 4096u;
3361       const_offset %= 4096u;
3362
3363       if (!voffset.id())
3364          voffset = bld.copy(bld.def(v1), Operand(excess_const_offset));
3365       else if (unlikely(voffset.regClass() == s1))
3366          voffset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), Operand(excess_const_offset), Operand(voffset));
3367       else if (likely(voffset.regClass() == v1))
3368          voffset = bld.vadd32(bld.def(v1), Operand(voffset), Operand(excess_const_offset));
3369       else
3370          unreachable("Unsupported register class of voffset");
3371    }
3372
3373    return const_offset;
3374 }
3375
3376 void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata,
3377                              unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false)
3378 {
3379    assert(vdata.id());
3380    assert(vdata.size() != 3 || ctx->program->chip_class != GFX6);
3381    assert(vdata.size() >= 1 && vdata.size() <= 4);
3382
3383    Builder bld(ctx->program, ctx->block);
3384    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_store_dword + vdata.size() - 1);
3385    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3386
3387    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3388    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3389    Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset,
3390                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3391                                  /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc);
3392
3393    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3394 }
3395
3396 void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset,
3397                                    unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask,
3398                                    bool allow_combining = true, bool reorder = true, bool slc = false)
3399 {
3400    Builder bld(ctx->program, ctx->block);
3401    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3402    assert(write_mask);
3403
3404    if (elem_size_bytes == 8) {
3405       elem_size_bytes = 4;
3406       write_mask = widen_mask(write_mask, 2);
3407    }
3408
3409    while (write_mask) {
3410       int start = 0;
3411       int count = 0;
3412       u_bit_scan_consecutive_range(&write_mask, &start, &count);
3413       assert(count > 0);
3414       assert(start >= 0);
3415
3416       while (count > 0) {
3417          unsigned sub_count = allow_combining ? MIN2(count, 4) : 1;
3418          unsigned const_offset = (unsigned) start * elem_size_bytes + base_const_offset;
3419
3420          /* GFX6 doesn't have buffer_store_dwordx3, so make sure not to emit that here either. */
3421          if (unlikely(ctx->program->chip_class == GFX6 && sub_count == 3))
3422             sub_count = 2;
3423
3424          Temp elem = extract_subvector(ctx, src, start, sub_count, RegType::vgpr);
3425          emit_single_mubuf_store(ctx, descriptor, voffset, soffset, elem, const_offset, reorder, slc);
3426
3427          count -= sub_count;
3428          start += sub_count;
3429       }
3430
3431       assert(count == 0);
3432    }
3433 }
3434
3435 Temp emit_single_mubuf_load(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset,
3436                             unsigned const_offset, unsigned size_dwords, bool allow_reorder = true)
3437 {
3438    assert(size_dwords != 3 || ctx->program->chip_class != GFX6);
3439    assert(size_dwords >= 1 && size_dwords <= 4);
3440
3441    Builder bld(ctx->program, ctx->block);
3442    Temp vdata = bld.tmp(RegClass(RegType::vgpr, size_dwords));
3443    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_load_dword + size_dwords - 1);
3444    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3445
3446    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3447    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3448    Builder::Result r = bld.mubuf(op, Definition(vdata), Operand(descriptor), voffset_op, soffset_op, const_offset,
3449                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3450                                  /* disable_wqm */ false, /* glc */ true,
3451                                  /* dlc*/ ctx->program->chip_class >= GFX10, /* slc */ false);
3452
3453    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3454
3455    return vdata;
3456 }
3457
3458 void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, Temp soffset,
3459                      unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components,
3460                      unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true)
3461 {
3462    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3463    assert((num_components * elem_size_bytes / 4) == dst.size());
3464    assert(!!stride != allow_combining);
3465
3466    Builder bld(ctx->program, ctx->block);
3467    unsigned split_cnt = num_components;
3468
3469    if (elem_size_bytes == 8) {
3470       elem_size_bytes = 4;
3471       num_components *= 2;
3472    }
3473
3474    if (!stride)
3475       stride = elem_size_bytes;
3476
3477    unsigned load_size = 1;
3478    if (allow_combining) {
3479       if ((num_components % 4) == 0)
3480          load_size = 4;
3481       else if ((num_components % 3) == 0 && ctx->program->chip_class != GFX6)
3482          load_size = 3;
3483       else if ((num_components % 2) == 0)
3484          load_size = 2;
3485    }
3486
3487    unsigned num_loads = num_components / load_size;
3488    std::array<Temp, NIR_MAX_VEC_COMPONENTS> elems;
3489
3490    for (unsigned i = 0; i < num_loads; ++i) {
3491       unsigned const_offset = i * stride * load_size + base_const_offset;
3492       elems[i] = emit_single_mubuf_load(ctx, descriptor, voffset, soffset, const_offset, load_size, allow_reorder);
3493    }
3494
3495    create_vec_from_array(ctx, elems.data(), num_loads, RegType::vgpr, load_size * 4u, split_cnt, dst);
3496 }
3497
3498 std::pair<Temp, unsigned> offset_add_from_nir(isel_context *ctx, const std::pair<Temp, unsigned> &base_offset, nir_src *off_src, unsigned stride = 1u)
3499 {
3500    Builder bld(ctx->program, ctx->block);
3501    Temp offset = base_offset.first;
3502    unsigned const_offset = base_offset.second;
3503
3504    if (!nir_src_is_const(*off_src)) {
3505       Temp indirect_offset_arg = get_ssa_temp(ctx, off_src->ssa);
3506       Temp with_stride;
3507
3508       /* Calculate indirect offset with stride */
3509       if (likely(indirect_offset_arg.regClass() == v1))
3510          with_stride = bld.v_mul_imm(bld.def(v1), indirect_offset_arg, stride);
3511       else if (indirect_offset_arg.regClass() == s1)
3512          with_stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), indirect_offset_arg);
3513       else
3514          unreachable("Unsupported register class of indirect offset");
3515
3516       /* Add to the supplied base offset */
3517       if (offset.id() == 0)
3518          offset = with_stride;
3519       else if (unlikely(offset.regClass() == s1 && with_stride.regClass() == s1))
3520          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), with_stride, offset);
3521       else if (offset.size() == 1 && with_stride.size() == 1)
3522          offset = bld.vadd32(bld.def(v1), with_stride, offset);
3523       else
3524          unreachable("Unsupported register class of indirect offset");
3525    } else {
3526       unsigned const_offset_arg = nir_src_as_uint(*off_src);
3527       const_offset += const_offset_arg * stride;
3528    }
3529
3530    return std::make_pair(offset, const_offset);
3531 }
3532
3533 std::pair<Temp, unsigned> offset_add(isel_context *ctx, const std::pair<Temp, unsigned> &off1, const std::pair<Temp, unsigned> &off2)
3534 {
3535    Builder bld(ctx->program, ctx->block);
3536    Temp offset;
3537
3538    if (off1.first.id() && off2.first.id()) {
3539       if (unlikely(off1.first.regClass() == s1 && off2.first.regClass() == s1))
3540          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), off1.first, off2.first);
3541       else if (off1.first.size() == 1 && off2.first.size() == 1)
3542          offset = bld.vadd32(bld.def(v1), off1.first, off2.first);
3543       else
3544          unreachable("Unsupported register class of indirect offset");
3545    } else {
3546       offset = off1.first.id() ? off1.first : off2.first;
3547    }
3548
3549    return std::make_pair(offset, off1.second + off2.second);
3550 }
3551
3552 std::pair<Temp, unsigned> offset_mul(isel_context *ctx, const std::pair<Temp, unsigned> &offs, unsigned multiplier)
3553 {
3554    Builder bld(ctx->program, ctx->block);
3555    unsigned const_offset = offs.second * multiplier;
3556
3557    if (!offs.first.id())
3558       return std::make_pair(offs.first, const_offset);
3559
3560    Temp offset = unlikely(offs.first.regClass() == s1)
3561                  ? bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(multiplier), offs.first)
3562                  : bld.v_mul_imm(bld.def(v1), offs.first, multiplier);
3563
3564    return std::make_pair(offset, const_offset);
3565 }
3566
3567 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride, unsigned component_stride)
3568 {
3569    Builder bld(ctx->program, ctx->block);
3570
3571    /* base is the driver_location, which is already multiplied by 4, so is in dwords */
3572    unsigned const_offset = nir_intrinsic_base(instr) * base_stride;
3573    /* component is in bytes */
3574    const_offset += nir_intrinsic_component(instr) * component_stride;
3575
3576    /* offset should be interpreted in relation to the base, so the instruction effectively reads/writes another input/output when it has an offset */
3577    nir_src *off_src = nir_get_io_offset_src(instr);
3578    return offset_add_from_nir(ctx, std::make_pair(Temp(), const_offset), off_src, 4u * base_stride);
3579 }
3580
3581 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned stride = 1u)
3582 {
3583    return get_intrinsic_io_basic_offset(ctx, instr, stride, stride);
3584 }
3585
3586 Temp get_tess_rel_patch_id(isel_context *ctx)
3587 {
3588    Builder bld(ctx->program, ctx->block);
3589
3590    switch (ctx->shader->info.stage) {
3591    case MESA_SHADER_TESS_CTRL:
3592       return bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffu),
3593                       get_arg(ctx, ctx->args->ac.tcs_rel_ids));
3594    case MESA_SHADER_TESS_EVAL:
3595       return get_arg(ctx, ctx->args->tes_rel_patch_id);
3596    default:
3597       unreachable("Unsupported stage in get_tess_rel_patch_id");
3598    }
3599 }
3600
3601 std::pair<Temp, unsigned> get_tcs_per_vertex_input_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3602 {
3603    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3604    Builder bld(ctx->program, ctx->block);
3605
3606    uint32_t tcs_in_patch_stride = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 4;
3607    uint32_t tcs_in_vertex_stride = ctx->tcs_num_inputs * 4;
3608
3609    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr);
3610
3611    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3612    offs = offset_add_from_nir(ctx, offs, vertex_index_src, tcs_in_vertex_stride);
3613
3614    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3615    Temp tcs_in_current_patch_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, tcs_in_patch_stride);
3616    offs = offset_add(ctx, offs, std::make_pair(tcs_in_current_patch_offset, 0));
3617
3618    return offset_mul(ctx, offs, 4u);
3619 }
3620
3621 std::pair<Temp, unsigned> get_tcs_output_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, bool per_vertex = false)
3622 {
3623    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3624    Builder bld(ctx->program, ctx->block);
3625
3626    uint32_t input_patch_size = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 16;
3627    uint32_t num_tcs_outputs = util_last_bit64(ctx->args->shader_info->tcs.outputs_written);
3628    uint32_t num_tcs_patch_outputs = util_last_bit64(ctx->args->shader_info->tcs.patch_outputs_written);
3629    uint32_t output_vertex_size = num_tcs_outputs * 16;
3630    uint32_t pervertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3631    uint32_t output_patch_stride = pervertex_output_patch_size + num_tcs_patch_outputs * 16;
3632
3633    std::pair<Temp, unsigned> offs = instr
3634                                     ? get_intrinsic_io_basic_offset(ctx, instr, 4u)
3635                                     : std::make_pair(Temp(), 0u);
3636
3637    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3638    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, output_patch_stride);
3639
3640    if (per_vertex) {
3641       assert(instr);
3642
3643       nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3644       offs = offset_add_from_nir(ctx, offs, vertex_index_src, output_vertex_size);
3645
3646       uint32_t output_patch0_offset = (input_patch_size * ctx->tcs_num_patches);
3647       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_offset));
3648    } else {
3649       uint32_t output_patch0_patch_data_offset = (input_patch_size * ctx->tcs_num_patches + pervertex_output_patch_size);
3650       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_patch_data_offset));
3651    }
3652
3653    return offs;
3654 }
3655
3656 std::pair<Temp, unsigned> get_tcs_per_vertex_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3657 {
3658    Builder bld(ctx->program, ctx->block);
3659
3660    unsigned vertices_per_patch = ctx->shader->info.tess.tcs_vertices_out;
3661    unsigned attr_stride = vertices_per_patch * ctx->tcs_num_patches;
3662
3663    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u);
3664
3665    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3666    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, vertices_per_patch * 16u);
3667    offs = offset_add(ctx, offs, std::make_pair(patch_off, 0u));
3668
3669    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3670    offs = offset_add_from_nir(ctx, offs, vertex_index_src, 16u);
3671
3672    return offs;
3673 }
3674
3675 std::pair<Temp, unsigned> get_tcs_per_patch_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, unsigned const_base_offset = 0u)
3676 {
3677    Builder bld(ctx->program, ctx->block);
3678
3679    unsigned num_tcs_outputs = ctx->shader->info.stage == MESA_SHADER_TESS_CTRL
3680                               ? util_last_bit64(ctx->args->shader_info->tcs.outputs_written)
3681                               : ctx->args->options->key.tes.tcs_num_outputs;
3682
3683    unsigned output_vertex_size = num_tcs_outputs * 16;
3684    unsigned per_vertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3685    unsigned per_patch_data_offset = per_vertex_output_patch_size * ctx->tcs_num_patches;
3686    unsigned attr_stride = ctx->tcs_num_patches;
3687
3688    std::pair<Temp, unsigned> offs = instr
3689                                     ? get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u)
3690                                     : std::make_pair(Temp(), 0u);
3691
3692    if (const_base_offset)
3693       offs.second += const_base_offset * attr_stride;
3694
3695    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3696    Temp patch_off = bld.v_mul_imm(bld.def(v1), rel_patch_id, 16u);
3697    offs = offset_add(ctx, offs, std::make_pair(patch_off, per_patch_data_offset));
3698
3699    return offs;
3700 }
3701
3702 bool tcs_driver_location_matches_api_mask(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex, uint64_t mask, bool *indirect)
3703 {
3704    unsigned off = nir_intrinsic_base(instr) * 4u;
3705    nir_src *off_src = nir_get_io_offset_src(instr);
3706
3707    if (!nir_src_is_const(*off_src)) {
3708       *indirect = true;
3709       return false;
3710    }
3711
3712    *indirect = false;
3713    off += nir_src_as_uint(*off_src) * 16u;
3714
3715    while (mask) {
3716       unsigned slot = u_bit_scan64(&mask) + (per_vertex ? 0 : VARYING_SLOT_PATCH0);
3717       if (off == shader_io_get_unique_index((gl_varying_slot) slot) * 16u)
3718          return true;
3719    }
3720
3721    return false;
3722 }
3723
3724 bool store_output_to_temps(isel_context *ctx, nir_intrinsic_instr *instr)
3725 {
3726    unsigned write_mask = nir_intrinsic_write_mask(instr);
3727    unsigned component = nir_intrinsic_component(instr);
3728    unsigned idx = nir_intrinsic_base(instr) + component;
3729
3730    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3731    if (off_instr->type != nir_instr_type_load_const)
3732       return false;
3733
3734    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3735    idx += nir_src_as_uint(instr->src[1]) * 4u;
3736
3737    if (instr->src[0].ssa->bit_size == 64)
3738       write_mask = widen_mask(write_mask, 2);
3739
3740    for (unsigned i = 0; i < 8; ++i) {
3741       if (write_mask & (1 << i)) {
3742          ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
3743          ctx->outputs.temps[idx] = emit_extract_vector(ctx, src, i, v1);
3744       }
3745       idx++;
3746    }
3747
3748    return true;
3749 }
3750
3751 bool load_input_from_temps(isel_context *ctx, nir_intrinsic_instr *instr, Temp dst)
3752 {
3753    /* Only TCS per-vertex inputs are supported by this function.
3754     * Per-vertex inputs only match between the VS/TCS invocation id when the number of invocations is the same.
3755     */
3756    if (ctx->shader->info.stage != MESA_SHADER_TESS_CTRL || !ctx->tcs_in_out_eq)
3757       return false;
3758
3759    nir_src *off_src = nir_get_io_offset_src(instr);
3760    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3761    nir_instr *vertex_index_instr = vertex_index_src->ssa->parent_instr;
3762    bool can_use_temps = nir_src_is_const(*off_src) &&
3763                         vertex_index_instr->type == nir_instr_type_intrinsic &&
3764                         nir_instr_as_intrinsic(vertex_index_instr)->intrinsic == nir_intrinsic_load_invocation_id;
3765
3766    if (!can_use_temps)
3767       return false;
3768
3769    unsigned idx = nir_intrinsic_base(instr) + nir_intrinsic_component(instr) + 4 * nir_src_as_uint(*off_src);
3770    Temp *src = &ctx->inputs.temps[idx];
3771    Temp vec = create_vec_from_array(ctx, src, dst.size(), dst.regClass().type(), 4u);
3772    assert(vec.size() == dst.size());
3773
3774    Builder bld(ctx->program, ctx->block);
3775    bld.copy(Definition(dst), vec);
3776    return true;
3777 }
3778
3779 void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr)
3780 {
3781    Builder bld(ctx->program, ctx->block);
3782
3783    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, 4u);
3784    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3785    unsigned write_mask = nir_intrinsic_write_mask(instr);
3786    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
3787
3788    if (ctx->tcs_in_out_eq && store_output_to_temps(ctx, instr)) {
3789       /* When the TCS only reads this output directly and for the same vertices as its invocation id, it is unnecessary to store the VS output to LDS. */
3790       bool indirect_write;
3791       bool temp_only_input = tcs_driver_location_matches_api_mask(ctx, instr, true, ctx->tcs_temp_only_inputs, &indirect_write);
3792       if (temp_only_input && !indirect_write)
3793          return;
3794    }
3795
3796    if (ctx->stage == vertex_es || ctx->stage == tess_eval_es) {
3797       /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */
3798       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
3799       Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset);
3800       store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true);
3801    } else {
3802       Temp lds_base;
3803
3804       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
3805          /* GFX9+: ES stage is merged into GS, data is passed between them using LDS. */
3806          unsigned itemsize = ctx->stage == vertex_geometry_gs
3807                              ? ctx->program->info->vs.es_info.esgs_itemsize
3808                              : ctx->program->info->tes.es_info.esgs_itemsize;
3809          Temp thread_id = emit_mbcnt(ctx, bld.def(v1));
3810          Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
3811          Temp vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), thread_id,
3812                                bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
3813          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
3814       } else if (ctx->stage == vertex_ls || ctx->stage == vertex_tess_control_hs) {
3815          /* GFX6-8: VS runs on LS stage when tessellation is used, but LS shares LDS space with HS.
3816           * GFX9+: LS is merged into HS, but still uses the same LDS layout.
3817           */
3818          unsigned num_tcs_inputs = util_last_bit64(ctx->args->shader_info->vs.ls_outputs_written);
3819          Temp vertex_idx = get_arg(ctx, ctx->args->rel_auto_id);
3820          lds_base = bld.v_mul_imm(bld.def(v1), vertex_idx, num_tcs_inputs * 16u);
3821       } else {
3822          unreachable("Invalid LS or ES stage");
3823       }
3824
3825       offs = offset_add(ctx, offs, std::make_pair(lds_base, 0u));
3826       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3827       store_lds(ctx, elem_size_bytes, src, write_mask, offs.first, offs.second, lds_align);
3828    }
3829 }
3830
3831 bool should_write_tcs_patch_output_to_vmem(isel_context *ctx, nir_intrinsic_instr *instr)
3832 {
3833    unsigned off = nir_intrinsic_base(instr) * 4u;
3834    return off != ctx->tcs_tess_lvl_out_loc &&
3835           off != ctx->tcs_tess_lvl_in_loc;
3836 }
3837
3838 bool should_write_tcs_output_to_lds(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3839 {
3840    /* When none of the appropriate outputs are read, we are OK to never write to LDS */
3841    if (per_vertex ? ctx->shader->info.outputs_read == 0U : ctx->shader->info.patch_outputs_read == 0u)
3842       return false;
3843
3844    uint64_t mask = per_vertex
3845                    ? ctx->shader->info.outputs_read
3846                    : ctx->shader->info.patch_outputs_read;
3847    bool indirect_write;
3848    bool output_read = tcs_driver_location_matches_api_mask(ctx, instr, per_vertex, mask, &indirect_write);
3849    return indirect_write || output_read;
3850 }
3851
3852 void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3853 {
3854    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3855    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3856
3857    Builder bld(ctx->program, ctx->block);
3858
3859    Temp store_val = get_ssa_temp(ctx, instr->src[0].ssa);
3860    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3861    unsigned write_mask = nir_intrinsic_write_mask(instr);
3862
3863    /* Only write to VMEM if the output is per-vertex or it's per-patch non tess factor */
3864    bool write_to_vmem = per_vertex || should_write_tcs_patch_output_to_vmem(ctx, instr);
3865    /* Only write to LDS if the output is read by the shader, or it's per-patch tess factor */
3866    bool write_to_lds = !write_to_vmem || should_write_tcs_output_to_lds(ctx, instr, per_vertex);
3867
3868    if (write_to_vmem) {
3869       std::pair<Temp, unsigned> vmem_offs = per_vertex
3870                                             ? get_tcs_per_vertex_output_vmem_offset(ctx, instr)
3871                                             : get_tcs_per_patch_output_vmem_offset(ctx, instr);
3872
3873       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
3874       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
3875       store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, false);
3876    }
3877
3878    if (write_to_lds) {
3879       std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3880       unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3881       store_lds(ctx, elem_size_bytes, store_val, write_mask, lds_offs.first, lds_offs.second, lds_align);
3882    }
3883 }
3884
3885 void visit_load_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3886 {
3887    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3888    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3889
3890    Builder bld(ctx->program, ctx->block);
3891
3892    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3893    std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3894    unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3895    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3896
3897    load_lds(ctx, elem_size_bytes, dst, lds_offs.first, lds_offs.second, lds_align);
3898 }
3899
3900 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
3901 {
3902    if (ctx->stage == vertex_vs ||
3903        ctx->stage == tess_eval_vs ||
3904        ctx->stage == fragment_fs ||
3905        ctx->stage == ngg_vertex_gs ||
3906        ctx->stage == ngg_tess_eval_gs ||
3907        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
3908       bool stored_to_temps = store_output_to_temps(ctx, instr);
3909       if (!stored_to_temps) {
3910          fprintf(stderr, "Unimplemented output offset instruction:\n");
3911          nir_print_instr(instr->src[1].ssa->parent_instr, stderr);
3912          fprintf(stderr, "\n");
3913          abort();
3914       }
3915    } else if (ctx->stage == vertex_es ||
3916               ctx->stage == vertex_ls ||
3917               ctx->stage == tess_eval_es ||
3918               (ctx->stage == vertex_tess_control_hs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3919               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3920               (ctx->stage == tess_eval_geometry_gs && ctx->shader->info.stage == MESA_SHADER_TESS_EVAL)) {
3921       visit_store_ls_or_es_output(ctx, instr);
3922    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
3923       visit_store_tcs_output(ctx, instr, false);
3924    } else {
3925       unreachable("Shader stage not implemented");
3926    }
3927 }
3928
3929 void visit_load_output(isel_context *ctx, nir_intrinsic_instr *instr)
3930 {
3931    visit_load_tcs_output(ctx, instr, false);
3932 }
3933
3934 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3935 {
3936    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3937    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3938
3939    Builder bld(ctx->program, ctx->block);
3940    Builder::Result interp_p1 = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3941    if (ctx->program->has_16bank_lds)
3942       interp_p1.instr->operands[0].setLateKill(true);
3943    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), interp_p1, idx, component);
3944 }
3945
3946 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3947 {
3948    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3949    for (unsigned i = 0; i < num_components; i++)
3950       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3951    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3952       assert(num_components == 4);
3953       Builder bld(ctx->program, ctx->block);
3954       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3955    }
3956
3957    for (Operand& op : vec->operands)
3958       op = op.isUndefined() ? Operand(0u) : op;
3959
3960    vec->definitions[0] = Definition(dst);
3961    ctx->block->instructions.emplace_back(std::move(vec));
3962    emit_split_vector(ctx, dst, num_components);
3963    return;
3964 }
3965
3966 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3967 {
3968    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3969    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3970    unsigned idx = nir_intrinsic_base(instr);
3971    unsigned component = nir_intrinsic_component(instr);
3972    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3973
3974    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3975    if (offset) {
3976       assert(offset->u32 == 0);
3977    } else {
3978       /* the lower 15bit of the prim_mask contain the offset into LDS
3979        * while the upper bits contain the number of prims */
3980       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3981       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3982       Builder bld(ctx->program, ctx->block);
3983       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3984       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3985       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3986       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3987       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3988    }
3989
3990    if (instr->dest.ssa.num_components == 1) {
3991       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3992    } else {
3993       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3994       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3995       {
3996          Temp tmp = {ctx->program->allocateId(), v1};
3997          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3998          vec->operands[i] = Operand(tmp);
3999       }
4000       vec->definitions[0] = Definition(dst);
4001       ctx->block->instructions.emplace_back(std::move(vec));
4002    }
4003 }
4004
4005 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
4006                              unsigned offset, unsigned stride, unsigned channels)
4007 {
4008    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
4009    if (vtx_info->chan_byte_size != 4 && channels == 3)
4010       return false;
4011    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
4012           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
4013 }
4014
4015 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
4016                               unsigned offset, unsigned stride, unsigned *channels)
4017 {
4018    if (!vtx_info->chan_byte_size) {
4019       *channels = vtx_info->num_channels;
4020       return vtx_info->chan_format;
4021    }
4022
4023    unsigned num_channels = *channels;
4024    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
4025       unsigned new_channels = num_channels + 1;
4026       /* first, assume more loads is worse and try using a larger data format */
4027       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
4028          new_channels++;
4029          /* don't make the attribute potentially out-of-bounds */
4030          if (offset + new_channels * vtx_info->chan_byte_size > stride)
4031             new_channels = 5;
4032       }
4033
4034       if (new_channels == 5) {
4035          /* then try decreasing load size (at the cost of more loads) */
4036          new_channels = *channels;
4037          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
4038             new_channels--;
4039       }
4040
4041       if (new_channels < *channels)
4042          *channels = new_channels;
4043       num_channels = new_channels;
4044    }
4045
4046    switch (vtx_info->chan_format) {
4047    case V_008F0C_BUF_DATA_FORMAT_8:
4048       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
4049                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
4050    case V_008F0C_BUF_DATA_FORMAT_16:
4051       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
4052                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
4053    case V_008F0C_BUF_DATA_FORMAT_32:
4054       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
4055                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
4056    }
4057    unreachable("shouldn't reach here");
4058    return V_008F0C_BUF_DATA_FORMAT_INVALID;
4059 }
4060
4061 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
4062  * so we may need to fix it up. */
4063 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
4064 {
4065    Builder bld(ctx->program, ctx->block);
4066
4067    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
4068       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
4069
4070    /* For the integer-like cases, do a natural sign extension.
4071     *
4072     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
4073     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
4074     * exponent.
4075     */
4076    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
4077    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
4078
4079    /* Convert back to the right type. */
4080    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
4081       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4082       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
4083       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
4084    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
4085       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4086    }
4087
4088    return alpha;
4089 }
4090
4091 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
4092 {
4093    Builder bld(ctx->program, ctx->block);
4094    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4095    if (ctx->shader->info.stage == MESA_SHADER_VERTEX) {
4096
4097       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
4098       if (off_instr->type != nir_instr_type_load_const) {
4099          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4100          nir_print_instr(off_instr, stderr);
4101          fprintf(stderr, "\n");
4102       }
4103       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
4104
4105       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
4106
4107       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
4108       unsigned component = nir_intrinsic_component(instr);
4109       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
4110       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
4111       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
4112       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
4113
4114       unsigned dfmt = attrib_format & 0xf;
4115       unsigned nfmt = (attrib_format >> 4) & 0x7;
4116       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
4117
4118       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
4119       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
4120       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
4121       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
4122       if (post_shuffle)
4123          num_channels = MAX2(num_channels, 3);
4124
4125       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
4126       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
4127
4128       Temp index;
4129       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
4130          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
4131          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
4132          if (divisor) {
4133             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
4134             if (divisor != 1) {
4135                Temp divided = bld.tmp(v1);
4136                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
4137                index = bld.vadd32(bld.def(v1), start_instance, divided);
4138             } else {
4139                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
4140             }
4141          } else {
4142             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
4143          }
4144       } else {
4145          index = bld.vadd32(bld.def(v1),
4146                             get_arg(ctx, ctx->args->ac.base_vertex),
4147                             get_arg(ctx, ctx->args->ac.vertex_id));
4148       }
4149
4150       Temp channels[num_channels];
4151       unsigned channel_start = 0;
4152       bool direct_fetch = false;
4153
4154       /* skip unused channels at the start */
4155       if (vtx_info->chan_byte_size && !post_shuffle) {
4156          channel_start = ffs(mask) - 1;
4157          for (unsigned i = 0; i < channel_start; i++)
4158             channels[i] = Temp(0, s1);
4159       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
4160          num_channels = 3 - (ffs(mask) - 1);
4161       }
4162
4163       /* load channels */
4164       while (channel_start < num_channels) {
4165          unsigned fetch_size = num_channels - channel_start;
4166          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
4167          bool expanded = false;
4168
4169          /* use MUBUF when possible to avoid possible alignment issues */
4170          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
4171          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
4172                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
4173                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
4174                           vtx_info->chan_byte_size == 4;
4175          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
4176          if (!use_mubuf) {
4177             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
4178          } else {
4179             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
4180                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
4181                fetch_size = 4;
4182                expanded = true;
4183             }
4184          }
4185
4186          Temp fetch_index = index;
4187          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
4188             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
4189             fetch_offset = fetch_offset % attrib_stride;
4190          }
4191
4192          Operand soffset(0u);
4193          if (fetch_offset >= 4096) {
4194             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
4195             fetch_offset %= 4096;
4196          }
4197
4198          aco_opcode opcode;
4199          switch (fetch_size) {
4200          case 1:
4201             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
4202             break;
4203          case 2:
4204             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
4205             break;
4206          case 3:
4207             assert(ctx->options->chip_class >= GFX7 ||
4208                    (!use_mubuf && ctx->options->chip_class == GFX6));
4209             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
4210             break;
4211          case 4:
4212             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
4213             break;
4214          default:
4215             unreachable("Unimplemented load_input vector size");
4216          }
4217
4218          Temp fetch_dst;
4219          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
4220              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
4221                            num_channels <= 3)) {
4222             direct_fetch = true;
4223             fetch_dst = dst;
4224          } else {
4225             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
4226          }
4227
4228          if (use_mubuf) {
4229             Instruction *mubuf = bld.mubuf(opcode,
4230                                            Definition(fetch_dst), list, fetch_index, soffset,
4231                                            fetch_offset, false, true).instr;
4232             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
4233          } else {
4234             Instruction *mtbuf = bld.mtbuf(opcode,
4235                                            Definition(fetch_dst), list, fetch_index, soffset,
4236                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
4237             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
4238          }
4239
4240          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
4241
4242          if (fetch_size == 1) {
4243             channels[channel_start] = fetch_dst;
4244          } else {
4245             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
4246                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
4247          }
4248
4249          channel_start += fetch_size;
4250       }
4251
4252       if (!direct_fetch) {
4253          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
4254                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
4255
4256          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
4257          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
4258          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
4259
4260          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4261          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4262          unsigned num_temp = 0;
4263          for (unsigned i = 0; i < dst.size(); i++) {
4264             unsigned idx = i + component;
4265             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
4266                Temp channel = channels[swizzle[idx]];
4267                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
4268                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
4269                vec->operands[i] = Operand(channel);
4270
4271                num_temp++;
4272                elems[i] = channel;
4273             } else if (is_float && idx == 3) {
4274                vec->operands[i] = Operand(0x3f800000u);
4275             } else if (!is_float && idx == 3) {
4276                vec->operands[i] = Operand(1u);
4277             } else {
4278                vec->operands[i] = Operand(0u);
4279             }
4280          }
4281          vec->definitions[0] = Definition(dst);
4282          ctx->block->instructions.emplace_back(std::move(vec));
4283          emit_split_vector(ctx, dst, dst.size());
4284
4285          if (num_temp == dst.size())
4286             ctx->allocated_vec.emplace(dst.id(), elems);
4287       }
4288    } else if (ctx->shader->info.stage == MESA_SHADER_FRAGMENT) {
4289       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
4290       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
4291       if (off_instr->type != nir_instr_type_load_const ||
4292           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
4293          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4294          nir_print_instr(off_instr, stderr);
4295          fprintf(stderr, "\n");
4296       }
4297
4298       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
4299       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
4300       if (offset) {
4301          assert(offset->u32 == 0);
4302       } else {
4303          /* the lower 15bit of the prim_mask contain the offset into LDS
4304           * while the upper bits contain the number of prims */
4305          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
4306          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
4307          Builder bld(ctx->program, ctx->block);
4308          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
4309          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
4310          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
4311          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
4312          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
4313       }
4314
4315       unsigned idx = nir_intrinsic_base(instr);
4316       unsigned component = nir_intrinsic_component(instr);
4317       unsigned vertex_id = 2; /* P0 */
4318
4319       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
4320          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
4321          switch (src0->u32) {
4322          case 0:
4323             vertex_id = 2; /* P0 */
4324             break;
4325          case 1:
4326             vertex_id = 0; /* P10 */
4327             break;
4328          case 2:
4329             vertex_id = 1; /* P20 */
4330             break;
4331          default:
4332             unreachable("invalid vertex index");
4333          }
4334       }
4335
4336       if (dst.size() == 1) {
4337          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
4338       } else {
4339          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4340          for (unsigned i = 0; i < dst.size(); i++)
4341             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
4342          vec->definitions[0] = Definition(dst);
4343          bld.insert(std::move(vec));
4344       }
4345
4346    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_EVAL) {
4347       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4348       Temp soffset = get_arg(ctx, ctx->args->oc_lds);
4349       std::pair<Temp, unsigned> offs = get_tcs_per_patch_output_vmem_offset(ctx, instr);
4350       unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8u;
4351
4352       load_vmem_mubuf(ctx, dst, ring, offs.first, soffset, offs.second, elem_size_bytes, instr->dest.ssa.num_components);
4353    } else {
4354       unreachable("Shader stage not implemented");
4355    }
4356 }
4357
4358 std::pair<Temp, unsigned> get_gs_per_vertex_input_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride = 1u)
4359 {
4360    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4361
4362    Builder bld(ctx->program, ctx->block);
4363    nir_src *vertex_src = nir_get_io_vertex_index_src(instr);
4364    Temp vertex_offset;
4365
4366    if (!nir_src_is_const(*vertex_src)) {
4367       /* better code could be created, but this case probably doesn't happen
4368        * much in practice */
4369       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, vertex_src->ssa));
4370       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
4371          Temp elem;
4372
4373          if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4374             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
4375             if (i % 2u)
4376                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
4377          } else {
4378             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
4379          }
4380
4381          if (vertex_offset.id()) {
4382             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)),
4383                                  Operand(i), indirect_vertex);
4384             vertex_offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), vertex_offset, elem, cond);
4385          } else {
4386             vertex_offset = elem;
4387          }
4388       }
4389
4390       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4391          vertex_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), vertex_offset);
4392    } else {
4393       unsigned vertex = nir_src_as_uint(*vertex_src);
4394       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4395          vertex_offset = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
4396                                   get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
4397                                   Operand((vertex % 2u) * 16u), Operand(16u));
4398       else
4399          vertex_offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
4400    }
4401
4402    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, base_stride);
4403    offs = offset_add(ctx, offs, std::make_pair(vertex_offset, 0u));
4404    return offset_mul(ctx, offs, 4u);
4405 }
4406
4407 void visit_load_gs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4408 {
4409    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4410
4411    Builder bld(ctx->program, ctx->block);
4412    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4413    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4414
4415    if (ctx->stage == geometry_gs) {
4416       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr, ctx->program->wave_size);
4417       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
4418       load_vmem_mubuf(ctx, dst, ring, offs.first, Temp(), offs.second, elem_size_bytes, instr->dest.ssa.num_components, 4u * ctx->program->wave_size, false, true);
4419    } else if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4420       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr);
4421       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4422       load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4423    } else {
4424       unreachable("Unsupported GS stage.");
4425    }
4426 }
4427
4428 void visit_load_tcs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4429 {
4430    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4431
4432    Builder bld(ctx->program, ctx->block);
4433    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4434
4435    if (load_input_from_temps(ctx, instr, dst))
4436       return;
4437
4438    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_input_lds_offset(ctx, instr);
4439    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4440    unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4441
4442    load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4443 }
4444
4445 void visit_load_tes_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4446 {
4447    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4448
4449    Builder bld(ctx->program, ctx->block);
4450
4451    Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4452    Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
4453    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4454
4455    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4456    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_output_vmem_offset(ctx, instr);
4457
4458    load_vmem_mubuf(ctx, dst, ring, offs.first, oc_lds, offs.second, elem_size_bytes, instr->dest.ssa.num_components, 0u, true, true);
4459 }
4460
4461 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4462 {
4463    switch (ctx->shader->info.stage) {
4464    case MESA_SHADER_GEOMETRY:
4465       visit_load_gs_per_vertex_input(ctx, instr);
4466       break;
4467    case MESA_SHADER_TESS_CTRL:
4468       visit_load_tcs_per_vertex_input(ctx, instr);
4469       break;
4470    case MESA_SHADER_TESS_EVAL:
4471       visit_load_tes_per_vertex_input(ctx, instr);
4472       break;
4473    default:
4474       unreachable("Unimplemented shader stage");
4475    }
4476 }
4477
4478 void visit_load_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4479 {
4480    visit_load_tcs_output(ctx, instr, true);
4481 }
4482
4483 void visit_store_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4484 {
4485    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
4486    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4487
4488    visit_store_tcs_output(ctx, instr, true);
4489 }
4490
4491 void visit_load_tess_coord(isel_context *ctx, nir_intrinsic_instr *instr)
4492 {
4493    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4494
4495    Builder bld(ctx->program, ctx->block);
4496    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4497
4498    Operand tes_u(get_arg(ctx, ctx->args->tes_u));
4499    Operand tes_v(get_arg(ctx, ctx->args->tes_v));
4500    Operand tes_w(0u);
4501
4502    if (ctx->shader->info.tess.primitive_mode == GL_TRIANGLES) {
4503       Temp tmp = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), tes_u, tes_v);
4504       tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0x3f800000u /* 1.0f */), tmp);
4505       tes_w = Operand(tmp);
4506    }
4507
4508    Temp tess_coord = bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tes_u, tes_v, tes_w);
4509    emit_split_vector(ctx, tess_coord, 3);
4510 }
4511
4512 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
4513 {
4514    if (ctx->program->info->need_indirect_descriptor_sets) {
4515       Builder bld(ctx->program, ctx->block);
4516       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
4517       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
4518       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
4519    }
4520
4521    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
4522 }
4523
4524
4525 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
4526 {
4527    Builder bld(ctx->program, ctx->block);
4528    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
4529    if (!ctx->divergent_vals[instr->dest.ssa.index])
4530       index = bld.as_uniform(index);
4531    unsigned desc_set = nir_intrinsic_desc_set(instr);
4532    unsigned binding = nir_intrinsic_binding(instr);
4533
4534    Temp desc_ptr;
4535    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
4536    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
4537    unsigned offset = layout->binding[binding].offset;
4538    unsigned stride;
4539    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
4540        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
4541       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
4542       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
4543       offset = pipeline_layout->push_constant_size + 16 * idx;
4544       stride = 16;
4545    } else {
4546       desc_ptr = load_desc_ptr(ctx, desc_set);
4547       stride = layout->binding[binding].size;
4548    }
4549
4550    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
4551    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
4552    if (stride != 1) {
4553       if (nir_const_index) {
4554          const_index = const_index * stride;
4555       } else if (index.type() == RegType::vgpr) {
4556          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
4557          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
4558       } else {
4559          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
4560       }
4561    }
4562    if (offset) {
4563       if (nir_const_index) {
4564          const_index = const_index + offset;
4565       } else if (index.type() == RegType::vgpr) {
4566          index = bld.vadd32(bld.def(v1), Operand(offset), index);
4567       } else {
4568          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
4569       }
4570    }
4571
4572    if (nir_const_index && const_index == 0) {
4573       index = desc_ptr;
4574    } else if (index.type() == RegType::vgpr) {
4575       index = bld.vadd32(bld.def(v1),
4576                          nir_const_index ? Operand(const_index) : Operand(index),
4577                          Operand(desc_ptr));
4578    } else {
4579       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4580                        nir_const_index ? Operand(const_index) : Operand(index),
4581                        Operand(desc_ptr));
4582    }
4583
4584    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
4585 }
4586
4587 void load_buffer(isel_context *ctx, unsigned num_components, unsigned component_size,
4588                  Temp dst, Temp rsrc, Temp offset, int byte_align,
4589                  bool glc=false, bool readonly=true)
4590 {
4591    Builder bld(ctx->program, ctx->block);
4592    bool dlc = glc && ctx->options->chip_class >= GFX10;
4593    unsigned num_bytes = num_components * component_size;
4594
4595    aco_opcode op;
4596    if (dst.type() == RegType::vgpr || ((ctx->options->chip_class < GFX8 || component_size < 4) && !readonly)) {
4597       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4598       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4599       unsigned const_offset = 0;
4600
4601       /* for small bit sizes add buffer for unaligned loads */
4602       if (byte_align) {
4603          if (num_bytes > 2)
4604             num_bytes += byte_align == -1 ? 4 - component_size : byte_align;
4605          else
4606             byte_align = 0;
4607       }
4608
4609       Temp lower = Temp();
4610       if (num_bytes > 16) {
4611          assert(num_components == 3 || num_components == 4);
4612          op = aco_opcode::buffer_load_dwordx4;
4613          lower = bld.tmp(v4);
4614          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4615          mubuf->definitions[0] = Definition(lower);
4616          mubuf->operands[0] = Operand(rsrc);
4617          mubuf->operands[1] = vaddr;
4618          mubuf->operands[2] = soffset;
4619          mubuf->offen = (offset.type() == RegType::vgpr);
4620          mubuf->glc = glc;
4621          mubuf->dlc = dlc;
4622          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4623          mubuf->can_reorder = readonly;
4624          bld.insert(std::move(mubuf));
4625          emit_split_vector(ctx, lower, 2);
4626          num_bytes -= 16;
4627          const_offset = 16;
4628       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
4629          /* GFX6 doesn't support loading vec3, expand to vec4. */
4630          num_bytes = 16;
4631       }
4632
4633       switch (num_bytes) {
4634          case 1:
4635             op = aco_opcode::buffer_load_ubyte;
4636             break;
4637          case 2:
4638             op = aco_opcode::buffer_load_ushort;
4639             break;
4640          case 3:
4641          case 4:
4642             op = aco_opcode::buffer_load_dword;
4643             break;
4644          case 5:
4645          case 6:
4646          case 7:
4647          case 8:
4648             op = aco_opcode::buffer_load_dwordx2;
4649             break;
4650          case 10:
4651          case 12:
4652             assert(ctx->options->chip_class > GFX6);
4653             op = aco_opcode::buffer_load_dwordx3;
4654             break;
4655          case 16:
4656             op = aco_opcode::buffer_load_dwordx4;
4657             break;
4658          default:
4659             unreachable("Load SSBO not implemented for this size.");
4660       }
4661       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4662       mubuf->operands[0] = Operand(rsrc);
4663       mubuf->operands[1] = vaddr;
4664       mubuf->operands[2] = soffset;
4665       mubuf->offen = (offset.type() == RegType::vgpr);
4666       mubuf->glc = glc;
4667       mubuf->dlc = dlc;
4668       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4669       mubuf->can_reorder = readonly;
4670       mubuf->offset = const_offset;
4671       aco_ptr<Instruction> instr = std::move(mubuf);
4672
4673       if (component_size < 4) {
4674          Temp vec = num_bytes <= 4 ? bld.tmp(v1) : num_bytes <= 8 ? bld.tmp(v2) : bld.tmp(v3);
4675          instr->definitions[0] = Definition(vec);
4676          bld.insert(std::move(instr));
4677
4678          if (byte_align == -1 || (byte_align && dst.type() == RegType::sgpr)) {
4679             Operand align = byte_align == -1 ? Operand(offset) : Operand((uint32_t)byte_align);
4680             Temp tmp[3] = {vec, vec, vec};
4681
4682             if (vec.size() == 3) {
4683                tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1);
4684                bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), vec);
4685             } else if (vec.size() == 2) {
4686                tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = tmp[1];
4687                bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), vec);
4688             }
4689             for (unsigned i = 0; i < dst.size(); i++)
4690                tmp[i] = bld.vop3(aco_opcode::v_alignbyte_b32, bld.def(v1), tmp[i + 1], tmp[i], align);
4691
4692             vec = tmp[0];
4693             if (dst.size() == 2)
4694                vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), tmp[0], tmp[1]);
4695
4696             byte_align = 0;
4697          }
4698
4699          if (dst.type() == RegType::vgpr && num_components == 1) {
4700             bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), vec, Operand(byte_align / component_size));
4701          } else {
4702             trim_subdword_vector(ctx, vec, dst, 4 * vec.size() / component_size, ((1 << num_components) - 1) << byte_align / component_size);
4703          }
4704
4705          return;
4706
4707       } else if (dst.size() > 4) {
4708          assert(lower != Temp());
4709          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
4710          instr->definitions[0] = Definition(upper);
4711          bld.insert(std::move(instr));
4712          if (dst.size() == 8)
4713             emit_split_vector(ctx, upper, 2);
4714          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
4715          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
4716          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
4717          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
4718          if (dst.size() == 8)
4719             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
4720       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
4721          Temp vec = bld.tmp(v4);
4722          instr->definitions[0] = Definition(vec);
4723          bld.insert(std::move(instr));
4724          emit_split_vector(ctx, vec, 4);
4725
4726          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4727          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4728          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4729          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4730       }
4731
4732       if (dst.type() == RegType::sgpr) {
4733          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4734          instr->definitions[0] = Definition(vec);
4735          bld.insert(std::move(instr));
4736          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4737       } else {
4738          instr->definitions[0] = Definition(dst);
4739          bld.insert(std::move(instr));
4740          emit_split_vector(ctx, dst, num_components);
4741       }
4742    } else {
4743       /* for small bit sizes add buffer for unaligned loads */
4744       if (byte_align)
4745          num_bytes += byte_align == -1 ? 4 - component_size : byte_align;
4746
4747       switch (num_bytes) {
4748          case 1:
4749          case 2:
4750          case 3:
4751          case 4:
4752             op = aco_opcode::s_buffer_load_dword;
4753             break;
4754          case 5:
4755          case 6:
4756          case 7:
4757          case 8:
4758             op = aco_opcode::s_buffer_load_dwordx2;
4759             break;
4760          case 10:
4761          case 12:
4762          case 16:
4763             op = aco_opcode::s_buffer_load_dwordx4;
4764             break;
4765          case 24:
4766          case 32:
4767             op = aco_opcode::s_buffer_load_dwordx8;
4768             break;
4769          default:
4770             unreachable("Load SSBO not implemented for this size.");
4771       }
4772       offset = bld.as_uniform(offset);
4773       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4774       load->operands[0] = Operand(rsrc);
4775       load->operands[1] = Operand(offset);
4776       assert(load->operands[1].getTemp().type() == RegType::sgpr);
4777       load->definitions[0] = Definition(dst);
4778       load->glc = glc;
4779       load->dlc = dlc;
4780       load->barrier = readonly ? barrier_none : barrier_buffer;
4781       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
4782       assert(ctx->options->chip_class >= GFX8 || !glc);
4783
4784       /* adjust misaligned small bit size loads */
4785       if (byte_align) {
4786          Temp vec = num_bytes <= 4 ? bld.tmp(s1) : num_bytes <= 8 ? bld.tmp(s2) : bld.tmp(s4);
4787          load->definitions[0] = Definition(vec);
4788          bld.insert(std::move(load));
4789          Operand byte_offset = byte_align > 0 ? Operand(uint32_t(byte_align)) : Operand(offset);
4790          byte_align_scalar(ctx, vec, byte_offset, dst);
4791
4792       /* trim vector */
4793       } else if (dst.size() == 3) {
4794          Temp vec = bld.tmp(s4);
4795          load->definitions[0] = Definition(vec);
4796          bld.insert(std::move(load));
4797          emit_split_vector(ctx, vec, 4);
4798
4799          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4800                     emit_extract_vector(ctx, vec, 0, s1),
4801                     emit_extract_vector(ctx, vec, 1, s1),
4802                     emit_extract_vector(ctx, vec, 2, s1));
4803       } else if (dst.size() == 6) {
4804          Temp vec = bld.tmp(s8);
4805          load->definitions[0] = Definition(vec);
4806          bld.insert(std::move(load));
4807          emit_split_vector(ctx, vec, 4);
4808
4809          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4810                     emit_extract_vector(ctx, vec, 0, s2),
4811                     emit_extract_vector(ctx, vec, 1, s2),
4812                     emit_extract_vector(ctx, vec, 2, s2));
4813       } else {
4814          bld.insert(std::move(load));
4815       }
4816       emit_split_vector(ctx, dst, num_components);
4817    }
4818 }
4819
4820 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
4821 {
4822    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4823    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
4824
4825    Builder bld(ctx->program, ctx->block);
4826
4827    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
4828    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
4829    unsigned binding = nir_intrinsic_binding(idx_instr);
4830    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
4831
4832    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
4833       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4834                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4835                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4836                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4837       if (ctx->options->chip_class >= GFX10) {
4838          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4839                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4840                       S_008F0C_RESOURCE_LEVEL(1);
4841       } else {
4842          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4843                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4844       }
4845       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
4846                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
4847                                      Operand(0xFFFFFFFFu),
4848                                      Operand(desc_type));
4849       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4850                         rsrc, upper_dwords);
4851    } else {
4852       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
4853       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4854    }
4855    unsigned size = instr->dest.ssa.bit_size / 8;
4856    int byte_align = 0;
4857    if (size < 4) {
4858       unsigned align_mul = nir_intrinsic_align_mul(instr);
4859       unsigned align_offset = nir_intrinsic_align_offset(instr);
4860       byte_align = align_mul % 4 == 0 ? align_offset : -1;
4861    }
4862    load_buffer(ctx, instr->num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), byte_align);
4863 }
4864
4865 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4866 {
4867    Builder bld(ctx->program, ctx->block);
4868    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4869    unsigned offset = nir_intrinsic_base(instr);
4870    unsigned count = instr->dest.ssa.num_components;
4871    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
4872
4873    if (index_cv && instr->dest.ssa.bit_size == 32) {
4874       unsigned start = (offset + index_cv->u32) / 4u;
4875       start -= ctx->args->ac.base_inline_push_consts;
4876       if (start + count <= ctx->args->ac.num_inline_push_consts) {
4877          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4878          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4879          for (unsigned i = 0; i < count; ++i) {
4880             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
4881             vec->operands[i] = Operand{elems[i]};
4882          }
4883          vec->definitions[0] = Definition(dst);
4884          ctx->block->instructions.emplace_back(std::move(vec));
4885          ctx->allocated_vec.emplace(dst.id(), elems);
4886          return;
4887       }
4888    }
4889
4890    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
4891    if (offset != 0) // TODO check if index != 0 as well
4892       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
4893    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
4894    Temp vec = dst;
4895    bool trim = false;
4896    bool aligned = true;
4897
4898    if (instr->dest.ssa.bit_size == 8) {
4899       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
4900       bool fits_in_dword = count == 1 || (index_cv && ((offset + index_cv->u32) % 4 + count) <= 4);
4901       if (!aligned)
4902          vec = fits_in_dword ? bld.tmp(s1) : bld.tmp(s2);
4903    } else if (instr->dest.ssa.bit_size == 16) {
4904       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
4905       if (!aligned)
4906          vec = count == 4 ? bld.tmp(s4) : count > 1 ? bld.tmp(s2) : bld.tmp(s1);
4907    }
4908
4909    aco_opcode op;
4910
4911    switch (vec.size()) {
4912    case 1:
4913       op = aco_opcode::s_load_dword;
4914       break;
4915    case 2:
4916       op = aco_opcode::s_load_dwordx2;
4917       break;
4918    case 3:
4919       vec = bld.tmp(s4);
4920       trim = true;
4921    case 4:
4922       op = aco_opcode::s_load_dwordx4;
4923       break;
4924    case 6:
4925       vec = bld.tmp(s8);
4926       trim = true;
4927    case 8:
4928       op = aco_opcode::s_load_dwordx8;
4929       break;
4930    default:
4931       unreachable("unimplemented or forbidden load_push_constant.");
4932    }
4933
4934    bld.smem(op, Definition(vec), ptr, index);
4935
4936    if (!aligned) {
4937       Operand byte_offset = index_cv ? Operand((offset + index_cv->u32) % 4) : Operand(index);
4938       byte_align_scalar(ctx, vec, byte_offset, dst);
4939       return;
4940    }
4941
4942    if (trim) {
4943       emit_split_vector(ctx, vec, 4);
4944       RegClass rc = dst.size() == 3 ? s1 : s2;
4945       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4946                  emit_extract_vector(ctx, vec, 0, rc),
4947                  emit_extract_vector(ctx, vec, 1, rc),
4948                  emit_extract_vector(ctx, vec, 2, rc));
4949
4950    }
4951    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4952 }
4953
4954 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4955 {
4956    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4957
4958    Builder bld(ctx->program, ctx->block);
4959
4960    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4961                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4962                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4963                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4964    if (ctx->options->chip_class >= GFX10) {
4965       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4966                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4967                    S_008F0C_RESOURCE_LEVEL(1);
4968    } else {
4969       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4970                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4971    }
4972
4973    unsigned base = nir_intrinsic_base(instr);
4974    unsigned range = nir_intrinsic_range(instr);
4975
4976    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
4977    if (base && offset.type() == RegType::sgpr)
4978       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
4979    else if (base && offset.type() == RegType::vgpr)
4980       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
4981
4982    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4983                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
4984                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
4985                           Operand(desc_type));
4986    unsigned size = instr->dest.ssa.bit_size / 8;
4987    // TODO: get alignment information for subdword constants
4988    unsigned byte_align = size < 4 ? -1 : 0;
4989    load_buffer(ctx, instr->num_components, size, dst, rsrc, offset, byte_align);
4990 }
4991
4992 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
4993 {
4994    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4995       ctx->cf_info.exec_potentially_empty_discard = true;
4996
4997    ctx->program->needs_exact = true;
4998
4999    // TODO: optimize uniform conditions
5000    Builder bld(ctx->program, ctx->block);
5001    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5002    assert(src.regClass() == bld.lm);
5003    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5004    bld.pseudo(aco_opcode::p_discard_if, src);
5005    ctx->block->kind |= block_kind_uses_discard_if;
5006    return;
5007 }
5008
5009 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
5010 {
5011    Builder bld(ctx->program, ctx->block);
5012
5013    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
5014       ctx->cf_info.exec_potentially_empty_discard = true;
5015
5016    bool divergent = ctx->cf_info.parent_if.is_divergent ||
5017                     ctx->cf_info.parent_loop.has_divergent_continue;
5018
5019    if (ctx->block->loop_nest_depth &&
5020        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
5021       /* we handle discards the same way as jump instructions */
5022       append_logical_end(ctx->block);
5023
5024       /* in loops, discard behaves like break */
5025       Block *linear_target = ctx->cf_info.parent_loop.exit;
5026       ctx->block->kind |= block_kind_discard;
5027
5028       if (!divergent) {
5029          /* uniform discard - loop ends here */
5030          assert(nir_instr_is_last(&instr->instr));
5031          ctx->block->kind |= block_kind_uniform;
5032          ctx->cf_info.has_branch = true;
5033          bld.branch(aco_opcode::p_branch);
5034          add_linear_edge(ctx->block->index, linear_target);
5035          return;
5036       }
5037
5038       /* we add a break right behind the discard() instructions */
5039       ctx->block->kind |= block_kind_break;
5040       unsigned idx = ctx->block->index;
5041
5042       ctx->cf_info.parent_loop.has_divergent_branch = true;
5043       ctx->cf_info.nir_to_aco[instr->instr.block->index] = idx;
5044
5045       /* remove critical edges from linear CFG */
5046       bld.branch(aco_opcode::p_branch);
5047       Block* break_block = ctx->program->create_and_insert_block();
5048       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5049       break_block->kind |= block_kind_uniform;
5050       add_linear_edge(idx, break_block);
5051       add_linear_edge(break_block->index, linear_target);
5052       bld.reset(break_block);
5053       bld.branch(aco_opcode::p_branch);
5054
5055       Block* continue_block = ctx->program->create_and_insert_block();
5056       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5057       add_linear_edge(idx, continue_block);
5058       append_logical_start(continue_block);
5059       ctx->block = continue_block;
5060
5061       return;
5062    }
5063
5064    /* it can currently happen that NIR doesn't remove the unreachable code */
5065    if (!nir_instr_is_last(&instr->instr)) {
5066       ctx->program->needs_exact = true;
5067       /* save exec somewhere temporarily so that it doesn't get
5068        * overwritten before the discard from outer exec masks */
5069       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
5070       bld.pseudo(aco_opcode::p_discard_if, cond);
5071       ctx->block->kind |= block_kind_uses_discard_if;
5072       return;
5073    }
5074
5075    /* This condition is incorrect for uniformly branched discards in a loop
5076     * predicated by a divergent condition, but the above code catches that case
5077     * and the discard would end up turning into a discard_if.
5078     * For example:
5079     * if (divergent) {
5080     *    while (...) {
5081     *       if (uniform) {
5082     *          discard;
5083     *       }
5084     *    }
5085     * }
5086     */
5087    if (!ctx->cf_info.parent_if.is_divergent) {
5088       /* program just ends here */
5089       ctx->block->kind |= block_kind_uniform;
5090       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
5091               0 /* enabled mask */, 9 /* dest */,
5092               false /* compressed */, true/* done */, true /* valid mask */);
5093       bld.sopp(aco_opcode::s_endpgm);
5094       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
5095    } else {
5096       ctx->block->kind |= block_kind_discard;
5097       /* branch and linear edge is added by visit_if() */
5098    }
5099 }
5100
5101 enum aco_descriptor_type {
5102    ACO_DESC_IMAGE,
5103    ACO_DESC_FMASK,
5104    ACO_DESC_SAMPLER,
5105    ACO_DESC_BUFFER,
5106    ACO_DESC_PLANE_0,
5107    ACO_DESC_PLANE_1,
5108    ACO_DESC_PLANE_2,
5109 };
5110
5111 static bool
5112 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
5113    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
5114       return false;
5115    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
5116    return dim == ac_image_cube ||
5117           dim == ac_image_1darray ||
5118           dim == ac_image_2darray ||
5119           dim == ac_image_2darraymsaa;
5120 }
5121
5122 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
5123                       enum aco_descriptor_type desc_type,
5124                       const nir_tex_instr *tex_instr, bool image, bool write)
5125 {
5126 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
5127    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
5128    if (it != ctx->tex_desc.end())
5129       return it->second;
5130 */
5131    Temp index = Temp();
5132    bool index_set = false;
5133    unsigned constant_index = 0;
5134    unsigned descriptor_set;
5135    unsigned base_index;
5136    Builder bld(ctx->program, ctx->block);
5137
5138    if (!deref_instr) {
5139       assert(tex_instr && !image);
5140       descriptor_set = 0;
5141       base_index = tex_instr->sampler_index;
5142    } else {
5143       while(deref_instr->deref_type != nir_deref_type_var) {
5144          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
5145          if (!array_size)
5146             array_size = 1;
5147
5148          assert(deref_instr->deref_type == nir_deref_type_array);
5149          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
5150          if (const_value) {
5151             constant_index += array_size * const_value->u32;
5152          } else {
5153             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
5154             if (indirect.type() == RegType::vgpr)
5155                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
5156
5157             if (array_size != 1)
5158                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
5159
5160             if (!index_set) {
5161                index = indirect;
5162                index_set = true;
5163             } else {
5164                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
5165             }
5166          }
5167
5168          deref_instr = nir_src_as_deref(deref_instr->parent);
5169       }
5170       descriptor_set = deref_instr->var->data.descriptor_set;
5171       base_index = deref_instr->var->data.binding;
5172    }
5173
5174    Temp list = load_desc_ptr(ctx, descriptor_set);
5175    list = convert_pointer_to_64_bit(ctx, list);
5176
5177    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
5178    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
5179    unsigned offset = binding->offset;
5180    unsigned stride = binding->size;
5181    aco_opcode opcode;
5182    RegClass type;
5183
5184    assert(base_index < layout->binding_count);
5185
5186    switch (desc_type) {
5187    case ACO_DESC_IMAGE:
5188       type = s8;
5189       opcode = aco_opcode::s_load_dwordx8;
5190       break;
5191    case ACO_DESC_FMASK:
5192       type = s8;
5193       opcode = aco_opcode::s_load_dwordx8;
5194       offset += 32;
5195       break;
5196    case ACO_DESC_SAMPLER:
5197       type = s4;
5198       opcode = aco_opcode::s_load_dwordx4;
5199       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
5200          offset += radv_combined_image_descriptor_sampler_offset(binding);
5201       break;
5202    case ACO_DESC_BUFFER:
5203       type = s4;
5204       opcode = aco_opcode::s_load_dwordx4;
5205       break;
5206    case ACO_DESC_PLANE_0:
5207    case ACO_DESC_PLANE_1:
5208       type = s8;
5209       opcode = aco_opcode::s_load_dwordx8;
5210       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
5211       break;
5212    case ACO_DESC_PLANE_2:
5213       type = s4;
5214       opcode = aco_opcode::s_load_dwordx4;
5215       offset += 64;
5216       break;
5217    default:
5218       unreachable("invalid desc_type\n");
5219    }
5220
5221    offset += constant_index * stride;
5222
5223    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
5224       (!index_set || binding->immutable_samplers_equal)) {
5225       if (binding->immutable_samplers_equal)
5226          constant_index = 0;
5227
5228       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
5229       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5230                         Operand(samplers[constant_index * 4 + 0]),
5231                         Operand(samplers[constant_index * 4 + 1]),
5232                         Operand(samplers[constant_index * 4 + 2]),
5233                         Operand(samplers[constant_index * 4 + 3]));
5234    }
5235
5236    Operand off;
5237    if (!index_set) {
5238       off = bld.copy(bld.def(s1), Operand(offset));
5239    } else {
5240       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
5241                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
5242    }
5243
5244    Temp res = bld.smem(opcode, bld.def(type), list, off);
5245
5246    if (desc_type == ACO_DESC_PLANE_2) {
5247       Temp components[8];
5248       for (unsigned i = 0; i < 8; i++)
5249          components[i] = bld.tmp(s1);
5250       bld.pseudo(aco_opcode::p_split_vector,
5251                  Definition(components[0]),
5252                  Definition(components[1]),
5253                  Definition(components[2]),
5254                  Definition(components[3]),
5255                  res);
5256
5257       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
5258       bld.pseudo(aco_opcode::p_split_vector,
5259                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
5260                  Definition(components[4]),
5261                  Definition(components[5]),
5262                  Definition(components[6]),
5263                  Definition(components[7]),
5264                  desc2);
5265
5266       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
5267                        components[0], components[1], components[2], components[3],
5268                        components[4], components[5], components[6], components[7]);
5269    }
5270
5271    return res;
5272 }
5273
5274 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
5275 {
5276    switch (dim) {
5277    case GLSL_SAMPLER_DIM_BUF:
5278       return 1;
5279    case GLSL_SAMPLER_DIM_1D:
5280       return array ? 2 : 1;
5281    case GLSL_SAMPLER_DIM_2D:
5282       return array ? 3 : 2;
5283    case GLSL_SAMPLER_DIM_MS:
5284       return array ? 4 : 3;
5285    case GLSL_SAMPLER_DIM_3D:
5286    case GLSL_SAMPLER_DIM_CUBE:
5287       return 3;
5288    case GLSL_SAMPLER_DIM_RECT:
5289    case GLSL_SAMPLER_DIM_SUBPASS:
5290       return 2;
5291    case GLSL_SAMPLER_DIM_SUBPASS_MS:
5292       return 3;
5293    default:
5294       break;
5295    }
5296    return 0;
5297 }
5298
5299
5300 /* Adjust the sample index according to FMASK.
5301  *
5302  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
5303  * which is the identity mapping. Each nibble says which physical sample
5304  * should be fetched to get that sample.
5305  *
5306  * For example, 0x11111100 means there are only 2 samples stored and
5307  * the second sample covers 3/4 of the pixel. When reading samples 0
5308  * and 1, return physical sample 0 (determined by the first two 0s
5309  * in FMASK), otherwise return physical sample 1.
5310  *
5311  * The sample index should be adjusted as follows:
5312  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
5313  */
5314 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
5315 {
5316    Builder bld(ctx->program, ctx->block);
5317    Temp fmask = bld.tmp(v1);
5318    unsigned dim = ctx->options->chip_class >= GFX10
5319                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
5320                   : 0;
5321
5322    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
5323                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
5324    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
5325    load->operands[0] = Operand(fmask_desc_ptr);
5326    load->operands[1] = Operand(s4); /* no sampler */
5327    load->operands[2] = Operand(coord);
5328    load->definitions[0] = Definition(fmask);
5329    load->glc = false;
5330    load->dlc = false;
5331    load->dmask = 0x1;
5332    load->unrm = true;
5333    load->da = da;
5334    load->dim = dim;
5335    load->can_reorder = true; /* fmask images shouldn't be modified */
5336    ctx->block->instructions.emplace_back(std::move(load));
5337
5338    Operand sample_index4;
5339    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
5340       sample_index4 = Operand(sample_index.constantValue() << 2);
5341    } else if (sample_index.regClass() == s1) {
5342       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
5343    } else {
5344       assert(sample_index.regClass() == v1);
5345       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
5346    }
5347
5348    Temp final_sample;
5349    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
5350       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
5351    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
5352       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
5353    else
5354       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
5355
5356    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
5357     * resource descriptor is 0 (invalid),
5358     */
5359    Temp compare = bld.tmp(bld.lm);
5360    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
5361                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
5362
5363    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
5364
5365    /* Replace the MSAA sample index. */
5366    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
5367 }
5368
5369 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
5370 {
5371
5372    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
5373    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5374    bool is_array = glsl_sampler_type_is_array(type);
5375    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5376    assert(!add_frag_pos && "Input attachments should be lowered.");
5377    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5378    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
5379    int count = image_type_to_components_count(dim, is_array);
5380    std::vector<Temp> coords(count);
5381    Builder bld(ctx->program, ctx->block);
5382
5383    if (is_ms) {
5384       count--;
5385       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
5386       /* get sample index */
5387       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
5388          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
5389          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
5390          std::vector<Temp> fmask_load_address;
5391          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
5392             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
5393
5394          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
5395          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
5396       } else {
5397          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
5398       }
5399    }
5400
5401    if (gfx9_1d) {
5402       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
5403       coords.resize(coords.size() + 1);
5404       coords[1] = bld.copy(bld.def(v1), Operand(0u));
5405       if (is_array)
5406          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
5407    } else {
5408       for (int i = 0; i < count; i++)
5409          coords[i] = emit_extract_vector(ctx, src0, i, v1);
5410    }
5411
5412    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
5413        instr->intrinsic == nir_intrinsic_image_deref_store) {
5414       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
5415       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
5416
5417       if (!level_zero)
5418          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
5419    }
5420
5421    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
5422    for (unsigned i = 0; i < coords.size(); i++)
5423       vec->operands[i] = Operand(coords[i]);
5424    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
5425    vec->definitions[0] = Definition(res);
5426    ctx->block->instructions.emplace_back(std::move(vec));
5427    return res;
5428 }
5429
5430
5431 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
5432 {
5433    Builder bld(ctx->program, ctx->block);
5434    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5435    const struct glsl_type *type = glsl_without_array(var->type);
5436    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5437    bool is_array = glsl_sampler_type_is_array(type);
5438    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5439
5440    if (dim == GLSL_SAMPLER_DIM_BUF) {
5441       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
5442       unsigned num_channels = util_last_bit(mask);
5443       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5444       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5445
5446       aco_opcode opcode;
5447       switch (num_channels) {
5448       case 1:
5449          opcode = aco_opcode::buffer_load_format_x;
5450          break;
5451       case 2:
5452          opcode = aco_opcode::buffer_load_format_xy;
5453          break;
5454       case 3:
5455          opcode = aco_opcode::buffer_load_format_xyz;
5456          break;
5457       case 4:
5458          opcode = aco_opcode::buffer_load_format_xyzw;
5459          break;
5460       default:
5461          unreachable(">4 channel buffer image load");
5462       }
5463       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
5464       load->operands[0] = Operand(rsrc);
5465       load->operands[1] = Operand(vindex);
5466       load->operands[2] = Operand((uint32_t) 0);
5467       Temp tmp;
5468       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5469          tmp = dst;
5470       else
5471          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
5472       load->definitions[0] = Definition(tmp);
5473       load->idxen = true;
5474       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
5475       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5476       load->barrier = barrier_image;
5477       ctx->block->instructions.emplace_back(std::move(load));
5478
5479       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
5480       return;
5481    }
5482
5483    Temp coords = get_image_coords(ctx, instr, type);
5484    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5485
5486    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
5487    unsigned num_components = util_bitcount(dmask);
5488    Temp tmp;
5489    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5490       tmp = dst;
5491    else
5492       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
5493
5494    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
5495    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
5496
5497    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
5498    load->operands[0] = Operand(resource);
5499    load->operands[1] = Operand(s4); /* no sampler */
5500    load->operands[2] = Operand(coords);
5501    load->definitions[0] = Definition(tmp);
5502    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
5503    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5504    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5505    load->dmask = dmask;
5506    load->unrm = true;
5507    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5508    load->barrier = barrier_image;
5509    ctx->block->instructions.emplace_back(std::move(load));
5510
5511    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
5512    return;
5513 }
5514
5515 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
5516 {
5517    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5518    const struct glsl_type *type = glsl_without_array(var->type);
5519    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5520    bool is_array = glsl_sampler_type_is_array(type);
5521    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5522
5523    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
5524
5525    if (dim == GLSL_SAMPLER_DIM_BUF) {
5526       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5527       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5528       aco_opcode opcode;
5529       switch (data.size()) {
5530       case 1:
5531          opcode = aco_opcode::buffer_store_format_x;
5532          break;
5533       case 2:
5534          opcode = aco_opcode::buffer_store_format_xy;
5535          break;
5536       case 3:
5537          opcode = aco_opcode::buffer_store_format_xyz;
5538          break;
5539       case 4:
5540          opcode = aco_opcode::buffer_store_format_xyzw;
5541          break;
5542       default:
5543          unreachable(">4 channel buffer image store");
5544       }
5545       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
5546       store->operands[0] = Operand(rsrc);
5547       store->operands[1] = Operand(vindex);
5548       store->operands[2] = Operand((uint32_t) 0);
5549       store->operands[3] = Operand(data);
5550       store->idxen = true;
5551       store->glc = glc;
5552       store->dlc = false;
5553       store->disable_wqm = true;
5554       store->barrier = barrier_image;
5555       ctx->program->needs_exact = true;
5556       ctx->block->instructions.emplace_back(std::move(store));
5557       return;
5558    }
5559
5560    assert(data.type() == RegType::vgpr);
5561    Temp coords = get_image_coords(ctx, instr, type);
5562    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5563
5564    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
5565    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
5566
5567    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
5568    store->operands[0] = Operand(resource);
5569    store->operands[1] = Operand(data);
5570    store->operands[2] = Operand(coords);
5571    store->glc = glc;
5572    store->dlc = false;
5573    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5574    store->dmask = (1 << data.size()) - 1;
5575    store->unrm = true;
5576    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5577    store->disable_wqm = true;
5578    store->barrier = barrier_image;
5579    ctx->program->needs_exact = true;
5580    ctx->block->instructions.emplace_back(std::move(store));
5581    return;
5582 }
5583
5584 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5585 {
5586    /* return the previous value if dest is ever used */
5587    bool return_previous = false;
5588    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5589       return_previous = true;
5590       break;
5591    }
5592    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5593       return_previous = true;
5594       break;
5595    }
5596
5597    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5598    const struct glsl_type *type = glsl_without_array(var->type);
5599    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5600    bool is_array = glsl_sampler_type_is_array(type);
5601    Builder bld(ctx->program, ctx->block);
5602
5603    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5604    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
5605
5606    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
5607       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
5608
5609    aco_opcode buf_op, image_op;
5610    switch (instr->intrinsic) {
5611       case nir_intrinsic_image_deref_atomic_add:
5612          buf_op = aco_opcode::buffer_atomic_add;
5613          image_op = aco_opcode::image_atomic_add;
5614          break;
5615       case nir_intrinsic_image_deref_atomic_umin:
5616          buf_op = aco_opcode::buffer_atomic_umin;
5617          image_op = aco_opcode::image_atomic_umin;
5618          break;
5619       case nir_intrinsic_image_deref_atomic_imin:
5620          buf_op = aco_opcode::buffer_atomic_smin;
5621          image_op = aco_opcode::image_atomic_smin;
5622          break;
5623       case nir_intrinsic_image_deref_atomic_umax:
5624          buf_op = aco_opcode::buffer_atomic_umax;
5625          image_op = aco_opcode::image_atomic_umax;
5626          break;
5627       case nir_intrinsic_image_deref_atomic_imax:
5628          buf_op = aco_opcode::buffer_atomic_smax;
5629          image_op = aco_opcode::image_atomic_smax;
5630          break;
5631       case nir_intrinsic_image_deref_atomic_and:
5632          buf_op = aco_opcode::buffer_atomic_and;
5633          image_op = aco_opcode::image_atomic_and;
5634          break;
5635       case nir_intrinsic_image_deref_atomic_or:
5636          buf_op = aco_opcode::buffer_atomic_or;
5637          image_op = aco_opcode::image_atomic_or;
5638          break;
5639       case nir_intrinsic_image_deref_atomic_xor:
5640          buf_op = aco_opcode::buffer_atomic_xor;
5641          image_op = aco_opcode::image_atomic_xor;
5642          break;
5643       case nir_intrinsic_image_deref_atomic_exchange:
5644          buf_op = aco_opcode::buffer_atomic_swap;
5645          image_op = aco_opcode::image_atomic_swap;
5646          break;
5647       case nir_intrinsic_image_deref_atomic_comp_swap:
5648          buf_op = aco_opcode::buffer_atomic_cmpswap;
5649          image_op = aco_opcode::image_atomic_cmpswap;
5650          break;
5651       default:
5652          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
5653    }
5654
5655    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5656
5657    if (dim == GLSL_SAMPLER_DIM_BUF) {
5658       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5659       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5660       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
5661       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5662       mubuf->operands[0] = Operand(resource);
5663       mubuf->operands[1] = Operand(vindex);
5664       mubuf->operands[2] = Operand((uint32_t)0);
5665       mubuf->operands[3] = Operand(data);
5666       if (return_previous)
5667          mubuf->definitions[0] = Definition(dst);
5668       mubuf->offset = 0;
5669       mubuf->idxen = true;
5670       mubuf->glc = return_previous;
5671       mubuf->dlc = false; /* Not needed for atomics */
5672       mubuf->disable_wqm = true;
5673       mubuf->barrier = barrier_image;
5674       ctx->program->needs_exact = true;
5675       ctx->block->instructions.emplace_back(std::move(mubuf));
5676       return;
5677    }
5678
5679    Temp coords = get_image_coords(ctx, instr, type);
5680    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5681    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
5682    mimg->operands[0] = Operand(resource);
5683    mimg->operands[1] = Operand(data);
5684    mimg->operands[2] = Operand(coords);
5685    if (return_previous)
5686       mimg->definitions[0] = Definition(dst);
5687    mimg->glc = return_previous;
5688    mimg->dlc = false; /* Not needed for atomics */
5689    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5690    mimg->dmask = (1 << data.size()) - 1;
5691    mimg->unrm = true;
5692    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5693    mimg->disable_wqm = true;
5694    mimg->barrier = barrier_image;
5695    ctx->program->needs_exact = true;
5696    ctx->block->instructions.emplace_back(std::move(mimg));
5697    return;
5698 }
5699
5700 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
5701 {
5702    if (in_elements && ctx->options->chip_class == GFX8) {
5703       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
5704       Builder bld(ctx->program, ctx->block);
5705
5706       Temp size = emit_extract_vector(ctx, desc, 2, s1);
5707
5708       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
5709       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
5710
5711       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
5712       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
5713
5714       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
5715       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
5716
5717       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
5718       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
5719                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
5720       if (dst.type() == RegType::vgpr)
5721          bld.copy(Definition(dst), shr_dst);
5722
5723       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
5724    } else {
5725       emit_extract_vector(ctx, desc, 2, dst);
5726    }
5727 }
5728
5729 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
5730 {
5731    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5732    const struct glsl_type *type = glsl_without_array(var->type);
5733    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5734    bool is_array = glsl_sampler_type_is_array(type);
5735    Builder bld(ctx->program, ctx->block);
5736
5737    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
5738       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
5739       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
5740    }
5741
5742    /* LOD */
5743    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
5744
5745    /* Resource */
5746    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
5747
5748    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5749
5750    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
5751    mimg->operands[0] = Operand(resource);
5752    mimg->operands[1] = Operand(s4); /* no sampler */
5753    mimg->operands[2] = Operand(lod);
5754    uint8_t& dmask = mimg->dmask;
5755    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5756    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
5757    mimg->da = glsl_sampler_type_is_array(type);
5758    mimg->can_reorder = true;
5759    Definition& def = mimg->definitions[0];
5760    ctx->block->instructions.emplace_back(std::move(mimg));
5761
5762    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
5763        glsl_sampler_type_is_array(type)) {
5764
5765       assert(instr->dest.ssa.num_components == 3);
5766       Temp tmp = {ctx->program->allocateId(), v3};
5767       def = Definition(tmp);
5768       emit_split_vector(ctx, tmp, 3);
5769
5770       /* divide 3rd value by 6 by multiplying with magic number */
5771       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
5772       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
5773
5774       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5775                  emit_extract_vector(ctx, tmp, 0, v1),
5776                  emit_extract_vector(ctx, tmp, 1, v1),
5777                  by_6);
5778
5779    } else if (ctx->options->chip_class == GFX9 &&
5780               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
5781               glsl_sampler_type_is_array(type)) {
5782       assert(instr->dest.ssa.num_components == 2);
5783       def = Definition(dst);
5784       dmask = 0x5;
5785    } else {
5786       def = Definition(dst);
5787    }
5788
5789    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
5790 }
5791
5792 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5793 {
5794    Builder bld(ctx->program, ctx->block);
5795    unsigned num_components = instr->num_components;
5796
5797    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5798    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5799    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5800
5801    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5802    unsigned size = instr->dest.ssa.bit_size / 8;
5803    int byte_align = 0;
5804    if (size < 4) {
5805       unsigned align_mul = nir_intrinsic_align_mul(instr);
5806       unsigned align_offset = nir_intrinsic_align_offset(instr);
5807       byte_align = align_mul % 4 == 0 ? align_offset : -1;
5808    }
5809    load_buffer(ctx, num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), byte_align, glc, false);
5810 }
5811
5812 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5813 {
5814    Builder bld(ctx->program, ctx->block);
5815    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5816    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5817    unsigned writemask = nir_intrinsic_write_mask(instr);
5818    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
5819
5820    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5821    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5822
5823    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
5824                ctx->options->chip_class >= GFX8 &&
5825                elem_size_bytes >= 4;
5826    if (smem)
5827       offset = bld.as_uniform(offset);
5828    bool smem_nonfs = smem && ctx->stage != fragment_fs;
5829
5830    while (writemask) {
5831       int start, count;
5832       u_bit_scan_consecutive_range(&writemask, &start, &count);
5833       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
5834          /* GFX6 doesn't support storing vec3, split it. */
5835          writemask |= 1u << (start + 2);
5836          count = 2;
5837       }
5838       int num_bytes = count * elem_size_bytes;
5839
5840       /* dword or larger stores have to be dword-aligned */
5841       if (elem_size_bytes < 4 && num_bytes > 2) {
5842          // TODO: improve alignment check of sub-dword stores
5843          unsigned count_new = 2 / elem_size_bytes;
5844          writemask |= ((1 << (count - count_new)) - 1) << (start + count_new);
5845          count = count_new;
5846          num_bytes = 2;
5847       }
5848
5849       if (num_bytes > 16) {
5850          assert(elem_size_bytes == 8);
5851          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5852          count = 2;
5853          num_bytes = 16;
5854       }
5855
5856       Temp write_data;
5857       if (elem_size_bytes < 4) {
5858          if (data.type() == RegType::sgpr) {
5859             data = as_vgpr(ctx, data);
5860             emit_split_vector(ctx, data, 4 * data.size() / elem_size_bytes);
5861          }
5862          RegClass rc = RegClass(RegType::vgpr, elem_size_bytes).as_subdword();
5863          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5864          for (int i = 0; i < count; i++)
5865             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, rc));
5866          write_data = bld.tmp(RegClass(RegType::vgpr, num_bytes).as_subdword());
5867          vec->definitions[0] = Definition(write_data);
5868          bld.insert(std::move(vec));
5869       } else if (count != instr->num_components) {
5870          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5871          for (int i = 0; i < count; i++) {
5872             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
5873             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
5874          }
5875          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
5876          vec->definitions[0] = Definition(write_data);
5877          ctx->block->instructions.emplace_back(std::move(vec));
5878       } else if (!smem && data.type() != RegType::vgpr) {
5879          assert(num_bytes % 4 == 0);
5880          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
5881       } else if (smem_nonfs && data.type() == RegType::vgpr) {
5882          assert(num_bytes % 4 == 0);
5883          write_data = bld.as_uniform(data);
5884       } else {
5885          write_data = data;
5886       }
5887
5888       aco_opcode vmem_op, smem_op = aco_opcode::last_opcode;
5889       switch (num_bytes) {
5890          case 1:
5891             vmem_op = aco_opcode::buffer_store_byte;
5892             break;
5893          case 2:
5894             vmem_op = aco_opcode::buffer_store_short;
5895             break;
5896          case 4:
5897             vmem_op = aco_opcode::buffer_store_dword;
5898             smem_op = aco_opcode::s_buffer_store_dword;
5899             break;
5900          case 8:
5901             vmem_op = aco_opcode::buffer_store_dwordx2;
5902             smem_op = aco_opcode::s_buffer_store_dwordx2;
5903             break;
5904          case 12:
5905             vmem_op = aco_opcode::buffer_store_dwordx3;
5906             assert(!smem && ctx->options->chip_class > GFX6);
5907             break;
5908          case 16:
5909             vmem_op = aco_opcode::buffer_store_dwordx4;
5910             smem_op = aco_opcode::s_buffer_store_dwordx4;
5911             break;
5912          default:
5913             unreachable("Store SSBO not implemented for this size.");
5914       }
5915       if (ctx->stage == fragment_fs)
5916          smem_op = aco_opcode::p_fs_buffer_store_smem;
5917
5918       if (smem) {
5919          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
5920          store->operands[0] = Operand(rsrc);
5921          if (start) {
5922             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
5923                                 offset, Operand(start * elem_size_bytes));
5924             store->operands[1] = Operand(off);
5925          } else {
5926             store->operands[1] = Operand(offset);
5927          }
5928          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
5929             store->operands[1].setFixed(m0);
5930          store->operands[2] = Operand(write_data);
5931          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5932          store->dlc = false;
5933          store->disable_wqm = true;
5934          store->barrier = barrier_buffer;
5935          ctx->block->instructions.emplace_back(std::move(store));
5936          ctx->program->wb_smem_l1_on_end = true;
5937          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
5938             ctx->block->kind |= block_kind_needs_lowering;
5939             ctx->program->needs_exact = true;
5940          }
5941       } else {
5942          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
5943          store->operands[0] = Operand(rsrc);
5944          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5945          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5946          store->operands[3] = Operand(write_data);
5947          store->offset = start * elem_size_bytes;
5948          store->offen = (offset.type() == RegType::vgpr);
5949          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5950          store->dlc = false;
5951          store->disable_wqm = true;
5952          store->barrier = barrier_buffer;
5953          ctx->program->needs_exact = true;
5954          ctx->block->instructions.emplace_back(std::move(store));
5955       }
5956    }
5957 }
5958
5959 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5960 {
5961    /* return the previous value if dest is ever used */
5962    bool return_previous = false;
5963    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5964       return_previous = true;
5965       break;
5966    }
5967    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5968       return_previous = true;
5969       break;
5970    }
5971
5972    Builder bld(ctx->program, ctx->block);
5973    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
5974
5975    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
5976       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5977                         get_ssa_temp(ctx, instr->src[3].ssa), data);
5978
5979    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
5980    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5981    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5982
5983    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5984
5985    aco_opcode op32, op64;
5986    switch (instr->intrinsic) {
5987       case nir_intrinsic_ssbo_atomic_add:
5988          op32 = aco_opcode::buffer_atomic_add;
5989          op64 = aco_opcode::buffer_atomic_add_x2;
5990          break;
5991       case nir_intrinsic_ssbo_atomic_imin:
5992          op32 = aco_opcode::buffer_atomic_smin;
5993          op64 = aco_opcode::buffer_atomic_smin_x2;
5994          break;
5995       case nir_intrinsic_ssbo_atomic_umin:
5996          op32 = aco_opcode::buffer_atomic_umin;
5997          op64 = aco_opcode::buffer_atomic_umin_x2;
5998          break;
5999       case nir_intrinsic_ssbo_atomic_imax:
6000          op32 = aco_opcode::buffer_atomic_smax;
6001          op64 = aco_opcode::buffer_atomic_smax_x2;
6002          break;
6003       case nir_intrinsic_ssbo_atomic_umax:
6004          op32 = aco_opcode::buffer_atomic_umax;
6005          op64 = aco_opcode::buffer_atomic_umax_x2;
6006          break;
6007       case nir_intrinsic_ssbo_atomic_and:
6008          op32 = aco_opcode::buffer_atomic_and;
6009          op64 = aco_opcode::buffer_atomic_and_x2;
6010          break;
6011       case nir_intrinsic_ssbo_atomic_or:
6012          op32 = aco_opcode::buffer_atomic_or;
6013          op64 = aco_opcode::buffer_atomic_or_x2;
6014          break;
6015       case nir_intrinsic_ssbo_atomic_xor:
6016          op32 = aco_opcode::buffer_atomic_xor;
6017          op64 = aco_opcode::buffer_atomic_xor_x2;
6018          break;
6019       case nir_intrinsic_ssbo_atomic_exchange:
6020          op32 = aco_opcode::buffer_atomic_swap;
6021          op64 = aco_opcode::buffer_atomic_swap_x2;
6022          break;
6023       case nir_intrinsic_ssbo_atomic_comp_swap:
6024          op32 = aco_opcode::buffer_atomic_cmpswap;
6025          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
6026          break;
6027       default:
6028          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
6029    }
6030    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6031    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
6032    mubuf->operands[0] = Operand(rsrc);
6033    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
6034    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
6035    mubuf->operands[3] = Operand(data);
6036    if (return_previous)
6037       mubuf->definitions[0] = Definition(dst);
6038    mubuf->offset = 0;
6039    mubuf->offen = (offset.type() == RegType::vgpr);
6040    mubuf->glc = return_previous;
6041    mubuf->dlc = false; /* Not needed for atomics */
6042    mubuf->disable_wqm = true;
6043    mubuf->barrier = barrier_buffer;
6044    ctx->program->needs_exact = true;
6045    ctx->block->instructions.emplace_back(std::move(mubuf));
6046 }
6047
6048 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
6049
6050    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6051    Builder bld(ctx->program, ctx->block);
6052    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
6053    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
6054 }
6055
6056 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
6057 {
6058    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6059                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6060
6061    if (addr.type() == RegType::vgpr)
6062       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
6063    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
6064 }
6065
6066 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
6067 {
6068    Builder bld(ctx->program, ctx->block);
6069    unsigned num_components = instr->num_components;
6070    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
6071
6072    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6073    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6074
6075    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
6076    bool dlc = glc && ctx->options->chip_class >= GFX10;
6077    aco_opcode op;
6078    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
6079       bool global = ctx->options->chip_class >= GFX9;
6080
6081       if (ctx->options->chip_class >= GFX7) {
6082          aco_opcode op;
6083          switch (num_bytes) {
6084          case 4:
6085             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
6086             break;
6087          case 8:
6088             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
6089             break;
6090          case 12:
6091             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
6092             break;
6093          case 16:
6094             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
6095             break;
6096          default:
6097             unreachable("load_global not implemented for this size.");
6098          }
6099
6100          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
6101          flat->operands[0] = Operand(addr);
6102          flat->operands[1] = Operand(s1);
6103          flat->glc = glc;
6104          flat->dlc = dlc;
6105          flat->barrier = barrier_buffer;
6106
6107          if (dst.type() == RegType::sgpr) {
6108             Temp vec = bld.tmp(RegType::vgpr, dst.size());
6109             flat->definitions[0] = Definition(vec);
6110             ctx->block->instructions.emplace_back(std::move(flat));
6111             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
6112          } else {
6113             flat->definitions[0] = Definition(dst);
6114             ctx->block->instructions.emplace_back(std::move(flat));
6115          }
6116          emit_split_vector(ctx, dst, num_components);
6117       } else {
6118          assert(ctx->options->chip_class == GFX6);
6119
6120          /* GFX6 doesn't support loading vec3, expand to vec4. */
6121          num_bytes = num_bytes == 12 ? 16 : num_bytes;
6122
6123          aco_opcode op;
6124          switch (num_bytes) {
6125          case 4:
6126             op = aco_opcode::buffer_load_dword;
6127             break;
6128          case 8:
6129             op = aco_opcode::buffer_load_dwordx2;
6130             break;
6131          case 16:
6132             op = aco_opcode::buffer_load_dwordx4;
6133             break;
6134          default:
6135             unreachable("load_global not implemented for this size.");
6136          }
6137
6138          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6139
6140          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6141          mubuf->operands[0] = Operand(rsrc);
6142          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6143          mubuf->operands[2] = Operand(0u);
6144          mubuf->glc = glc;
6145          mubuf->dlc = false;
6146          mubuf->offset = 0;
6147          mubuf->addr64 = addr.type() == RegType::vgpr;
6148          mubuf->disable_wqm = false;
6149          mubuf->barrier = barrier_buffer;
6150          aco_ptr<Instruction> instr = std::move(mubuf);
6151
6152          /* expand vector */
6153          if (dst.size() == 3) {
6154             Temp vec = bld.tmp(v4);
6155             instr->definitions[0] = Definition(vec);
6156             bld.insert(std::move(instr));
6157             emit_split_vector(ctx, vec, 4);
6158
6159             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
6160             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
6161             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
6162             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
6163          }
6164
6165          if (dst.type() == RegType::sgpr) {
6166             Temp vec = bld.tmp(RegType::vgpr, dst.size());
6167             instr->definitions[0] = Definition(vec);
6168             bld.insert(std::move(instr));
6169             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
6170             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
6171          } else {
6172             instr->definitions[0] = Definition(dst);
6173             bld.insert(std::move(instr));
6174             emit_split_vector(ctx, dst, num_components);
6175          }
6176       }
6177    } else {
6178       switch (num_bytes) {
6179          case 4:
6180             op = aco_opcode::s_load_dword;
6181             break;
6182          case 8:
6183             op = aco_opcode::s_load_dwordx2;
6184             break;
6185          case 12:
6186          case 16:
6187             op = aco_opcode::s_load_dwordx4;
6188             break;
6189          default:
6190             unreachable("load_global not implemented for this size.");
6191       }
6192       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
6193       load->operands[0] = Operand(addr);
6194       load->operands[1] = Operand(0u);
6195       load->definitions[0] = Definition(dst);
6196       load->glc = glc;
6197       load->dlc = dlc;
6198       load->barrier = barrier_buffer;
6199       assert(ctx->options->chip_class >= GFX8 || !glc);
6200
6201       if (dst.size() == 3) {
6202          /* trim vector */
6203          Temp vec = bld.tmp(s4);
6204          load->definitions[0] = Definition(vec);
6205          ctx->block->instructions.emplace_back(std::move(load));
6206          emit_split_vector(ctx, vec, 4);
6207
6208          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6209                     emit_extract_vector(ctx, vec, 0, s1),
6210                     emit_extract_vector(ctx, vec, 1, s1),
6211                     emit_extract_vector(ctx, vec, 2, s1));
6212       } else {
6213          ctx->block->instructions.emplace_back(std::move(load));
6214       }
6215    }
6216 }
6217
6218 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
6219 {
6220    Builder bld(ctx->program, ctx->block);
6221    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6222
6223    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6224    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
6225
6226    if (ctx->options->chip_class >= GFX7)
6227       addr = as_vgpr(ctx, addr);
6228
6229    unsigned writemask = nir_intrinsic_write_mask(instr);
6230    while (writemask) {
6231       int start, count;
6232       u_bit_scan_consecutive_range(&writemask, &start, &count);
6233       if (count == 3 && ctx->options->chip_class == GFX6) {
6234          /* GFX6 doesn't support storing vec3, split it. */
6235          writemask |= 1u << (start + 2);
6236          count = 2;
6237       }
6238       unsigned num_bytes = count * elem_size_bytes;
6239
6240       Temp write_data = data;
6241       if (count != instr->num_components) {
6242          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6243          for (int i = 0; i < count; i++)
6244             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
6245          write_data = bld.tmp(RegType::vgpr, count);
6246          vec->definitions[0] = Definition(write_data);
6247          ctx->block->instructions.emplace_back(std::move(vec));
6248       }
6249
6250       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
6251       unsigned offset = start * elem_size_bytes;
6252
6253       if (ctx->options->chip_class >= GFX7) {
6254          if (offset > 0 && ctx->options->chip_class < GFX9) {
6255             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
6256             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
6257             Temp carry = bld.tmp(bld.lm);
6258             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
6259
6260             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
6261                      Operand(offset), addr0);
6262             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
6263                      Operand(0u), addr1,
6264                      carry).def(1).setHint(vcc);
6265
6266             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
6267
6268             offset = 0;
6269          }
6270
6271          bool global = ctx->options->chip_class >= GFX9;
6272          aco_opcode op;
6273          switch (num_bytes) {
6274          case 4:
6275             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
6276             break;
6277          case 8:
6278             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
6279             break;
6280          case 12:
6281             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
6282             break;
6283          case 16:
6284             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
6285             break;
6286          default:
6287             unreachable("store_global not implemented for this size.");
6288          }
6289
6290          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
6291          flat->operands[0] = Operand(addr);
6292          flat->operands[1] = Operand(s1);
6293          flat->operands[2] = Operand(data);
6294          flat->glc = glc;
6295          flat->dlc = false;
6296          flat->offset = offset;
6297          flat->disable_wqm = true;
6298          flat->barrier = barrier_buffer;
6299          ctx->program->needs_exact = true;
6300          ctx->block->instructions.emplace_back(std::move(flat));
6301       } else {
6302          assert(ctx->options->chip_class == GFX6);
6303
6304          aco_opcode op;
6305          switch (num_bytes) {
6306          case 4:
6307             op = aco_opcode::buffer_store_dword;
6308             break;
6309          case 8:
6310             op = aco_opcode::buffer_store_dwordx2;
6311             break;
6312          case 16:
6313             op = aco_opcode::buffer_store_dwordx4;
6314             break;
6315          default:
6316             unreachable("store_global not implemented for this size.");
6317          }
6318
6319          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6320
6321          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
6322          mubuf->operands[0] = Operand(rsrc);
6323          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6324          mubuf->operands[2] = Operand(0u);
6325          mubuf->operands[3] = Operand(write_data);
6326          mubuf->glc = glc;
6327          mubuf->dlc = false;
6328          mubuf->offset = offset;
6329          mubuf->addr64 = addr.type() == RegType::vgpr;
6330          mubuf->disable_wqm = true;
6331          mubuf->barrier = barrier_buffer;
6332          ctx->program->needs_exact = true;
6333          ctx->block->instructions.emplace_back(std::move(mubuf));
6334       }
6335    }
6336 }
6337
6338 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6339 {
6340    /* return the previous value if dest is ever used */
6341    bool return_previous = false;
6342    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6343       return_previous = true;
6344       break;
6345    }
6346    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6347       return_previous = true;
6348       break;
6349    }
6350
6351    Builder bld(ctx->program, ctx->block);
6352    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6353    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6354
6355    if (ctx->options->chip_class >= GFX7)
6356       addr = as_vgpr(ctx, addr);
6357
6358    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
6359       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
6360                         get_ssa_temp(ctx, instr->src[2].ssa), data);
6361
6362    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6363
6364    aco_opcode op32, op64;
6365
6366    if (ctx->options->chip_class >= GFX7) {
6367       bool global = ctx->options->chip_class >= GFX9;
6368       switch (instr->intrinsic) {
6369          case nir_intrinsic_global_atomic_add:
6370             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
6371             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
6372             break;
6373          case nir_intrinsic_global_atomic_imin:
6374             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
6375             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
6376             break;
6377          case nir_intrinsic_global_atomic_umin:
6378             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
6379             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
6380             break;
6381          case nir_intrinsic_global_atomic_imax:
6382             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
6383             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
6384             break;
6385          case nir_intrinsic_global_atomic_umax:
6386             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
6387             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
6388             break;
6389          case nir_intrinsic_global_atomic_and:
6390             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
6391             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
6392             break;
6393          case nir_intrinsic_global_atomic_or:
6394             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
6395             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
6396             break;
6397          case nir_intrinsic_global_atomic_xor:
6398             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
6399             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
6400             break;
6401          case nir_intrinsic_global_atomic_exchange:
6402             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
6403             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
6404             break;
6405          case nir_intrinsic_global_atomic_comp_swap:
6406             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
6407             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
6408             break;
6409          default:
6410             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6411       }
6412
6413       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6414       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
6415       flat->operands[0] = Operand(addr);
6416       flat->operands[1] = Operand(s1);
6417       flat->operands[2] = Operand(data);
6418       if (return_previous)
6419          flat->definitions[0] = Definition(dst);
6420       flat->glc = return_previous;
6421       flat->dlc = false; /* Not needed for atomics */
6422       flat->offset = 0;
6423       flat->disable_wqm = true;
6424       flat->barrier = barrier_buffer;
6425       ctx->program->needs_exact = true;
6426       ctx->block->instructions.emplace_back(std::move(flat));
6427    } else {
6428       assert(ctx->options->chip_class == GFX6);
6429
6430       switch (instr->intrinsic) {
6431          case nir_intrinsic_global_atomic_add:
6432             op32 = aco_opcode::buffer_atomic_add;
6433             op64 = aco_opcode::buffer_atomic_add_x2;
6434             break;
6435          case nir_intrinsic_global_atomic_imin:
6436             op32 = aco_opcode::buffer_atomic_smin;
6437             op64 = aco_opcode::buffer_atomic_smin_x2;
6438             break;
6439          case nir_intrinsic_global_atomic_umin:
6440             op32 = aco_opcode::buffer_atomic_umin;
6441             op64 = aco_opcode::buffer_atomic_umin_x2;
6442             break;
6443          case nir_intrinsic_global_atomic_imax:
6444             op32 = aco_opcode::buffer_atomic_smax;
6445             op64 = aco_opcode::buffer_atomic_smax_x2;
6446             break;
6447          case nir_intrinsic_global_atomic_umax:
6448             op32 = aco_opcode::buffer_atomic_umax;
6449             op64 = aco_opcode::buffer_atomic_umax_x2;
6450             break;
6451          case nir_intrinsic_global_atomic_and:
6452             op32 = aco_opcode::buffer_atomic_and;
6453             op64 = aco_opcode::buffer_atomic_and_x2;
6454             break;
6455          case nir_intrinsic_global_atomic_or:
6456             op32 = aco_opcode::buffer_atomic_or;
6457             op64 = aco_opcode::buffer_atomic_or_x2;
6458             break;
6459          case nir_intrinsic_global_atomic_xor:
6460             op32 = aco_opcode::buffer_atomic_xor;
6461             op64 = aco_opcode::buffer_atomic_xor_x2;
6462             break;
6463          case nir_intrinsic_global_atomic_exchange:
6464             op32 = aco_opcode::buffer_atomic_swap;
6465             op64 = aco_opcode::buffer_atomic_swap_x2;
6466             break;
6467          case nir_intrinsic_global_atomic_comp_swap:
6468             op32 = aco_opcode::buffer_atomic_cmpswap;
6469             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
6470             break;
6471          default:
6472             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6473       }
6474
6475       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6476
6477       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6478
6479       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
6480       mubuf->operands[0] = Operand(rsrc);
6481       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6482       mubuf->operands[2] = Operand(0u);
6483       mubuf->operands[3] = Operand(data);
6484       if (return_previous)
6485          mubuf->definitions[0] = Definition(dst);
6486       mubuf->glc = return_previous;
6487       mubuf->dlc = false;
6488       mubuf->offset = 0;
6489       mubuf->addr64 = addr.type() == RegType::vgpr;
6490       mubuf->disable_wqm = true;
6491       mubuf->barrier = barrier_buffer;
6492       ctx->program->needs_exact = true;
6493       ctx->block->instructions.emplace_back(std::move(mubuf));
6494    }
6495 }
6496
6497 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
6498    Builder bld(ctx->program, ctx->block);
6499    switch(instr->intrinsic) {
6500       case nir_intrinsic_group_memory_barrier:
6501       case nir_intrinsic_memory_barrier:
6502          bld.barrier(aco_opcode::p_memory_barrier_common);
6503          break;
6504       case nir_intrinsic_memory_barrier_buffer:
6505          bld.barrier(aco_opcode::p_memory_barrier_buffer);
6506          break;
6507       case nir_intrinsic_memory_barrier_image:
6508          bld.barrier(aco_opcode::p_memory_barrier_image);
6509          break;
6510       case nir_intrinsic_memory_barrier_tcs_patch:
6511       case nir_intrinsic_memory_barrier_shared:
6512          bld.barrier(aco_opcode::p_memory_barrier_shared);
6513          break;
6514       default:
6515          unreachable("Unimplemented memory barrier intrinsic");
6516          break;
6517    }
6518 }
6519
6520 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6521 {
6522    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
6523    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6524    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
6525    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6526    Builder bld(ctx->program, ctx->block);
6527
6528    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
6529    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6530    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
6531 }
6532
6533 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6534 {
6535    unsigned writemask = nir_intrinsic_write_mask(instr);
6536    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
6537    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6538    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6539    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
6540
6541    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6542    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
6543 }
6544
6545 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6546 {
6547    unsigned offset = nir_intrinsic_base(instr);
6548    Operand m = load_lds_size_m0(ctx);
6549    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6550    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6551
6552    unsigned num_operands = 3;
6553    aco_opcode op32, op64, op32_rtn, op64_rtn;
6554    switch(instr->intrinsic) {
6555       case nir_intrinsic_shared_atomic_add:
6556          op32 = aco_opcode::ds_add_u32;
6557          op64 = aco_opcode::ds_add_u64;
6558          op32_rtn = aco_opcode::ds_add_rtn_u32;
6559          op64_rtn = aco_opcode::ds_add_rtn_u64;
6560          break;
6561       case nir_intrinsic_shared_atomic_imin:
6562          op32 = aco_opcode::ds_min_i32;
6563          op64 = aco_opcode::ds_min_i64;
6564          op32_rtn = aco_opcode::ds_min_rtn_i32;
6565          op64_rtn = aco_opcode::ds_min_rtn_i64;
6566          break;
6567       case nir_intrinsic_shared_atomic_umin:
6568          op32 = aco_opcode::ds_min_u32;
6569          op64 = aco_opcode::ds_min_u64;
6570          op32_rtn = aco_opcode::ds_min_rtn_u32;
6571          op64_rtn = aco_opcode::ds_min_rtn_u64;
6572          break;
6573       case nir_intrinsic_shared_atomic_imax:
6574          op32 = aco_opcode::ds_max_i32;
6575          op64 = aco_opcode::ds_max_i64;
6576          op32_rtn = aco_opcode::ds_max_rtn_i32;
6577          op64_rtn = aco_opcode::ds_max_rtn_i64;
6578          break;
6579       case nir_intrinsic_shared_atomic_umax:
6580          op32 = aco_opcode::ds_max_u32;
6581          op64 = aco_opcode::ds_max_u64;
6582          op32_rtn = aco_opcode::ds_max_rtn_u32;
6583          op64_rtn = aco_opcode::ds_max_rtn_u64;
6584          break;
6585       case nir_intrinsic_shared_atomic_and:
6586          op32 = aco_opcode::ds_and_b32;
6587          op64 = aco_opcode::ds_and_b64;
6588          op32_rtn = aco_opcode::ds_and_rtn_b32;
6589          op64_rtn = aco_opcode::ds_and_rtn_b64;
6590          break;
6591       case nir_intrinsic_shared_atomic_or:
6592          op32 = aco_opcode::ds_or_b32;
6593          op64 = aco_opcode::ds_or_b64;
6594          op32_rtn = aco_opcode::ds_or_rtn_b32;
6595          op64_rtn = aco_opcode::ds_or_rtn_b64;
6596          break;
6597       case nir_intrinsic_shared_atomic_xor:
6598          op32 = aco_opcode::ds_xor_b32;
6599          op64 = aco_opcode::ds_xor_b64;
6600          op32_rtn = aco_opcode::ds_xor_rtn_b32;
6601          op64_rtn = aco_opcode::ds_xor_rtn_b64;
6602          break;
6603       case nir_intrinsic_shared_atomic_exchange:
6604          op32 = aco_opcode::ds_write_b32;
6605          op64 = aco_opcode::ds_write_b64;
6606          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
6607          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
6608          break;
6609       case nir_intrinsic_shared_atomic_comp_swap:
6610          op32 = aco_opcode::ds_cmpst_b32;
6611          op64 = aco_opcode::ds_cmpst_b64;
6612          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
6613          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
6614          num_operands = 4;
6615          break;
6616       default:
6617          unreachable("Unhandled shared atomic intrinsic");
6618    }
6619
6620    /* return the previous value if dest is ever used */
6621    bool return_previous = false;
6622    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6623       return_previous = true;
6624       break;
6625    }
6626    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6627       return_previous = true;
6628       break;
6629    }
6630
6631    aco_opcode op;
6632    if (data.size() == 1) {
6633       assert(instr->dest.ssa.bit_size == 32);
6634       op = return_previous ? op32_rtn : op32;
6635    } else {
6636       assert(instr->dest.ssa.bit_size == 64);
6637       op = return_previous ? op64_rtn : op64;
6638    }
6639
6640    if (offset > 65535) {
6641       Builder bld(ctx->program, ctx->block);
6642       address = bld.vadd32(bld.def(v1), Operand(offset), address);
6643       offset = 0;
6644    }
6645
6646    aco_ptr<DS_instruction> ds;
6647    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
6648    ds->operands[0] = Operand(address);
6649    ds->operands[1] = Operand(data);
6650    if (num_operands == 4)
6651       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
6652    ds->operands[num_operands - 1] = m;
6653    ds->offset0 = offset;
6654    if (return_previous)
6655       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
6656    ctx->block->instructions.emplace_back(std::move(ds));
6657 }
6658
6659 Temp get_scratch_resource(isel_context *ctx)
6660 {
6661    Builder bld(ctx->program, ctx->block);
6662    Temp scratch_addr = ctx->program->private_segment_buffer;
6663    if (ctx->stage != compute_cs)
6664       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
6665
6666    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
6667                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
6668
6669    if (ctx->program->chip_class >= GFX10) {
6670       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
6671                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
6672                    S_008F0C_RESOURCE_LEVEL(1);
6673    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
6674       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6675                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6676    }
6677
6678    /* older generations need element size = 16 bytes. element size removed in GFX9 */
6679    if (ctx->program->chip_class <= GFX8)
6680       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
6681
6682    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
6683 }
6684
6685 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6686    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
6687    Builder bld(ctx->program, ctx->block);
6688    Temp rsrc = get_scratch_resource(ctx);
6689    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6690    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6691
6692    aco_opcode op;
6693    switch (dst.size()) {
6694       case 1:
6695          op = aco_opcode::buffer_load_dword;
6696          break;
6697       case 2:
6698          op = aco_opcode::buffer_load_dwordx2;
6699          break;
6700       case 3:
6701          op = aco_opcode::buffer_load_dwordx3;
6702          break;
6703       case 4:
6704          op = aco_opcode::buffer_load_dwordx4;
6705          break;
6706       case 6:
6707       case 8: {
6708          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
6709          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
6710                                 bld.def(v4), rsrc, offset,
6711                                 ctx->program->scratch_offset, 0, true);
6712          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
6713                                                   aco_opcode::buffer_load_dwordx4,
6714                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
6715                                 rsrc, offset, ctx->program->scratch_offset, 16, true);
6716          emit_split_vector(ctx, lower, 2);
6717          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
6718          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
6719          if (dst.size() == 8) {
6720             emit_split_vector(ctx, upper, 2);
6721             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
6722             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
6723          } else {
6724             elems[2] = upper;
6725          }
6726
6727          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6728                                                                          Format::PSEUDO, dst.size() / 2, 1)};
6729          for (unsigned i = 0; i < dst.size() / 2; i++)
6730             vec->operands[i] = Operand(elems[i]);
6731          vec->definitions[0] = Definition(dst);
6732          bld.insert(std::move(vec));
6733          ctx->allocated_vec.emplace(dst.id(), elems);
6734          return;
6735       }
6736       default:
6737          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
6738    }
6739
6740    bld.mubuf(op, Definition(dst), rsrc, offset, ctx->program->scratch_offset, 0, true);
6741    emit_split_vector(ctx, dst, instr->num_components);
6742 }
6743
6744 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6745    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
6746    Builder bld(ctx->program, ctx->block);
6747    Temp rsrc = get_scratch_resource(ctx);
6748    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6749    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6750
6751    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6752    unsigned writemask = nir_intrinsic_write_mask(instr);
6753
6754    while (writemask) {
6755       int start, count;
6756       u_bit_scan_consecutive_range(&writemask, &start, &count);
6757       int num_bytes = count * elem_size_bytes;
6758
6759       if (num_bytes > 16) {
6760          assert(elem_size_bytes == 8);
6761          writemask |= (((count - 2) << 1) - 1) << (start + 2);
6762          count = 2;
6763          num_bytes = 16;
6764       }
6765
6766       // TODO: check alignment of sub-dword stores
6767       // TODO: split 3 bytes. there is no store instruction for that
6768
6769       Temp write_data;
6770       if (count != instr->num_components) {
6771          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6772          for (int i = 0; i < count; i++) {
6773             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
6774             vec->operands[i] = Operand(elem);
6775          }
6776          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
6777          vec->definitions[0] = Definition(write_data);
6778          ctx->block->instructions.emplace_back(std::move(vec));
6779       } else {
6780          write_data = data;
6781       }
6782
6783       aco_opcode op;
6784       switch (num_bytes) {
6785          case 4:
6786             op = aco_opcode::buffer_store_dword;
6787             break;
6788          case 8:
6789             op = aco_opcode::buffer_store_dwordx2;
6790             break;
6791          case 12:
6792             op = aco_opcode::buffer_store_dwordx3;
6793             break;
6794          case 16:
6795             op = aco_opcode::buffer_store_dwordx4;
6796             break;
6797          default:
6798             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
6799       }
6800
6801       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
6802    }
6803 }
6804
6805 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
6806    uint8_t log2_ps_iter_samples;
6807    if (ctx->program->info->ps.force_persample) {
6808       log2_ps_iter_samples =
6809          util_logbase2(ctx->options->key.fs.num_samples);
6810    } else {
6811       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
6812    }
6813
6814    /* The bit pattern matches that used by fixed function fragment
6815     * processing. */
6816    static const unsigned ps_iter_masks[] = {
6817       0xffff, /* not used */
6818       0x5555,
6819       0x1111,
6820       0x0101,
6821       0x0001,
6822    };
6823    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
6824
6825    Builder bld(ctx->program, ctx->block);
6826
6827    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
6828                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6829    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
6830    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
6831    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6832    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
6833 }
6834
6835 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
6836    Builder bld(ctx->program, ctx->block);
6837
6838    unsigned stream = nir_intrinsic_stream_id(instr);
6839    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6840    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
6841    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
6842
6843    /* get GSVS ring */
6844    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
6845
6846    unsigned num_components =
6847       ctx->program->info->gs.num_stream_output_components[stream];
6848    assert(num_components);
6849
6850    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
6851    unsigned stream_offset = 0;
6852    for (unsigned i = 0; i < stream; i++) {
6853       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
6854       stream_offset += prev_stride * ctx->program->wave_size;
6855    }
6856
6857    /* Limit on the stride field for <= GFX7. */
6858    assert(stride < (1 << 14));
6859
6860    Temp gsvs_dwords[4];
6861    for (unsigned i = 0; i < 4; i++)
6862       gsvs_dwords[i] = bld.tmp(s1);
6863    bld.pseudo(aco_opcode::p_split_vector,
6864               Definition(gsvs_dwords[0]),
6865               Definition(gsvs_dwords[1]),
6866               Definition(gsvs_dwords[2]),
6867               Definition(gsvs_dwords[3]),
6868               gsvs_ring);
6869
6870    if (stream_offset) {
6871       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
6872
6873       Temp carry = bld.tmp(s1);
6874       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
6875       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
6876    }
6877
6878    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
6879    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
6880
6881    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6882                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
6883
6884    unsigned offset = 0;
6885    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
6886       if (ctx->program->info->gs.output_streams[i] != stream)
6887          continue;
6888
6889       for (unsigned j = 0; j < 4; j++) {
6890          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
6891             continue;
6892
6893          if (ctx->outputs.mask[i] & (1 << j)) {
6894             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
6895             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
6896             if (const_offset >= 4096u) {
6897                if (vaddr_offset.isUndefined())
6898                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
6899                else
6900                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
6901                const_offset %= 4096u;
6902             }
6903
6904             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
6905             mtbuf->operands[0] = Operand(gsvs_ring);
6906             mtbuf->operands[1] = vaddr_offset;
6907             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
6908             mtbuf->operands[3] = Operand(ctx->outputs.temps[i * 4u + j]);
6909             mtbuf->offen = !vaddr_offset.isUndefined();
6910             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
6911             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
6912             mtbuf->offset = const_offset;
6913             mtbuf->glc = true;
6914             mtbuf->slc = true;
6915             mtbuf->barrier = barrier_gs_data;
6916             mtbuf->can_reorder = true;
6917             bld.insert(std::move(mtbuf));
6918          }
6919
6920          offset += ctx->shader->info.gs.vertices_out;
6921       }
6922
6923       /* outputs for the next vertex are undefined and keeping them around can
6924        * create invalid IR with control flow */
6925       ctx->outputs.mask[i] = 0;
6926    }
6927
6928    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
6929 }
6930
6931 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
6932 {
6933    Builder bld(ctx->program, ctx->block);
6934
6935    if (cluster_size == 1) {
6936       return src;
6937    } if (op == nir_op_iand && cluster_size == 4) {
6938       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
6939       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6940       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
6941                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
6942    } else if (op == nir_op_ior && cluster_size == 4) {
6943       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
6944       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
6945                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
6946    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
6947       //subgroupAnd(val) -> (exec & ~val) == 0
6948       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6949       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6950       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
6951    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
6952       //subgroupOr(val) -> (val & exec) != 0
6953       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
6954       return bool_to_vector_condition(ctx, tmp);
6955    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
6956       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
6957       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6958       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
6959       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
6960       return bool_to_vector_condition(ctx, tmp);
6961    } else {
6962       //subgroupClustered{And,Or,Xor}(val, n) ->
6963       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
6964       //cluster_offset = ~(n - 1) & lane_id
6965       //cluster_mask = ((1 << n) - 1)
6966       //subgroupClusteredAnd():
6967       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
6968       //subgroupClusteredOr():
6969       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
6970       //subgroupClusteredXor():
6971       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
6972       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
6973       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
6974
6975       Temp tmp;
6976       if (op == nir_op_iand)
6977          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6978       else
6979          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6980
6981       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
6982
6983       if (ctx->program->chip_class <= GFX7)
6984          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
6985       else if (ctx->program->wave_size == 64)
6986          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
6987       else
6988          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
6989       tmp = emit_extract_vector(ctx, tmp, 0, v1);
6990       if (cluster_mask != 0xffffffff)
6991          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
6992
6993       Definition cmp_def = Definition();
6994       if (op == nir_op_iand) {
6995          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
6996       } else if (op == nir_op_ior) {
6997          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6998       } else if (op == nir_op_ixor) {
6999          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
7000                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
7001          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
7002       }
7003       cmp_def.setHint(vcc);
7004       return cmp_def.getTemp();
7005    }
7006 }
7007
7008 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
7009 {
7010    Builder bld(ctx->program, ctx->block);
7011
7012    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
7013    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
7014    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
7015    Temp tmp;
7016    if (op == nir_op_iand)
7017       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
7018    else
7019       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
7020
7021    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
7022    Temp lo = lohi.def(0).getTemp();
7023    Temp hi = lohi.def(1).getTemp();
7024    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
7025
7026    Definition cmp_def = Definition();
7027    if (op == nir_op_iand)
7028       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
7029    else if (op == nir_op_ior)
7030       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
7031    else if (op == nir_op_ixor)
7032       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
7033                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
7034    cmp_def.setHint(vcc);
7035    return cmp_def.getTemp();
7036 }
7037
7038 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
7039 {
7040    Builder bld(ctx->program, ctx->block);
7041
7042    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
7043    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
7044    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
7045    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
7046    if (op == nir_op_iand)
7047       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7048    else if (op == nir_op_ior)
7049       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7050    else if (op == nir_op_ixor)
7051       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7052
7053    assert(false);
7054    return Temp();
7055 }
7056
7057 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
7058 {
7059    Builder bld(ctx->program, ctx->block);
7060    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
7061    if (src.regClass().type() == RegType::vgpr) {
7062       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
7063    } else if (src.regClass() == s1) {
7064       bld.sop1(aco_opcode::s_mov_b32, dst, src);
7065    } else if (src.regClass() == s2) {
7066       bld.sop1(aco_opcode::s_mov_b64, dst, src);
7067    } else {
7068       fprintf(stderr, "Unimplemented NIR instr bit size: ");
7069       nir_print_instr(&instr->instr, stderr);
7070       fprintf(stderr, "\n");
7071    }
7072 }
7073
7074 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
7075 {
7076    Builder bld(ctx->program, ctx->block);
7077    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
7078    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
7079    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
7080
7081    Temp ddx_1, ddx_2, ddy_1, ddy_2;
7082    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
7083    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
7084    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
7085
7086    /* Build DD X/Y */
7087    if (ctx->program->chip_class >= GFX8) {
7088       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
7089       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
7090       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
7091       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
7092       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
7093       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
7094    } else {
7095       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
7096       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
7097       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
7098       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
7099       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
7100       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
7101       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
7102       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
7103       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
7104       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
7105    }
7106
7107    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
7108    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
7109    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
7110    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
7111    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
7112    Temp wqm1 = bld.tmp(v1);
7113    emit_wqm(ctx, tmp1, wqm1, true);
7114    Temp wqm2 = bld.tmp(v1);
7115    emit_wqm(ctx, tmp2, wqm2, true);
7116    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
7117    return;
7118 }
7119
7120 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
7121 {
7122    Builder bld(ctx->program, ctx->block);
7123    switch(instr->intrinsic) {
7124    case nir_intrinsic_load_barycentric_sample:
7125    case nir_intrinsic_load_barycentric_pixel:
7126    case nir_intrinsic_load_barycentric_centroid: {
7127       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
7128       Temp bary = Temp(0, s2);
7129       switch (mode) {
7130       case INTERP_MODE_SMOOTH:
7131       case INTERP_MODE_NONE:
7132          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7133             bary = get_arg(ctx, ctx->args->ac.persp_center);
7134          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7135             bary = ctx->persp_centroid;
7136          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7137             bary = get_arg(ctx, ctx->args->ac.persp_sample);
7138          break;
7139       case INTERP_MODE_NOPERSPECTIVE:
7140          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7141             bary = get_arg(ctx, ctx->args->ac.linear_center);
7142          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7143             bary = ctx->linear_centroid;
7144          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7145             bary = get_arg(ctx, ctx->args->ac.linear_sample);
7146          break;
7147       default:
7148          break;
7149       }
7150       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7151       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
7152       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
7153       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7154                  Operand(p1), Operand(p2));
7155       emit_split_vector(ctx, dst, 2);
7156       break;
7157    }
7158    case nir_intrinsic_load_barycentric_model: {
7159       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
7160
7161       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7162       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
7163       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
7164       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
7165       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7166                  Operand(p1), Operand(p2), Operand(p3));
7167       emit_split_vector(ctx, dst, 3);
7168       break;
7169    }
7170    case nir_intrinsic_load_barycentric_at_sample: {
7171       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
7172       switch (ctx->options->key.fs.num_samples) {
7173          case 2: sample_pos_offset += 1 << 3; break;
7174          case 4: sample_pos_offset += 3 << 3; break;
7175          case 8: sample_pos_offset += 7 << 3; break;
7176          default: break;
7177       }
7178       Temp sample_pos;
7179       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
7180       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
7181       Temp private_segment_buffer = ctx->program->private_segment_buffer;
7182       if (addr.type() == RegType::sgpr) {
7183          Operand offset;
7184          if (const_addr) {
7185             sample_pos_offset += const_addr->u32 << 3;
7186             offset = Operand(sample_pos_offset);
7187          } else if (ctx->options->chip_class >= GFX9) {
7188             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7189          } else {
7190             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
7191             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7192          }
7193
7194          Operand off = bld.copy(bld.def(s1), Operand(offset));
7195          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
7196
7197       } else if (ctx->options->chip_class >= GFX9) {
7198          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7199          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
7200       } else if (ctx->options->chip_class >= GFX7) {
7201          /* addr += private_segment_buffer + sample_pos_offset */
7202          Temp tmp0 = bld.tmp(s1);
7203          Temp tmp1 = bld.tmp(s1);
7204          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
7205          Definition scc_tmp = bld.def(s1, scc);
7206          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
7207          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
7208          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7209          Temp pck0 = bld.tmp(v1);
7210          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
7211          tmp1 = as_vgpr(ctx, tmp1);
7212          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
7213          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
7214
7215          /* sample_pos = flat_load_dwordx2 addr */
7216          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
7217       } else {
7218          assert(ctx->options->chip_class == GFX6);
7219
7220          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
7221                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
7222          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
7223
7224          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7225          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
7226
7227          sample_pos = bld.tmp(v2);
7228
7229          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
7230          load->definitions[0] = Definition(sample_pos);
7231          load->operands[0] = Operand(rsrc);
7232          load->operands[1] = Operand(addr);
7233          load->operands[2] = Operand(0u);
7234          load->offset = sample_pos_offset;
7235          load->offen = 0;
7236          load->addr64 = true;
7237          load->glc = false;
7238          load->dlc = false;
7239          load->disable_wqm = false;
7240          load->barrier = barrier_none;
7241          load->can_reorder = true;
7242          ctx->block->instructions.emplace_back(std::move(load));
7243       }
7244
7245       /* sample_pos -= 0.5 */
7246       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
7247       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
7248       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
7249       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
7250       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
7251
7252       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7253       break;
7254    }
7255    case nir_intrinsic_load_barycentric_at_offset: {
7256       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
7257       RegClass rc = RegClass(offset.type(), 1);
7258       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
7259       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
7260       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7261       break;
7262    }
7263    case nir_intrinsic_load_front_face: {
7264       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7265                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
7266       break;
7267    }
7268    case nir_intrinsic_load_view_index: {
7269       if (ctx->stage & (sw_vs | sw_gs | sw_tcs | sw_tes)) {
7270          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7271          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
7272          break;
7273       }
7274
7275       /* fallthrough */
7276    }
7277    case nir_intrinsic_load_layer_id: {
7278       unsigned idx = nir_intrinsic_base(instr);
7279       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7280                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
7281       break;
7282    }
7283    case nir_intrinsic_load_frag_coord: {
7284       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
7285       break;
7286    }
7287    case nir_intrinsic_load_sample_pos: {
7288       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
7289       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
7290       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7291                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
7292                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
7293       break;
7294    }
7295    case nir_intrinsic_load_tess_coord:
7296       visit_load_tess_coord(ctx, instr);
7297       break;
7298    case nir_intrinsic_load_interpolated_input:
7299       visit_load_interpolated_input(ctx, instr);
7300       break;
7301    case nir_intrinsic_store_output:
7302       visit_store_output(ctx, instr);
7303       break;
7304    case nir_intrinsic_load_input:
7305    case nir_intrinsic_load_input_vertex:
7306       visit_load_input(ctx, instr);
7307       break;
7308    case nir_intrinsic_load_output:
7309       visit_load_output(ctx, instr);
7310       break;
7311    case nir_intrinsic_load_per_vertex_input:
7312       visit_load_per_vertex_input(ctx, instr);
7313       break;
7314    case nir_intrinsic_load_per_vertex_output:
7315       visit_load_per_vertex_output(ctx, instr);
7316       break;
7317    case nir_intrinsic_store_per_vertex_output:
7318       visit_store_per_vertex_output(ctx, instr);
7319       break;
7320    case nir_intrinsic_load_ubo:
7321       visit_load_ubo(ctx, instr);
7322       break;
7323    case nir_intrinsic_load_push_constant:
7324       visit_load_push_constant(ctx, instr);
7325       break;
7326    case nir_intrinsic_load_constant:
7327       visit_load_constant(ctx, instr);
7328       break;
7329    case nir_intrinsic_vulkan_resource_index:
7330       visit_load_resource(ctx, instr);
7331       break;
7332    case nir_intrinsic_discard:
7333       visit_discard(ctx, instr);
7334       break;
7335    case nir_intrinsic_discard_if:
7336       visit_discard_if(ctx, instr);
7337       break;
7338    case nir_intrinsic_load_shared:
7339       visit_load_shared(ctx, instr);
7340       break;
7341    case nir_intrinsic_store_shared:
7342       visit_store_shared(ctx, instr);
7343       break;
7344    case nir_intrinsic_shared_atomic_add:
7345    case nir_intrinsic_shared_atomic_imin:
7346    case nir_intrinsic_shared_atomic_umin:
7347    case nir_intrinsic_shared_atomic_imax:
7348    case nir_intrinsic_shared_atomic_umax:
7349    case nir_intrinsic_shared_atomic_and:
7350    case nir_intrinsic_shared_atomic_or:
7351    case nir_intrinsic_shared_atomic_xor:
7352    case nir_intrinsic_shared_atomic_exchange:
7353    case nir_intrinsic_shared_atomic_comp_swap:
7354       visit_shared_atomic(ctx, instr);
7355       break;
7356    case nir_intrinsic_image_deref_load:
7357       visit_image_load(ctx, instr);
7358       break;
7359    case nir_intrinsic_image_deref_store:
7360       visit_image_store(ctx, instr);
7361       break;
7362    case nir_intrinsic_image_deref_atomic_add:
7363    case nir_intrinsic_image_deref_atomic_umin:
7364    case nir_intrinsic_image_deref_atomic_imin:
7365    case nir_intrinsic_image_deref_atomic_umax:
7366    case nir_intrinsic_image_deref_atomic_imax:
7367    case nir_intrinsic_image_deref_atomic_and:
7368    case nir_intrinsic_image_deref_atomic_or:
7369    case nir_intrinsic_image_deref_atomic_xor:
7370    case nir_intrinsic_image_deref_atomic_exchange:
7371    case nir_intrinsic_image_deref_atomic_comp_swap:
7372       visit_image_atomic(ctx, instr);
7373       break;
7374    case nir_intrinsic_image_deref_size:
7375       visit_image_size(ctx, instr);
7376       break;
7377    case nir_intrinsic_load_ssbo:
7378       visit_load_ssbo(ctx, instr);
7379       break;
7380    case nir_intrinsic_store_ssbo:
7381       visit_store_ssbo(ctx, instr);
7382       break;
7383    case nir_intrinsic_load_global:
7384       visit_load_global(ctx, instr);
7385       break;
7386    case nir_intrinsic_store_global:
7387       visit_store_global(ctx, instr);
7388       break;
7389    case nir_intrinsic_global_atomic_add:
7390    case nir_intrinsic_global_atomic_imin:
7391    case nir_intrinsic_global_atomic_umin:
7392    case nir_intrinsic_global_atomic_imax:
7393    case nir_intrinsic_global_atomic_umax:
7394    case nir_intrinsic_global_atomic_and:
7395    case nir_intrinsic_global_atomic_or:
7396    case nir_intrinsic_global_atomic_xor:
7397    case nir_intrinsic_global_atomic_exchange:
7398    case nir_intrinsic_global_atomic_comp_swap:
7399       visit_global_atomic(ctx, instr);
7400       break;
7401    case nir_intrinsic_ssbo_atomic_add:
7402    case nir_intrinsic_ssbo_atomic_imin:
7403    case nir_intrinsic_ssbo_atomic_umin:
7404    case nir_intrinsic_ssbo_atomic_imax:
7405    case nir_intrinsic_ssbo_atomic_umax:
7406    case nir_intrinsic_ssbo_atomic_and:
7407    case nir_intrinsic_ssbo_atomic_or:
7408    case nir_intrinsic_ssbo_atomic_xor:
7409    case nir_intrinsic_ssbo_atomic_exchange:
7410    case nir_intrinsic_ssbo_atomic_comp_swap:
7411       visit_atomic_ssbo(ctx, instr);
7412       break;
7413    case nir_intrinsic_load_scratch:
7414       visit_load_scratch(ctx, instr);
7415       break;
7416    case nir_intrinsic_store_scratch:
7417       visit_store_scratch(ctx, instr);
7418       break;
7419    case nir_intrinsic_get_buffer_size:
7420       visit_get_buffer_size(ctx, instr);
7421       break;
7422    case nir_intrinsic_control_barrier: {
7423       if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7424          /* GFX6 only (thanks to a hw bug workaround):
7425           * The real barrier instruction isn’t needed, because an entire patch
7426           * always fits into a single wave.
7427           */
7428          break;
7429       }
7430
7431       if (ctx->program->workgroup_size > ctx->program->wave_size)
7432          bld.sopp(aco_opcode::s_barrier);
7433
7434       break;
7435    }
7436    case nir_intrinsic_memory_barrier_tcs_patch:
7437    case nir_intrinsic_group_memory_barrier:
7438    case nir_intrinsic_memory_barrier:
7439    case nir_intrinsic_memory_barrier_buffer:
7440    case nir_intrinsic_memory_barrier_image:
7441    case nir_intrinsic_memory_barrier_shared:
7442       emit_memory_barrier(ctx, instr);
7443       break;
7444    case nir_intrinsic_load_num_work_groups: {
7445       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7446       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
7447       emit_split_vector(ctx, dst, 3);
7448       break;
7449    }
7450    case nir_intrinsic_load_local_invocation_id: {
7451       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7452       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
7453       emit_split_vector(ctx, dst, 3);
7454       break;
7455    }
7456    case nir_intrinsic_load_work_group_id: {
7457       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7458       struct ac_arg *args = ctx->args->ac.workgroup_ids;
7459       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7460                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
7461                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
7462                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
7463       emit_split_vector(ctx, dst, 3);
7464       break;
7465    }
7466    case nir_intrinsic_load_local_invocation_index: {
7467       Temp id = emit_mbcnt(ctx, bld.def(v1));
7468
7469       /* The tg_size bits [6:11] contain the subgroup id,
7470        * we need this multiplied by the wave size, and then OR the thread id to it.
7471        */
7472       if (ctx->program->wave_size == 64) {
7473          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
7474          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
7475                                 get_arg(ctx, ctx->args->ac.tg_size));
7476          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
7477       } else {
7478          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
7479          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7480                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7481          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
7482       }
7483       break;
7484    }
7485    case nir_intrinsic_load_subgroup_id: {
7486       if (ctx->stage == compute_cs) {
7487          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
7488                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7489       } else {
7490          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
7491       }
7492       break;
7493    }
7494    case nir_intrinsic_load_subgroup_invocation: {
7495       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
7496       break;
7497    }
7498    case nir_intrinsic_load_num_subgroups: {
7499       if (ctx->stage == compute_cs)
7500          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
7501                   get_arg(ctx, ctx->args->ac.tg_size));
7502       else
7503          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
7504       break;
7505    }
7506    case nir_intrinsic_ballot: {
7507       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7508       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7509       Definition tmp = bld.def(dst.regClass());
7510       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
7511       if (instr->src[0].ssa->bit_size == 1) {
7512          assert(src.regClass() == bld.lm);
7513          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
7514       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
7515          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
7516       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
7517          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
7518       } else {
7519          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7520          nir_print_instr(&instr->instr, stderr);
7521          fprintf(stderr, "\n");
7522       }
7523       if (dst.size() != bld.lm.size()) {
7524          /* Wave32 with ballot size set to 64 */
7525          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
7526       }
7527       emit_wqm(ctx, tmp.getTemp(), dst);
7528       break;
7529    }
7530    case nir_intrinsic_shuffle:
7531    case nir_intrinsic_read_invocation: {
7532       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7533       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
7534          emit_uniform_subgroup(ctx, instr, src);
7535       } else {
7536          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
7537          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
7538             tid = bld.as_uniform(tid);
7539          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7540          if (src.regClass() == v1) {
7541             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
7542          } else if (src.regClass() == v2) {
7543             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7544             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7545             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
7546             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
7547             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7548             emit_split_vector(ctx, dst, 2);
7549          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
7550             assert(src.regClass() == bld.lm);
7551             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
7552             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7553          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
7554             assert(src.regClass() == bld.lm);
7555             Temp tmp;
7556             if (ctx->program->chip_class <= GFX7)
7557                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
7558             else if (ctx->program->wave_size == 64)
7559                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
7560             else
7561                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
7562             tmp = emit_extract_vector(ctx, tmp, 0, v1);
7563             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
7564             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
7565          } else {
7566             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7567             nir_print_instr(&instr->instr, stderr);
7568             fprintf(stderr, "\n");
7569          }
7570       }
7571       break;
7572    }
7573    case nir_intrinsic_load_sample_id: {
7574       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7575                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
7576       break;
7577    }
7578    case nir_intrinsic_load_sample_mask_in: {
7579       visit_load_sample_mask_in(ctx, instr);
7580       break;
7581    }
7582    case nir_intrinsic_read_first_invocation: {
7583       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7584       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7585       if (src.regClass() == v1) {
7586          emit_wqm(ctx,
7587                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
7588                   dst);
7589       } else if (src.regClass() == v2) {
7590          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7591          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7592          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
7593          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
7594          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7595          emit_split_vector(ctx, dst, 2);
7596       } else if (instr->dest.ssa.bit_size == 1) {
7597          assert(src.regClass() == bld.lm);
7598          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
7599                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
7600          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7601       } else if (src.regClass() == s1) {
7602          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
7603       } else if (src.regClass() == s2) {
7604          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
7605       } else {
7606          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7607          nir_print_instr(&instr->instr, stderr);
7608          fprintf(stderr, "\n");
7609       }
7610       break;
7611    }
7612    case nir_intrinsic_vote_all: {
7613       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7614       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7615       assert(src.regClass() == bld.lm);
7616       assert(dst.regClass() == bld.lm);
7617
7618       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
7619       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
7620       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
7621       break;
7622    }
7623    case nir_intrinsic_vote_any: {
7624       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7625       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7626       assert(src.regClass() == bld.lm);
7627       assert(dst.regClass() == bld.lm);
7628
7629       Temp tmp = bool_to_scalar_condition(ctx, src);
7630       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7631       break;
7632    }
7633    case nir_intrinsic_reduce:
7634    case nir_intrinsic_inclusive_scan:
7635    case nir_intrinsic_exclusive_scan: {
7636       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7637       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7638       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
7639       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
7640          nir_intrinsic_cluster_size(instr) : 0;
7641       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
7642
7643       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
7644          emit_uniform_subgroup(ctx, instr, src);
7645       } else if (instr->dest.ssa.bit_size == 1) {
7646          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
7647             op = nir_op_iand;
7648          else if (op == nir_op_iadd)
7649             op = nir_op_ixor;
7650          else if (op == nir_op_umax || op == nir_op_imax)
7651             op = nir_op_ior;
7652          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
7653
7654          switch (instr->intrinsic) {
7655          case nir_intrinsic_reduce:
7656             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
7657             break;
7658          case nir_intrinsic_exclusive_scan:
7659             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
7660             break;
7661          case nir_intrinsic_inclusive_scan:
7662             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
7663             break;
7664          default:
7665             assert(false);
7666          }
7667       } else if (cluster_size == 1) {
7668          bld.copy(Definition(dst), src);
7669       } else {
7670          src = as_vgpr(ctx, src);
7671
7672          ReduceOp reduce_op;
7673          switch (op) {
7674          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
7675             CASE(iadd)
7676             CASE(imul)
7677             CASE(fadd)
7678             CASE(fmul)
7679             CASE(imin)
7680             CASE(umin)
7681             CASE(fmin)
7682             CASE(imax)
7683             CASE(umax)
7684             CASE(fmax)
7685             CASE(iand)
7686             CASE(ior)
7687             CASE(ixor)
7688             default:
7689                unreachable("unknown reduction op");
7690          #undef CASE
7691          }
7692
7693          aco_opcode aco_op;
7694          switch (instr->intrinsic) {
7695             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
7696             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
7697             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
7698             default:
7699                unreachable("unknown reduce intrinsic");
7700          }
7701
7702          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
7703          reduce->operands[0] = Operand(src);
7704          // filled in by aco_reduce_assign.cpp, used internally as part of the
7705          // reduce sequence
7706          assert(dst.size() == 1 || dst.size() == 2);
7707          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
7708          reduce->operands[2] = Operand(v1.as_linear());
7709
7710          Temp tmp_dst = bld.tmp(dst.regClass());
7711          reduce->definitions[0] = Definition(tmp_dst);
7712          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
7713          reduce->definitions[2] = Definition();
7714          reduce->definitions[3] = Definition(scc, s1);
7715          reduce->definitions[4] = Definition();
7716          reduce->reduce_op = reduce_op;
7717          reduce->cluster_size = cluster_size;
7718          ctx->block->instructions.emplace_back(std::move(reduce));
7719
7720          emit_wqm(ctx, tmp_dst, dst);
7721       }
7722       break;
7723    }
7724    case nir_intrinsic_quad_broadcast: {
7725       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7726       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7727          emit_uniform_subgroup(ctx, instr, src);
7728       } else {
7729          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7730          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
7731          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
7732
7733          if (instr->dest.ssa.bit_size == 1) {
7734             assert(src.regClass() == bld.lm);
7735             assert(dst.regClass() == bld.lm);
7736             uint32_t half_mask = 0x11111111u << lane;
7737             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
7738             Temp tmp = bld.tmp(bld.lm);
7739             bld.sop1(Builder::s_wqm, Definition(tmp),
7740                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
7741                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
7742             emit_wqm(ctx, tmp, dst);
7743          } else if (instr->dest.ssa.bit_size == 32) {
7744             if (ctx->program->chip_class >= GFX8)
7745                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
7746             else
7747                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
7748          } else if (instr->dest.ssa.bit_size == 64) {
7749             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7750             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7751             if (ctx->program->chip_class >= GFX8) {
7752                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7753                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7754             } else {
7755                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
7756                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
7757             }
7758             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7759             emit_split_vector(ctx, dst, 2);
7760          } else {
7761             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7762             nir_print_instr(&instr->instr, stderr);
7763             fprintf(stderr, "\n");
7764          }
7765       }
7766       break;
7767    }
7768    case nir_intrinsic_quad_swap_horizontal:
7769    case nir_intrinsic_quad_swap_vertical:
7770    case nir_intrinsic_quad_swap_diagonal:
7771    case nir_intrinsic_quad_swizzle_amd: {
7772       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7773       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7774          emit_uniform_subgroup(ctx, instr, src);
7775          break;
7776       }
7777       uint16_t dpp_ctrl = 0;
7778       switch (instr->intrinsic) {
7779       case nir_intrinsic_quad_swap_horizontal:
7780          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
7781          break;
7782       case nir_intrinsic_quad_swap_vertical:
7783          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
7784          break;
7785       case nir_intrinsic_quad_swap_diagonal:
7786          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
7787          break;
7788       case nir_intrinsic_quad_swizzle_amd:
7789          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
7790          break;
7791       default:
7792          break;
7793       }
7794       if (ctx->program->chip_class < GFX8)
7795          dpp_ctrl |= (1 << 15);
7796
7797       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7798       if (instr->dest.ssa.bit_size == 1) {
7799          assert(src.regClass() == bld.lm);
7800          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
7801          if (ctx->program->chip_class >= GFX8)
7802             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7803          else
7804             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7805          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
7806          emit_wqm(ctx, tmp, dst);
7807       } else if (instr->dest.ssa.bit_size == 32) {
7808          Temp tmp;
7809          if (ctx->program->chip_class >= GFX8)
7810             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7811          else
7812             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7813          emit_wqm(ctx, tmp, dst);
7814       } else if (instr->dest.ssa.bit_size == 64) {
7815          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7816          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7817          if (ctx->program->chip_class >= GFX8) {
7818             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7819             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7820          } else {
7821             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
7822             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
7823          }
7824          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7825          emit_split_vector(ctx, dst, 2);
7826       } else {
7827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7828          nir_print_instr(&instr->instr, stderr);
7829          fprintf(stderr, "\n");
7830       }
7831       break;
7832    }
7833    case nir_intrinsic_masked_swizzle_amd: {
7834       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7835       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7836          emit_uniform_subgroup(ctx, instr, src);
7837          break;
7838       }
7839       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7840       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
7841       if (dst.regClass() == v1) {
7842          emit_wqm(ctx,
7843                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
7844                   dst);
7845       } else if (dst.regClass() == v2) {
7846          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7847          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7848          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
7849          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
7850          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7851          emit_split_vector(ctx, dst, 2);
7852       } else {
7853          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7854          nir_print_instr(&instr->instr, stderr);
7855          fprintf(stderr, "\n");
7856       }
7857       break;
7858    }
7859    case nir_intrinsic_write_invocation_amd: {
7860       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
7861       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
7862       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
7863       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7864       if (dst.regClass() == v1) {
7865          /* src2 is ignored for writelane. RA assigns the same reg for dst */
7866          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
7867       } else if (dst.regClass() == v2) {
7868          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
7869          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
7870          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
7871          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
7872          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
7873          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
7874          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7875          emit_split_vector(ctx, dst, 2);
7876       } else {
7877          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7878          nir_print_instr(&instr->instr, stderr);
7879          fprintf(stderr, "\n");
7880       }
7881       break;
7882    }
7883    case nir_intrinsic_mbcnt_amd: {
7884       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7885       RegClass rc = RegClass(src.type(), 1);
7886       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
7887       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
7888       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7889       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
7890       emit_wqm(ctx, wqm_tmp, dst);
7891       break;
7892    }
7893    case nir_intrinsic_load_helper_invocation: {
7894       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7895       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
7896       ctx->block->kind |= block_kind_needs_lowering;
7897       ctx->program->needs_exact = true;
7898       break;
7899    }
7900    case nir_intrinsic_is_helper_invocation: {
7901       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7902       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
7903       ctx->block->kind |= block_kind_needs_lowering;
7904       ctx->program->needs_exact = true;
7905       break;
7906    }
7907    case nir_intrinsic_demote:
7908       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
7909
7910       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7911          ctx->cf_info.exec_potentially_empty_discard = true;
7912       ctx->block->kind |= block_kind_uses_demote;
7913       ctx->program->needs_exact = true;
7914       break;
7915    case nir_intrinsic_demote_if: {
7916       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7917       assert(src.regClass() == bld.lm);
7918       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
7919       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
7920
7921       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7922          ctx->cf_info.exec_potentially_empty_discard = true;
7923       ctx->block->kind |= block_kind_uses_demote;
7924       ctx->program->needs_exact = true;
7925       break;
7926    }
7927    case nir_intrinsic_first_invocation: {
7928       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
7929                get_ssa_temp(ctx, &instr->dest.ssa));
7930       break;
7931    }
7932    case nir_intrinsic_shader_clock:
7933       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
7934       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
7935       break;
7936    case nir_intrinsic_load_vertex_id_zero_base: {
7937       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7938       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
7939       break;
7940    }
7941    case nir_intrinsic_load_first_vertex: {
7942       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7943       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
7944       break;
7945    }
7946    case nir_intrinsic_load_base_instance: {
7947       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7948       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
7949       break;
7950    }
7951    case nir_intrinsic_load_instance_id: {
7952       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7953       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
7954       break;
7955    }
7956    case nir_intrinsic_load_draw_id: {
7957       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7958       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
7959       break;
7960    }
7961    case nir_intrinsic_load_invocation_id: {
7962       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7963
7964       if (ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
7965          if (ctx->options->chip_class >= GFX10)
7966             bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7967          else
7968             bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7969       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7970          bld.vop3(aco_opcode::v_bfe_u32, Definition(dst),
7971                   get_arg(ctx, ctx->args->ac.tcs_rel_ids), Operand(8u), Operand(5u));
7972       } else {
7973          unreachable("Unsupported stage for load_invocation_id");
7974       }
7975
7976       break;
7977    }
7978    case nir_intrinsic_load_primitive_id: {
7979       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7980
7981       switch (ctx->shader->info.stage) {
7982       case MESA_SHADER_GEOMETRY:
7983          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
7984          break;
7985       case MESA_SHADER_TESS_CTRL:
7986          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tcs_patch_id));
7987          break;
7988       case MESA_SHADER_TESS_EVAL:
7989          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tes_patch_id));
7990          break;
7991       default:
7992          unreachable("Unimplemented shader stage for nir_intrinsic_load_primitive_id");
7993       }
7994
7995       break;
7996    }
7997    case nir_intrinsic_load_patch_vertices_in: {
7998       assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL ||
7999              ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
8000
8001       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8002       bld.copy(Definition(dst), Operand(ctx->args->options->key.tcs.input_vertices));
8003       break;
8004    }
8005    case nir_intrinsic_emit_vertex_with_counter: {
8006       visit_emit_vertex_with_counter(ctx, instr);
8007       break;
8008    }
8009    case nir_intrinsic_end_primitive_with_counter: {
8010       unsigned stream = nir_intrinsic_stream_id(instr);
8011       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
8012       break;
8013    }
8014    case nir_intrinsic_set_vertex_count: {
8015       /* unused, the HW keeps track of this for us */
8016       break;
8017    }
8018    default:
8019       fprintf(stderr, "Unimplemented intrinsic instr: ");
8020       nir_print_instr(&instr->instr, stderr);
8021       fprintf(stderr, "\n");
8022       abort();
8023
8024       break;
8025    }
8026 }
8027
8028
8029 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
8030                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
8031                     enum glsl_base_type *stype)
8032 {
8033    nir_deref_instr *texture_deref_instr = NULL;
8034    nir_deref_instr *sampler_deref_instr = NULL;
8035    int plane = -1;
8036
8037    for (unsigned i = 0; i < instr->num_srcs; i++) {
8038       switch (instr->src[i].src_type) {
8039       case nir_tex_src_texture_deref:
8040          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
8041          break;
8042       case nir_tex_src_sampler_deref:
8043          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
8044          break;
8045       case nir_tex_src_plane:
8046          plane = nir_src_as_int(instr->src[i].src);
8047          break;
8048       default:
8049          break;
8050       }
8051    }
8052
8053    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
8054
8055    if (!sampler_deref_instr)
8056       sampler_deref_instr = texture_deref_instr;
8057
8058    if (plane >= 0) {
8059       assert(instr->op != nir_texop_txf_ms &&
8060              instr->op != nir_texop_samples_identical);
8061       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
8062       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
8063    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
8064       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
8065    } else if (instr->op == nir_texop_fragment_mask_fetch) {
8066       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8067    } else {
8068       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
8069    }
8070    if (samp_ptr) {
8071       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
8072
8073       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
8074          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
8075          Builder bld(ctx->program, ctx->block);
8076
8077          /* to avoid unnecessary moves, we split and recombine sampler and image */
8078          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
8079                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8080          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8081          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
8082                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
8083                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
8084          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
8085                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
8086
8087          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
8088          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
8089                                img[0], img[1], img[2], img[3],
8090                                img[4], img[5], img[6], img[7]);
8091          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
8092                                 samp[0], samp[1], samp[2], samp[3]);
8093       }
8094    }
8095    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
8096                      instr->op == nir_texop_samples_identical))
8097       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8098 }
8099
8100 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
8101                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
8102 {
8103    Builder bld(ctx->program, ctx->block);
8104
8105    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
8106    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
8107    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
8108
8109    Operand neg_one(0xbf800000u);
8110    Operand one(0x3f800000u);
8111    Operand two(0x40000000u);
8112    Operand four(0x40800000u);
8113
8114    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
8115    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
8116    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
8117
8118    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
8119    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
8120    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
8121    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
8122
8123    // select sc
8124    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
8125    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
8126                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
8127                        one, is_ma_y);
8128    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8129
8130    // select tc
8131    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
8132    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
8133    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8134
8135    // select ma
8136    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8137                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
8138                   deriv_z, is_ma_z);
8139    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
8140    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
8141 }
8142
8143 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
8144 {
8145    Builder bld(ctx->program, ctx->block);
8146    Temp ma, tc, sc, id;
8147
8148    if (is_array) {
8149       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
8150
8151       // see comment in ac_prepare_cube_coords()
8152       if (ctx->options->chip_class <= GFX8)
8153          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
8154    }
8155
8156    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8157
8158    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
8159    vop3a->operands[0] = Operand(ma);
8160    vop3a->abs[0] = true;
8161    Temp invma = bld.tmp(v1);
8162    vop3a->definitions[0] = Definition(invma);
8163    ctx->block->instructions.emplace_back(std::move(vop3a));
8164
8165    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8166    if (!is_deriv)
8167       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
8168
8169    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8170    if (!is_deriv)
8171       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
8172
8173    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8174
8175    if (is_deriv) {
8176       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
8177       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
8178
8179       for (unsigned i = 0; i < 2; i++) {
8180          // see comment in ac_prepare_cube_coords()
8181          Temp deriv_ma;
8182          Temp deriv_sc, deriv_tc;
8183          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
8184                            &deriv_ma, &deriv_sc, &deriv_tc);
8185
8186          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
8187
8188          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8189                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
8190                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
8191          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8192                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
8193                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
8194          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
8195       }
8196
8197       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
8198       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
8199    }
8200
8201    if (is_array)
8202       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
8203    coords.resize(3);
8204    coords[0] = sc;
8205    coords[1] = tc;
8206    coords[2] = id;
8207 }
8208
8209 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
8210 {
8211    if (vec->parent_instr->type != nir_instr_type_alu)
8212       return;
8213    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
8214    if (vec_instr->op != nir_op_vec(vec->num_components))
8215       return;
8216
8217    for (unsigned i = 0; i < vec->num_components; i++) {
8218       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
8219               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
8220    }
8221 }
8222
8223 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
8224 {
8225    Builder bld(ctx->program, ctx->block);
8226    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
8227         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
8228    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
8229         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp();
8230    std::vector<Temp> coords;
8231    std::vector<Temp> derivs;
8232    nir_const_value *sample_index_cv = NULL;
8233    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
8234    enum glsl_base_type stype;
8235    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
8236
8237    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
8238                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
8239    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
8240                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
8241
8242    for (unsigned i = 0; i < instr->num_srcs; i++) {
8243       switch (instr->src[i].src_type) {
8244       case nir_tex_src_coord: {
8245          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
8246          for (unsigned i = 0; i < coord.size(); i++)
8247             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
8248          break;
8249       }
8250       case nir_tex_src_bias:
8251          if (instr->op == nir_texop_txb) {
8252             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
8253             has_bias = true;
8254          }
8255          break;
8256       case nir_tex_src_lod: {
8257          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
8258
8259          if (val && val->f32 <= 0.0) {
8260             level_zero = true;
8261          } else {
8262             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
8263             has_lod = true;
8264          }
8265          break;
8266       }
8267       case nir_tex_src_comparator:
8268          if (instr->is_shadow) {
8269             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
8270             has_compare = true;
8271          }
8272          break;
8273       case nir_tex_src_offset:
8274          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
8275          get_const_vec(instr->src[i].src.ssa, const_offset);
8276          has_offset = true;
8277          break;
8278       case nir_tex_src_ddx:
8279          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
8280          has_ddx = true;
8281          break;
8282       case nir_tex_src_ddy:
8283          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
8284          has_ddy = true;
8285          break;
8286       case nir_tex_src_ms_index:
8287          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
8288          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
8289          has_sample_index = true;
8290          break;
8291       case nir_tex_src_texture_offset:
8292       case nir_tex_src_sampler_offset:
8293       default:
8294          break;
8295       }
8296    }
8297
8298    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
8299       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
8300
8301    if (instr->op == nir_texop_texture_samples) {
8302       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
8303
8304       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
8305       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
8306       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
8307       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
8308
8309       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
8310                samples, Operand(1u), bld.scc(is_msaa));
8311       return;
8312    }
8313
8314    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
8315       aco_ptr<Instruction> tmp_instr;
8316       Temp acc, pack = Temp();
8317
8318       uint32_t pack_const = 0;
8319       for (unsigned i = 0; i < offset.size(); i++) {
8320          if (!const_offset[i])
8321             continue;
8322          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
8323       }
8324
8325       if (offset.type() == RegType::sgpr) {
8326          for (unsigned i = 0; i < offset.size(); i++) {
8327             if (const_offset[i])
8328                continue;
8329
8330             acc = emit_extract_vector(ctx, offset, i, s1);
8331             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
8332
8333             if (i) {
8334                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
8335             }
8336
8337             if (pack == Temp()) {
8338                pack = acc;
8339             } else {
8340                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
8341             }
8342          }
8343
8344          if (pack_const && pack != Temp())
8345             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
8346       } else {
8347          for (unsigned i = 0; i < offset.size(); i++) {
8348             if (const_offset[i])
8349                continue;
8350
8351             acc = emit_extract_vector(ctx, offset, i, v1);
8352             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
8353
8354             if (i) {
8355                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
8356             }
8357
8358             if (pack == Temp()) {
8359                pack = acc;
8360             } else {
8361                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
8362             }
8363          }
8364
8365          if (pack_const && pack != Temp())
8366             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
8367       }
8368       if (pack_const && pack == Temp())
8369          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
8370       else if (pack == Temp())
8371          has_offset = false;
8372       else
8373          offset = pack;
8374    }
8375
8376    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
8377       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
8378
8379    /* pack derivatives */
8380    if (has_ddx || has_ddy) {
8381       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
8382          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
8383          Temp zero = bld.copy(bld.def(v1), Operand(0u));
8384          derivs = {ddx, zero, ddy, zero};
8385       } else {
8386          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
8387             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
8388          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
8389             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
8390       }
8391       has_derivs = true;
8392    }
8393
8394    if (instr->coord_components > 1 &&
8395        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8396        instr->is_array &&
8397        instr->op != nir_texop_txf)
8398       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
8399
8400    if (instr->coord_components > 2 &&
8401       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
8402        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8403        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
8404        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8405        instr->is_array &&
8406        instr->op != nir_texop_txf &&
8407        instr->op != nir_texop_txf_ms &&
8408        instr->op != nir_texop_fragment_fetch &&
8409        instr->op != nir_texop_fragment_mask_fetch)
8410       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
8411
8412    if (ctx->options->chip_class == GFX9 &&
8413        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8414        instr->op != nir_texop_lod && instr->coord_components) {
8415       assert(coords.size() > 0 && coords.size() < 3);
8416
8417       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
8418                                                                      Operand((uint32_t) 0) :
8419                                                                      Operand((uint32_t) 0x3f000000)));
8420    }
8421
8422    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
8423
8424    if (instr->op == nir_texop_samples_identical)
8425       resource = fmask_ptr;
8426
8427    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8428              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8429             instr->op != nir_texop_txs &&
8430             instr->op != nir_texop_fragment_fetch &&
8431             instr->op != nir_texop_fragment_mask_fetch) {
8432       assert(has_sample_index);
8433       Operand op(sample_index);
8434       if (sample_index_cv)
8435          op = Operand(sample_index_cv->u32);
8436       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
8437    }
8438
8439    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
8440       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
8441          Temp off = emit_extract_vector(ctx, offset, i, v1);
8442          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
8443       }
8444       has_offset = false;
8445    }
8446
8447    /* Build tex instruction */
8448    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
8449    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
8450                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
8451                   : 0;
8452    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8453    Temp tmp_dst = dst;
8454
8455    /* gather4 selects the component by dmask and always returns vec4 */
8456    if (instr->op == nir_texop_tg4) {
8457       assert(instr->dest.ssa.num_components == 4);
8458       if (instr->is_shadow)
8459          dmask = 1;
8460       else
8461          dmask = 1 << instr->component;
8462       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
8463          tmp_dst = bld.tmp(v4);
8464    } else if (instr->op == nir_texop_samples_identical) {
8465       tmp_dst = bld.tmp(v1);
8466    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
8467       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
8468    }
8469
8470    aco_ptr<MIMG_instruction> tex;
8471    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
8472       if (!has_lod)
8473          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8474
8475       bool div_by_6 = instr->op == nir_texop_txs &&
8476                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
8477                       instr->is_array &&
8478                       (dmask & (1 << 2));
8479       if (tmp_dst.id() == dst.id() && div_by_6)
8480          tmp_dst = bld.tmp(tmp_dst.regClass());
8481
8482       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8483       tex->operands[0] = Operand(resource);
8484       tex->operands[1] = Operand(s4); /* no sampler */
8485       tex->operands[2] = Operand(as_vgpr(ctx,lod));
8486       if (ctx->options->chip_class == GFX9 &&
8487           instr->op == nir_texop_txs &&
8488           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8489           instr->is_array) {
8490          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
8491       } else if (instr->op == nir_texop_query_levels) {
8492          tex->dmask = 1 << 3;
8493       } else {
8494          tex->dmask = dmask;
8495       }
8496       tex->da = da;
8497       tex->definitions[0] = Definition(tmp_dst);
8498       tex->dim = dim;
8499       tex->can_reorder = true;
8500       ctx->block->instructions.emplace_back(std::move(tex));
8501
8502       if (div_by_6) {
8503          /* divide 3rd value by 6 by multiplying with magic number */
8504          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8505          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
8506          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
8507          assert(instr->dest.ssa.num_components == 3);
8508          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
8509          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8510                               emit_extract_vector(ctx, tmp_dst, 0, v1),
8511                               emit_extract_vector(ctx, tmp_dst, 1, v1),
8512                               by_6);
8513
8514       }
8515
8516       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8517       return;
8518    }
8519
8520    Temp tg4_compare_cube_wa64 = Temp();
8521
8522    if (tg4_integer_workarounds) {
8523       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8524       tex->operands[0] = Operand(resource);
8525       tex->operands[1] = Operand(s4); /* no sampler */
8526       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8527       tex->dim = dim;
8528       tex->dmask = 0x3;
8529       tex->da = da;
8530       Temp size = bld.tmp(v2);
8531       tex->definitions[0] = Definition(size);
8532       tex->can_reorder = true;
8533       ctx->block->instructions.emplace_back(std::move(tex));
8534       emit_split_vector(ctx, size, size.size());
8535
8536       Temp half_texel[2];
8537       for (unsigned i = 0; i < 2; i++) {
8538          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
8539          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
8540          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
8541          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
8542       }
8543
8544       Temp new_coords[2] = {
8545          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
8546          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
8547       };
8548
8549       if (tg4_integer_cube_workaround) {
8550          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
8551          Temp desc[resource.size()];
8552          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
8553                                                                            Format::PSEUDO, 1, resource.size())};
8554          split->operands[0] = Operand(resource);
8555          for (unsigned i = 0; i < resource.size(); i++) {
8556             desc[i] = bld.tmp(s1);
8557             split->definitions[i] = Definition(desc[i]);
8558          }
8559          ctx->block->instructions.emplace_back(std::move(split));
8560
8561          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
8562          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
8563                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
8564
8565          Temp nfmt;
8566          if (stype == GLSL_TYPE_UINT) {
8567             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8568                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
8569                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
8570                             bld.scc(compare_cube_wa));
8571          } else {
8572             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8573                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
8574                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
8575                             bld.scc(compare_cube_wa));
8576          }
8577          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
8578          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
8579
8580          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
8581
8582          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
8583                             Operand((uint32_t)C_008F14_NUM_FORMAT));
8584          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
8585
8586          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
8587                                                                          Format::PSEUDO, resource.size(), 1)};
8588          for (unsigned i = 0; i < resource.size(); i++)
8589             vec->operands[i] = Operand(desc[i]);
8590          resource = bld.tmp(resource.regClass());
8591          vec->definitions[0] = Definition(resource);
8592          ctx->block->instructions.emplace_back(std::move(vec));
8593
8594          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8595                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
8596          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8597                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
8598       }
8599       coords[0] = new_coords[0];
8600       coords[1] = new_coords[1];
8601    }
8602
8603    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
8604       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
8605
8606       assert(coords.size() == 1);
8607       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
8608       aco_opcode op;
8609       switch (last_bit) {
8610       case 1:
8611          op = aco_opcode::buffer_load_format_x; break;
8612       case 2:
8613          op = aco_opcode::buffer_load_format_xy; break;
8614       case 3:
8615          op = aco_opcode::buffer_load_format_xyz; break;
8616       case 4:
8617          op = aco_opcode::buffer_load_format_xyzw; break;
8618       default:
8619          unreachable("Tex instruction loads more than 4 components.");
8620       }
8621
8622       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
8623       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
8624          tmp_dst = dst;
8625       else
8626          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
8627
8628       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
8629       mubuf->operands[0] = Operand(resource);
8630       mubuf->operands[1] = Operand(coords[0]);
8631       mubuf->operands[2] = Operand((uint32_t) 0);
8632       mubuf->definitions[0] = Definition(tmp_dst);
8633       mubuf->idxen = true;
8634       mubuf->can_reorder = true;
8635       ctx->block->instructions.emplace_back(std::move(mubuf));
8636
8637       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
8638       return;
8639    }
8640
8641    /* gather MIMG address components */
8642    std::vector<Temp> args;
8643    if (has_offset)
8644       args.emplace_back(offset);
8645    if (has_bias)
8646       args.emplace_back(bias);
8647    if (has_compare)
8648       args.emplace_back(compare);
8649    if (has_derivs)
8650       args.insert(args.end(), derivs.begin(), derivs.end());
8651
8652    args.insert(args.end(), coords.begin(), coords.end());
8653    if (has_sample_index)
8654       args.emplace_back(sample_index);
8655    if (has_lod)
8656       args.emplace_back(lod);
8657
8658    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
8659    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
8660    vec->definitions[0] = Definition(arg);
8661    for (unsigned i = 0; i < args.size(); i++)
8662       vec->operands[i] = Operand(args[i]);
8663    ctx->block->instructions.emplace_back(std::move(vec));
8664
8665
8666    if (instr->op == nir_texop_txf ||
8667        instr->op == nir_texop_txf_ms ||
8668        instr->op == nir_texop_samples_identical ||
8669        instr->op == nir_texop_fragment_fetch ||
8670        instr->op == nir_texop_fragment_mask_fetch) {
8671       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
8672       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
8673       tex->operands[0] = Operand(resource);
8674       tex->operands[1] = Operand(s4); /* no sampler */
8675       tex->operands[2] = Operand(arg);
8676       tex->dim = dim;
8677       tex->dmask = dmask;
8678       tex->unrm = true;
8679       tex->da = da;
8680       tex->definitions[0] = Definition(tmp_dst);
8681       tex->can_reorder = true;
8682       ctx->block->instructions.emplace_back(std::move(tex));
8683
8684       if (instr->op == nir_texop_samples_identical) {
8685          assert(dmask == 1 && dst.regClass() == v1);
8686          assert(dst.id() != tmp_dst.id());
8687
8688          Temp tmp = bld.tmp(bld.lm);
8689          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
8690          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
8691
8692       } else {
8693          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8694       }
8695       return;
8696    }
8697
8698    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
8699    aco_opcode opcode = aco_opcode::image_sample;
8700    if (has_offset) { /* image_sample_*_o */
8701       if (has_compare) {
8702          opcode = aco_opcode::image_sample_c_o;
8703          if (has_derivs)
8704             opcode = aco_opcode::image_sample_c_d_o;
8705          if (has_bias)
8706             opcode = aco_opcode::image_sample_c_b_o;
8707          if (level_zero)
8708             opcode = aco_opcode::image_sample_c_lz_o;
8709          if (has_lod)
8710             opcode = aco_opcode::image_sample_c_l_o;
8711       } else {
8712          opcode = aco_opcode::image_sample_o;
8713          if (has_derivs)
8714             opcode = aco_opcode::image_sample_d_o;
8715          if (has_bias)
8716             opcode = aco_opcode::image_sample_b_o;
8717          if (level_zero)
8718             opcode = aco_opcode::image_sample_lz_o;
8719          if (has_lod)
8720             opcode = aco_opcode::image_sample_l_o;
8721       }
8722    } else { /* no offset */
8723       if (has_compare) {
8724          opcode = aco_opcode::image_sample_c;
8725          if (has_derivs)
8726             opcode = aco_opcode::image_sample_c_d;
8727          if (has_bias)
8728             opcode = aco_opcode::image_sample_c_b;
8729          if (level_zero)
8730             opcode = aco_opcode::image_sample_c_lz;
8731          if (has_lod)
8732             opcode = aco_opcode::image_sample_c_l;
8733       } else {
8734          opcode = aco_opcode::image_sample;
8735          if (has_derivs)
8736             opcode = aco_opcode::image_sample_d;
8737          if (has_bias)
8738             opcode = aco_opcode::image_sample_b;
8739          if (level_zero)
8740             opcode = aco_opcode::image_sample_lz;
8741          if (has_lod)
8742             opcode = aco_opcode::image_sample_l;
8743       }
8744    }
8745
8746    if (instr->op == nir_texop_tg4) {
8747       if (has_offset) {
8748          opcode = aco_opcode::image_gather4_lz_o;
8749          if (has_compare)
8750             opcode = aco_opcode::image_gather4_c_lz_o;
8751       } else {
8752          opcode = aco_opcode::image_gather4_lz;
8753          if (has_compare)
8754             opcode = aco_opcode::image_gather4_c_lz;
8755       }
8756    } else if (instr->op == nir_texop_lod) {
8757       opcode = aco_opcode::image_get_lod;
8758    }
8759
8760    /* we don't need the bias, sample index, compare value or offset to be
8761     * computed in WQM but if the p_create_vector copies the coordinates, then it
8762     * needs to be in WQM */
8763    if (ctx->stage == fragment_fs &&
8764        !has_derivs && !has_lod && !level_zero &&
8765        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
8766        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
8767       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
8768
8769    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
8770    tex->operands[0] = Operand(resource);
8771    tex->operands[1] = Operand(sampler);
8772    tex->operands[2] = Operand(arg);
8773    tex->dim = dim;
8774    tex->dmask = dmask;
8775    tex->da = da;
8776    tex->definitions[0] = Definition(tmp_dst);
8777    tex->can_reorder = true;
8778    ctx->block->instructions.emplace_back(std::move(tex));
8779
8780    if (tg4_integer_cube_workaround) {
8781       assert(tmp_dst.id() != dst.id());
8782       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
8783
8784       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8785       Temp val[4];
8786       for (unsigned i = 0; i < dst.size(); i++) {
8787          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
8788          Temp cvt_val;
8789          if (stype == GLSL_TYPE_UINT)
8790             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
8791          else
8792             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
8793          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
8794       }
8795       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
8796       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8797                            val[0], val[1], val[2], val[3]);
8798    }
8799    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
8800    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
8801
8802 }
8803
8804
8805 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
8806 {
8807    Temp tmp = get_ssa_temp(ctx, ssa);
8808    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
8809       return Operand(tmp.regClass());
8810    else
8811       return Operand(tmp);
8812 }
8813
8814 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
8815 {
8816    aco_ptr<Pseudo_instruction> phi;
8817    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8818    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
8819
8820    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
8821    logical |= ctx->block->kind & block_kind_merge;
8822    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
8823
8824    /* we want a sorted list of sources, since the predecessor list is also sorted */
8825    std::map<unsigned, nir_ssa_def*> phi_src;
8826    nir_foreach_phi_src(src, instr)
8827       phi_src[src->pred->index] = src->src.ssa;
8828
8829    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
8830    unsigned num_operands = 0;
8831    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size()) + 1];
8832    unsigned num_defined = 0;
8833    unsigned cur_pred_idx = 0;
8834    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
8835       if (cur_pred_idx < preds.size()) {
8836          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
8837          unsigned block = ctx->cf_info.nir_to_aco[src.first];
8838          unsigned skipped = 0;
8839          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
8840             skipped++;
8841          if (cur_pred_idx + skipped < preds.size()) {
8842             for (unsigned i = 0; i < skipped; i++)
8843                operands[num_operands++] = Operand(dst.regClass());
8844             cur_pred_idx += skipped;
8845          } else {
8846             continue;
8847          }
8848       }
8849       /* Handle missing predecessors at the end. This shouldn't happen with loop
8850        * headers and we can't ignore these sources for loop header phis. */
8851       if (!(ctx->block->kind & block_kind_loop_header) && cur_pred_idx >= preds.size())
8852          continue;
8853       cur_pred_idx++;
8854       Operand op = get_phi_operand(ctx, src.second);
8855       operands[num_operands++] = op;
8856       num_defined += !op.isUndefined();
8857    }
8858    /* handle block_kind_continue_or_break at loop exit blocks */
8859    while (cur_pred_idx++ < preds.size())
8860       operands[num_operands++] = Operand(dst.regClass());
8861
8862    /* If the loop ends with a break, still add a linear continue edge in case
8863     * that break is divergent or continue_or_break is used. We'll either remove
8864     * this operand later in visit_loop() if it's not necessary or replace the
8865     * undef with something correct. */
8866    if (!logical && ctx->block->kind & block_kind_loop_header) {
8867       nir_loop *loop = nir_cf_node_as_loop(instr->instr.block->cf_node.parent);
8868       nir_block *last = nir_loop_last_block(loop);
8869       if (last->successors[0] != instr->instr.block)
8870          operands[num_operands++] = Operand(RegClass());
8871    }
8872
8873    if (num_defined == 0) {
8874       Builder bld(ctx->program, ctx->block);
8875       if (dst.regClass() == s1) {
8876          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
8877       } else if (dst.regClass() == v1) {
8878          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
8879       } else {
8880          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8881          for (unsigned i = 0; i < dst.size(); i++)
8882             vec->operands[i] = Operand(0u);
8883          vec->definitions[0] = Definition(dst);
8884          ctx->block->instructions.emplace_back(std::move(vec));
8885       }
8886       return;
8887    }
8888
8889    /* we can use a linear phi in some cases if one src is undef */
8890    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
8891       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
8892
8893       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
8894       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
8895       assert(invert->kind & block_kind_invert);
8896
8897       unsigned then_block = invert->linear_preds[0];
8898
8899       Block* insert_block = NULL;
8900       for (unsigned i = 0; i < num_operands; i++) {
8901          Operand op = operands[i];
8902          if (op.isUndefined())
8903             continue;
8904          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
8905          phi->operands[0] = op;
8906          break;
8907       }
8908       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
8909       phi->operands[1] = Operand(dst.regClass());
8910       phi->definitions[0] = Definition(dst);
8911       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
8912       return;
8913    }
8914
8915    /* try to scalarize vector phis */
8916    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
8917       // TODO: scalarize linear phis on divergent ifs
8918       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
8919       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
8920       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
8921          Operand src = operands[i];
8922          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
8923             can_scalarize = false;
8924       }
8925       if (can_scalarize) {
8926          unsigned num_components = instr->dest.ssa.num_components;
8927          assert(dst.size() % num_components == 0);
8928          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
8929
8930          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
8931          for (unsigned k = 0; k < num_components; k++) {
8932             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8933             for (unsigned i = 0; i < num_operands; i++) {
8934                Operand src = operands[i];
8935                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
8936             }
8937             Temp phi_dst = {ctx->program->allocateId(), rc};
8938             phi->definitions[0] = Definition(phi_dst);
8939             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8940             new_vec[k] = phi_dst;
8941             vec->operands[k] = Operand(phi_dst);
8942          }
8943          vec->definitions[0] = Definition(dst);
8944          ctx->block->instructions.emplace_back(std::move(vec));
8945          ctx->allocated_vec.emplace(dst.id(), new_vec);
8946          return;
8947       }
8948    }
8949
8950    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8951    for (unsigned i = 0; i < num_operands; i++)
8952       phi->operands[i] = operands[i];
8953    phi->definitions[0] = Definition(dst);
8954    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8955 }
8956
8957
8958 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
8959 {
8960    Temp dst = get_ssa_temp(ctx, &instr->def);
8961
8962    assert(dst.type() == RegType::sgpr);
8963
8964    if (dst.size() == 1) {
8965       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
8966    } else {
8967       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8968       for (unsigned i = 0; i < dst.size(); i++)
8969          vec->operands[i] = Operand(0u);
8970       vec->definitions[0] = Definition(dst);
8971       ctx->block->instructions.emplace_back(std::move(vec));
8972    }
8973 }
8974
8975 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
8976 {
8977    Builder bld(ctx->program, ctx->block);
8978    Block *logical_target;
8979    append_logical_end(ctx->block);
8980    unsigned idx = ctx->block->index;
8981
8982    switch (instr->type) {
8983    case nir_jump_break:
8984       logical_target = ctx->cf_info.parent_loop.exit;
8985       add_logical_edge(idx, logical_target);
8986       ctx->block->kind |= block_kind_break;
8987
8988       if (!ctx->cf_info.parent_if.is_divergent &&
8989           !ctx->cf_info.parent_loop.has_divergent_continue) {
8990          /* uniform break - directly jump out of the loop */
8991          ctx->block->kind |= block_kind_uniform;
8992          ctx->cf_info.has_branch = true;
8993          bld.branch(aco_opcode::p_branch);
8994          add_linear_edge(idx, logical_target);
8995          return;
8996       }
8997       ctx->cf_info.parent_loop.has_divergent_branch = true;
8998       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8999       break;
9000    case nir_jump_continue:
9001       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
9002       add_logical_edge(idx, logical_target);
9003       ctx->block->kind |= block_kind_continue;
9004
9005       if (ctx->cf_info.parent_if.is_divergent) {
9006          /* for potential uniform breaks after this continue,
9007             we must ensure that they are handled correctly */
9008          ctx->cf_info.parent_loop.has_divergent_continue = true;
9009          ctx->cf_info.parent_loop.has_divergent_branch = true;
9010          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
9011       } else {
9012          /* uniform continue - directly jump to the loop header */
9013          ctx->block->kind |= block_kind_uniform;
9014          ctx->cf_info.has_branch = true;
9015          bld.branch(aco_opcode::p_branch);
9016          add_linear_edge(idx, logical_target);
9017          return;
9018       }
9019       break;
9020    default:
9021       fprintf(stderr, "Unknown NIR jump instr: ");
9022       nir_print_instr(&instr->instr, stderr);
9023       fprintf(stderr, "\n");
9024       abort();
9025    }
9026
9027    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
9028       ctx->cf_info.exec_potentially_empty_break = true;
9029       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
9030    }
9031
9032    /* remove critical edges from linear CFG */
9033    bld.branch(aco_opcode::p_branch);
9034    Block* break_block = ctx->program->create_and_insert_block();
9035    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9036    break_block->kind |= block_kind_uniform;
9037    add_linear_edge(idx, break_block);
9038    /* the loop_header pointer might be invalidated by this point */
9039    if (instr->type == nir_jump_continue)
9040       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
9041    add_linear_edge(break_block->index, logical_target);
9042    bld.reset(break_block);
9043    bld.branch(aco_opcode::p_branch);
9044
9045    Block* continue_block = ctx->program->create_and_insert_block();
9046    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9047    add_linear_edge(idx, continue_block);
9048    append_logical_start(continue_block);
9049    ctx->block = continue_block;
9050    return;
9051 }
9052
9053 void visit_block(isel_context *ctx, nir_block *block)
9054 {
9055    nir_foreach_instr(instr, block) {
9056       switch (instr->type) {
9057       case nir_instr_type_alu:
9058          visit_alu_instr(ctx, nir_instr_as_alu(instr));
9059          break;
9060       case nir_instr_type_load_const:
9061          visit_load_const(ctx, nir_instr_as_load_const(instr));
9062          break;
9063       case nir_instr_type_intrinsic:
9064          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
9065          break;
9066       case nir_instr_type_tex:
9067          visit_tex(ctx, nir_instr_as_tex(instr));
9068          break;
9069       case nir_instr_type_phi:
9070          visit_phi(ctx, nir_instr_as_phi(instr));
9071          break;
9072       case nir_instr_type_ssa_undef:
9073          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
9074          break;
9075       case nir_instr_type_deref:
9076          break;
9077       case nir_instr_type_jump:
9078          visit_jump(ctx, nir_instr_as_jump(instr));
9079          break;
9080       default:
9081          fprintf(stderr, "Unknown NIR instr type: ");
9082          nir_print_instr(instr, stderr);
9083          fprintf(stderr, "\n");
9084          //abort();
9085       }
9086    }
9087
9088    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9089       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
9090 }
9091
9092
9093
9094 static Operand create_continue_phis(isel_context *ctx, unsigned first, unsigned last,
9095                                     aco_ptr<Instruction>& header_phi, Operand *vals)
9096 {
9097    vals[0] = Operand(header_phi->definitions[0].getTemp());
9098    RegClass rc = vals[0].regClass();
9099
9100    unsigned loop_nest_depth = ctx->program->blocks[first].loop_nest_depth;
9101
9102    unsigned next_pred = 1;
9103
9104    for (unsigned idx = first + 1; idx <= last; idx++) {
9105       Block& block = ctx->program->blocks[idx];
9106       if (block.loop_nest_depth != loop_nest_depth) {
9107          vals[idx - first] = vals[idx - 1 - first];
9108          continue;
9109       }
9110
9111       if (block.kind & block_kind_continue) {
9112          vals[idx - first] = header_phi->operands[next_pred];
9113          next_pred++;
9114          continue;
9115       }
9116
9117       bool all_same = true;
9118       for (unsigned i = 1; all_same && (i < block.linear_preds.size()); i++)
9119          all_same = vals[block.linear_preds[i] - first] == vals[block.linear_preds[0] - first];
9120
9121       Operand val;
9122       if (all_same) {
9123          val = vals[block.linear_preds[0] - first];
9124       } else {
9125          aco_ptr<Instruction> phi(create_instruction<Pseudo_instruction>(
9126             aco_opcode::p_linear_phi, Format::PSEUDO, block.linear_preds.size(), 1));
9127          for (unsigned i = 0; i < block.linear_preds.size(); i++)
9128             phi->operands[i] = vals[block.linear_preds[i] - first];
9129          val = Operand(Temp(ctx->program->allocateId(), rc));
9130          phi->definitions[0] = Definition(val.getTemp());
9131          block.instructions.emplace(block.instructions.begin(), std::move(phi));
9132       }
9133       vals[idx - first] = val;
9134    }
9135
9136    return vals[last - first];
9137 }
9138
9139 static void visit_loop(isel_context *ctx, nir_loop *loop)
9140 {
9141    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
9142    append_logical_end(ctx->block);
9143    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
9144    Builder bld(ctx->program, ctx->block);
9145    bld.branch(aco_opcode::p_branch);
9146    unsigned loop_preheader_idx = ctx->block->index;
9147
9148    Block loop_exit = Block();
9149    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9150    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
9151
9152    Block* loop_header = ctx->program->create_and_insert_block();
9153    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
9154    loop_header->kind |= block_kind_loop_header;
9155    add_edge(loop_preheader_idx, loop_header);
9156    ctx->block = loop_header;
9157
9158    /* emit loop body */
9159    unsigned loop_header_idx = loop_header->index;
9160    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
9161    append_logical_start(ctx->block);
9162    bool unreachable = visit_cf_list(ctx, &loop->body);
9163
9164    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
9165    if (!ctx->cf_info.has_branch) {
9166       append_logical_end(ctx->block);
9167       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
9168          /* Discards can result in code running with an empty exec mask.
9169           * This would result in divergent breaks not ever being taken. As a
9170           * workaround, break the loop when the loop mask is empty instead of
9171           * always continuing. */
9172          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
9173          unsigned block_idx = ctx->block->index;
9174
9175          /* create helper blocks to avoid critical edges */
9176          Block *break_block = ctx->program->create_and_insert_block();
9177          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9178          break_block->kind = block_kind_uniform;
9179          bld.reset(break_block);
9180          bld.branch(aco_opcode::p_branch);
9181          add_linear_edge(block_idx, break_block);
9182          add_linear_edge(break_block->index, &loop_exit);
9183
9184          Block *continue_block = ctx->program->create_and_insert_block();
9185          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9186          continue_block->kind = block_kind_uniform;
9187          bld.reset(continue_block);
9188          bld.branch(aco_opcode::p_branch);
9189          add_linear_edge(block_idx, continue_block);
9190          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
9191
9192          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9193             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
9194          ctx->block = &ctx->program->blocks[block_idx];
9195       } else {
9196          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
9197          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9198             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9199          else
9200             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9201       }
9202
9203       bld.reset(ctx->block);
9204       bld.branch(aco_opcode::p_branch);
9205    }
9206
9207    /* Fixup phis in loop header from unreachable blocks.
9208     * has_branch/has_divergent_branch also indicates if the loop ends with a
9209     * break/continue instruction, but we don't emit those if unreachable=true */
9210    if (unreachable) {
9211       assert(ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch);
9212       bool linear = ctx->cf_info.has_branch;
9213       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
9214       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9215          if ((logical && instr->opcode == aco_opcode::p_phi) ||
9216              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
9217             /* the last operand should be the one that needs to be removed */
9218             instr->operands.pop_back();
9219          } else if (!is_phi(instr)) {
9220             break;
9221          }
9222       }
9223    }
9224
9225    /* Fixup linear phis in loop header from expecting a continue. Both this fixup
9226     * and the previous one shouldn't both happen at once because a break in the
9227     * merge block would get CSE'd */
9228    if (nir_loop_last_block(loop)->successors[0] != nir_loop_first_block(loop)) {
9229       unsigned num_vals = ctx->cf_info.has_branch ? 1 : (ctx->block->index - loop_header_idx + 1);
9230       Operand vals[num_vals];
9231       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9232          if (instr->opcode == aco_opcode::p_linear_phi) {
9233             if (ctx->cf_info.has_branch)
9234                instr->operands.pop_back();
9235             else
9236                instr->operands.back() = create_continue_phis(ctx, loop_header_idx, ctx->block->index, instr, vals);
9237          } else if (!is_phi(instr)) {
9238             break;
9239          }
9240       }
9241    }
9242
9243    ctx->cf_info.has_branch = false;
9244
9245    // TODO: if the loop has not a single exit, we must add one °°
9246    /* emit loop successor block */
9247    ctx->block = ctx->program->insert_block(std::move(loop_exit));
9248    append_logical_start(ctx->block);
9249
9250    #if 0
9251    // TODO: check if it is beneficial to not branch on continues
9252    /* trim linear phis in loop header */
9253    for (auto&& instr : loop_entry->instructions) {
9254       if (instr->opcode == aco_opcode::p_linear_phi) {
9255          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
9256          new_phi->definitions[0] = instr->definitions[0];
9257          for (unsigned i = 0; i < new_phi->operands.size(); i++)
9258             new_phi->operands[i] = instr->operands[i];
9259          /* check that the remaining operands are all the same */
9260          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
9261             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
9262          instr.swap(new_phi);
9263       } else if (instr->opcode == aco_opcode::p_phi) {
9264          continue;
9265       } else {
9266          break;
9267       }
9268    }
9269    #endif
9270 }
9271
9272 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
9273 {
9274    ic->cond = cond;
9275
9276    append_logical_end(ctx->block);
9277    ctx->block->kind |= block_kind_branch;
9278
9279    /* branch to linear then block */
9280    assert(cond.regClass() == ctx->program->lane_mask);
9281    aco_ptr<Pseudo_branch_instruction> branch;
9282    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
9283    branch->operands[0] = Operand(cond);
9284    ctx->block->instructions.push_back(std::move(branch));
9285
9286    ic->BB_if_idx = ctx->block->index;
9287    ic->BB_invert = Block();
9288    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9289    /* Invert blocks are intentionally not marked as top level because they
9290     * are not part of the logical cfg. */
9291    ic->BB_invert.kind |= block_kind_invert;
9292    ic->BB_endif = Block();
9293    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9294    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
9295
9296    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
9297    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
9298    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
9299    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
9300    ctx->cf_info.parent_if.is_divergent = true;
9301
9302    /* divergent branches use cbranch_execz */
9303    ctx->cf_info.exec_potentially_empty_discard = false;
9304    ctx->cf_info.exec_potentially_empty_break = false;
9305    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9306
9307    /** emit logical then block */
9308    Block* BB_then_logical = ctx->program->create_and_insert_block();
9309    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9310    add_edge(ic->BB_if_idx, BB_then_logical);
9311    ctx->block = BB_then_logical;
9312    append_logical_start(BB_then_logical);
9313 }
9314
9315 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
9316 {
9317    Block *BB_then_logical = ctx->block;
9318    append_logical_end(BB_then_logical);
9319     /* branch from logical then block to invert block */
9320    aco_ptr<Pseudo_branch_instruction> branch;
9321    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9322    BB_then_logical->instructions.emplace_back(std::move(branch));
9323    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
9324    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9325       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
9326    BB_then_logical->kind |= block_kind_uniform;
9327    assert(!ctx->cf_info.has_branch);
9328    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9329    ctx->cf_info.parent_loop.has_divergent_branch = false;
9330
9331    /** emit linear then block */
9332    Block* BB_then_linear = ctx->program->create_and_insert_block();
9333    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9334    BB_then_linear->kind |= block_kind_uniform;
9335    add_linear_edge(ic->BB_if_idx, BB_then_linear);
9336    /* branch from linear then block to invert block */
9337    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9338    BB_then_linear->instructions.emplace_back(std::move(branch));
9339    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
9340
9341    /** emit invert merge block */
9342    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
9343    ic->invert_idx = ctx->block->index;
9344
9345    /* branch to linear else block (skip else) */
9346    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
9347    branch->operands[0] = Operand(ic->cond);
9348    ctx->block->instructions.push_back(std::move(branch));
9349
9350    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
9351    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
9352    ic->exec_potentially_empty_break_depth_old =
9353       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9354    /* divergent branches use cbranch_execz */
9355    ctx->cf_info.exec_potentially_empty_discard = false;
9356    ctx->cf_info.exec_potentially_empty_break = false;
9357    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9358
9359    /** emit logical else block */
9360    Block* BB_else_logical = ctx->program->create_and_insert_block();
9361    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9362    add_logical_edge(ic->BB_if_idx, BB_else_logical);
9363    add_linear_edge(ic->invert_idx, BB_else_logical);
9364    ctx->block = BB_else_logical;
9365    append_logical_start(BB_else_logical);
9366 }
9367
9368 static void end_divergent_if(isel_context *ctx, if_context *ic)
9369 {
9370    Block *BB_else_logical = ctx->block;
9371    append_logical_end(BB_else_logical);
9372
9373    /* branch from logical else block to endif block */
9374    aco_ptr<Pseudo_branch_instruction> branch;
9375    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9376    BB_else_logical->instructions.emplace_back(std::move(branch));
9377    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
9378    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9379       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
9380    BB_else_logical->kind |= block_kind_uniform;
9381
9382    assert(!ctx->cf_info.has_branch);
9383    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9384
9385
9386    /** emit linear else block */
9387    Block* BB_else_linear = ctx->program->create_and_insert_block();
9388    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9389    BB_else_linear->kind |= block_kind_uniform;
9390    add_linear_edge(ic->invert_idx, BB_else_linear);
9391
9392    /* branch from linear else block to endif block */
9393    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9394    BB_else_linear->instructions.emplace_back(std::move(branch));
9395    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
9396
9397
9398    /** emit endif merge block */
9399    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9400    append_logical_start(ctx->block);
9401
9402
9403    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
9404    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
9405    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
9406    ctx->cf_info.exec_potentially_empty_break_depth =
9407       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9408    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
9409        !ctx->cf_info.parent_if.is_divergent) {
9410       ctx->cf_info.exec_potentially_empty_break = false;
9411       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9412    }
9413    /* uniform control flow never has an empty exec-mask */
9414    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
9415       ctx->cf_info.exec_potentially_empty_discard = false;
9416       ctx->cf_info.exec_potentially_empty_break = false;
9417       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9418    }
9419 }
9420
9421 static void begin_uniform_if_then(isel_context *ctx, if_context *ic, Temp cond)
9422 {
9423    assert(cond.regClass() == s1);
9424
9425    append_logical_end(ctx->block);
9426    ctx->block->kind |= block_kind_uniform;
9427
9428    aco_ptr<Pseudo_branch_instruction> branch;
9429    aco_opcode branch_opcode = aco_opcode::p_cbranch_z;
9430    branch.reset(create_instruction<Pseudo_branch_instruction>(branch_opcode, Format::PSEUDO_BRANCH, 1, 0));
9431    branch->operands[0] = Operand(cond);
9432    branch->operands[0].setFixed(scc);
9433    ctx->block->instructions.emplace_back(std::move(branch));
9434
9435    ic->BB_if_idx = ctx->block->index;
9436    ic->BB_endif = Block();
9437    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9438    ic->BB_endif.kind |= ctx->block->kind & block_kind_top_level;
9439
9440    ctx->cf_info.has_branch = false;
9441    ctx->cf_info.parent_loop.has_divergent_branch = false;
9442
9443    /** emit then block */
9444    Block* BB_then = ctx->program->create_and_insert_block();
9445    BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9446    add_edge(ic->BB_if_idx, BB_then);
9447    append_logical_start(BB_then);
9448    ctx->block = BB_then;
9449 }
9450
9451 static void begin_uniform_if_else(isel_context *ctx, if_context *ic)
9452 {
9453    Block *BB_then = ctx->block;
9454
9455    ic->uniform_has_then_branch = ctx->cf_info.has_branch;
9456    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9457
9458    if (!ic->uniform_has_then_branch) {
9459       append_logical_end(BB_then);
9460       /* branch from then block to endif block */
9461       aco_ptr<Pseudo_branch_instruction> branch;
9462       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9463       BB_then->instructions.emplace_back(std::move(branch));
9464       add_linear_edge(BB_then->index, &ic->BB_endif);
9465       if (!ic->then_branch_divergent)
9466          add_logical_edge(BB_then->index, &ic->BB_endif);
9467       BB_then->kind |= block_kind_uniform;
9468    }
9469
9470    ctx->cf_info.has_branch = false;
9471    ctx->cf_info.parent_loop.has_divergent_branch = false;
9472
9473    /** emit else block */
9474    Block* BB_else = ctx->program->create_and_insert_block();
9475    BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9476    add_edge(ic->BB_if_idx, BB_else);
9477    append_logical_start(BB_else);
9478    ctx->block = BB_else;
9479 }
9480
9481 static void end_uniform_if(isel_context *ctx, if_context *ic)
9482 {
9483    Block *BB_else = ctx->block;
9484
9485    if (!ctx->cf_info.has_branch) {
9486       append_logical_end(BB_else);
9487       /* branch from then block to endif block */
9488       aco_ptr<Pseudo_branch_instruction> branch;
9489       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9490       BB_else->instructions.emplace_back(std::move(branch));
9491       add_linear_edge(BB_else->index, &ic->BB_endif);
9492       if (!ctx->cf_info.parent_loop.has_divergent_branch)
9493          add_logical_edge(BB_else->index, &ic->BB_endif);
9494       BB_else->kind |= block_kind_uniform;
9495    }
9496
9497    ctx->cf_info.has_branch &= ic->uniform_has_then_branch;
9498    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9499
9500    /** emit endif merge block */
9501    if (!ctx->cf_info.has_branch) {
9502       ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9503       append_logical_start(ctx->block);
9504    }
9505 }
9506
9507 static bool visit_if(isel_context *ctx, nir_if *if_stmt)
9508 {
9509    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
9510    Builder bld(ctx->program, ctx->block);
9511    aco_ptr<Pseudo_branch_instruction> branch;
9512    if_context ic;
9513
9514    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
9515       /**
9516        * Uniform conditionals are represented in the following way*) :
9517        *
9518        * The linear and logical CFG:
9519        *                        BB_IF
9520        *                        /    \
9521        *       BB_THEN (logical)      BB_ELSE (logical)
9522        *                        \    /
9523        *                        BB_ENDIF
9524        *
9525        * *) Exceptions may be due to break and continue statements within loops
9526        *    If a break/continue happens within uniform control flow, it branches
9527        *    to the loop exit/entry block. Otherwise, it branches to the next
9528        *    merge block.
9529        **/
9530
9531       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
9532       assert(cond.regClass() == ctx->program->lane_mask);
9533       cond = bool_to_scalar_condition(ctx, cond);
9534
9535       begin_uniform_if_then(ctx, &ic, cond);
9536       visit_cf_list(ctx, &if_stmt->then_list);
9537
9538       begin_uniform_if_else(ctx, &ic);
9539       visit_cf_list(ctx, &if_stmt->else_list);
9540
9541       end_uniform_if(ctx, &ic);
9542
9543       return !ctx->cf_info.has_branch;
9544    } else { /* non-uniform condition */
9545       /**
9546        * To maintain a logical and linear CFG without critical edges,
9547        * non-uniform conditionals are represented in the following way*) :
9548        *
9549        * The linear CFG:
9550        *                        BB_IF
9551        *                        /    \
9552        *       BB_THEN (logical)      BB_THEN (linear)
9553        *                        \    /
9554        *                        BB_INVERT (linear)
9555        *                        /    \
9556        *       BB_ELSE (logical)      BB_ELSE (linear)
9557        *                        \    /
9558        *                        BB_ENDIF
9559        *
9560        * The logical CFG:
9561        *                        BB_IF
9562        *                        /    \
9563        *       BB_THEN (logical)      BB_ELSE (logical)
9564        *                        \    /
9565        *                        BB_ENDIF
9566        *
9567        * *) Exceptions may be due to break and continue statements within loops
9568        **/
9569
9570       begin_divergent_if_then(ctx, &ic, cond);
9571       visit_cf_list(ctx, &if_stmt->then_list);
9572
9573       begin_divergent_if_else(ctx, &ic);
9574       visit_cf_list(ctx, &if_stmt->else_list);
9575
9576       end_divergent_if(ctx, &ic);
9577
9578       return true;
9579    }
9580 }
9581
9582 static bool visit_cf_list(isel_context *ctx,
9583                           struct exec_list *list)
9584 {
9585    foreach_list_typed(nir_cf_node, node, node, list) {
9586       switch (node->type) {
9587       case nir_cf_node_block:
9588          visit_block(ctx, nir_cf_node_as_block(node));
9589          break;
9590       case nir_cf_node_if:
9591          if (!visit_if(ctx, nir_cf_node_as_if(node)))
9592             return true;
9593          break;
9594       case nir_cf_node_loop:
9595          visit_loop(ctx, nir_cf_node_as_loop(node));
9596          break;
9597       default:
9598          unreachable("unimplemented cf list type");
9599       }
9600    }
9601    return false;
9602 }
9603
9604 static void create_null_export(isel_context *ctx)
9605 {
9606    /* Some shader stages always need to have exports.
9607     * So when there is none, we need to add a null export.
9608     */
9609
9610    unsigned dest = (ctx->program->stage & hw_fs) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS;
9611    bool vm = (ctx->program->stage & hw_fs) || ctx->program->chip_class >= GFX10;
9612    Builder bld(ctx->program, ctx->block);
9613    bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
9614            /* enabled_mask */ 0, dest, /* compr */ false, /* done */ true, vm);
9615 }
9616
9617 static bool export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
9618 {
9619    assert(ctx->stage == vertex_vs ||
9620           ctx->stage == tess_eval_vs ||
9621           ctx->stage == gs_copy_vs ||
9622           ctx->stage == ngg_vertex_gs ||
9623           ctx->stage == ngg_tess_eval_gs);
9624
9625    int offset = (ctx->stage & sw_tes)
9626                 ? ctx->program->info->tes.outinfo.vs_output_param_offset[slot]
9627                 : ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
9628    uint64_t mask = ctx->outputs.mask[slot];
9629    if (!is_pos && !mask)
9630       return false;
9631    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
9632       return false;
9633    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9634    exp->enabled_mask = mask;
9635    for (unsigned i = 0; i < 4; ++i) {
9636       if (mask & (1 << i))
9637          exp->operands[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9638       else
9639          exp->operands[i] = Operand(v1);
9640    }
9641    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
9642     * Setting valid_mask=1 prevents it and has no other effect.
9643     */
9644    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
9645    exp->done = false;
9646    exp->compressed = false;
9647    if (is_pos)
9648       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9649    else
9650       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
9651    ctx->block->instructions.emplace_back(std::move(exp));
9652
9653    return true;
9654 }
9655
9656 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
9657 {
9658    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9659    exp->enabled_mask = 0;
9660    for (unsigned i = 0; i < 4; ++i)
9661       exp->operands[i] = Operand(v1);
9662    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
9663       exp->operands[0] = Operand(ctx->outputs.temps[VARYING_SLOT_PSIZ * 4u]);
9664       exp->enabled_mask |= 0x1;
9665    }
9666    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
9667       exp->operands[2] = Operand(ctx->outputs.temps[VARYING_SLOT_LAYER * 4u]);
9668       exp->enabled_mask |= 0x4;
9669    }
9670    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
9671       if (ctx->options->chip_class < GFX9) {
9672          exp->operands[3] = Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]);
9673          exp->enabled_mask |= 0x8;
9674       } else {
9675          Builder bld(ctx->program, ctx->block);
9676
9677          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
9678                              Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]));
9679          if (exp->operands[2].isTemp())
9680             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
9681
9682          exp->operands[2] = Operand(out);
9683          exp->enabled_mask |= 0x4;
9684       }
9685    }
9686    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
9687    exp->done = false;
9688    exp->compressed = false;
9689    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9690    ctx->block->instructions.emplace_back(std::move(exp));
9691 }
9692
9693 static void create_export_phis(isel_context *ctx)
9694 {
9695    /* Used when exports are needed, but the output temps are defined in a preceding block.
9696     * This function will set up phis in order to access the outputs in the next block.
9697     */
9698
9699    assert(ctx->block->instructions.back()->opcode == aco_opcode::p_logical_start);
9700    aco_ptr<Instruction> logical_start = aco_ptr<Instruction>(ctx->block->instructions.back().release());
9701    ctx->block->instructions.pop_back();
9702
9703    Builder bld(ctx->program, ctx->block);
9704
9705    for (unsigned slot = 0; slot <= VARYING_SLOT_VAR31; ++slot) {
9706       uint64_t mask = ctx->outputs.mask[slot];
9707       for (unsigned i = 0; i < 4; ++i) {
9708          if (!(mask & (1 << i)))
9709             continue;
9710
9711          Temp old = ctx->outputs.temps[slot * 4 + i];
9712          Temp phi = bld.pseudo(aco_opcode::p_phi, bld.def(v1), old, Operand(v1));
9713          ctx->outputs.temps[slot * 4 + i] = phi;
9714       }
9715    }
9716
9717    bld.insert(std::move(logical_start));
9718 }
9719
9720 static void create_vs_exports(isel_context *ctx)
9721 {
9722    assert(ctx->stage == vertex_vs ||
9723           ctx->stage == tess_eval_vs ||
9724           ctx->stage == gs_copy_vs ||
9725           ctx->stage == ngg_vertex_gs ||
9726           ctx->stage == ngg_tess_eval_gs);
9727
9728    radv_vs_output_info *outinfo = (ctx->stage & sw_tes)
9729                                   ? &ctx->program->info->tes.outinfo
9730                                   : &ctx->program->info->vs.outinfo;
9731
9732    if (outinfo->export_prim_id && !(ctx->stage & hw_ngg_gs)) {
9733       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
9734       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = get_arg(ctx, ctx->args->vs_prim_id);
9735    }
9736
9737    if (ctx->options->key.has_multiview_view_index) {
9738       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
9739       ctx->outputs.temps[VARYING_SLOT_LAYER * 4u] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
9740    }
9741
9742    /* the order these position exports are created is important */
9743    int next_pos = 0;
9744    bool exported_pos = export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
9745    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
9746       export_vs_psiz_layer_viewport(ctx, &next_pos);
9747       exported_pos = true;
9748    }
9749    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9750       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
9751    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9752       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
9753
9754    if (ctx->export_clip_dists) {
9755       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9756          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
9757       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9758          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
9759    }
9760
9761    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9762       if (i < VARYING_SLOT_VAR0 &&
9763           i != VARYING_SLOT_LAYER &&
9764           i != VARYING_SLOT_PRIMITIVE_ID &&
9765           i != VARYING_SLOT_VIEWPORT)
9766          continue;
9767
9768       export_vs_varying(ctx, i, false, NULL);
9769    }
9770
9771    if (!exported_pos)
9772       create_null_export(ctx);
9773 }
9774
9775 static bool export_fs_mrt_z(isel_context *ctx)
9776 {
9777    Builder bld(ctx->program, ctx->block);
9778    unsigned enabled_channels = 0;
9779    bool compr = false;
9780    Operand values[4];
9781
9782    for (unsigned i = 0; i < 4; ++i) {
9783       values[i] = Operand(v1);
9784    }
9785
9786    /* Both stencil and sample mask only need 16-bits. */
9787    if (!ctx->program->info->ps.writes_z &&
9788        (ctx->program->info->ps.writes_stencil ||
9789         ctx->program->info->ps.writes_sample_mask)) {
9790       compr = true; /* COMPR flag */
9791
9792       if (ctx->program->info->ps.writes_stencil) {
9793          /* Stencil should be in X[23:16]. */
9794          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9795          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
9796          enabled_channels |= 0x3;
9797       }
9798
9799       if (ctx->program->info->ps.writes_sample_mask) {
9800          /* SampleMask should be in Y[15:0]. */
9801          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9802          enabled_channels |= 0xc;
9803      }
9804    } else {
9805       if (ctx->program->info->ps.writes_z) {
9806          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]);
9807          enabled_channels |= 0x1;
9808       }
9809
9810       if (ctx->program->info->ps.writes_stencil) {
9811          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9812          enabled_channels |= 0x2;
9813       }
9814
9815       if (ctx->program->info->ps.writes_sample_mask) {
9816          values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9817          enabled_channels |= 0x4;
9818       }
9819    }
9820
9821    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
9822     * writemask component.
9823     */
9824    if (ctx->options->chip_class == GFX6 &&
9825        ctx->options->family != CHIP_OLAND &&
9826        ctx->options->family != CHIP_HAINAN) {
9827             enabled_channels |= 0x1;
9828    }
9829
9830    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9831            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
9832
9833    return true;
9834 }
9835
9836 static bool export_fs_mrt_color(isel_context *ctx, int slot)
9837 {
9838    Builder bld(ctx->program, ctx->block);
9839    unsigned write_mask = ctx->outputs.mask[slot];
9840    Operand values[4];
9841
9842    for (unsigned i = 0; i < 4; ++i) {
9843       if (write_mask & (1 << i)) {
9844          values[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9845       } else {
9846          values[i] = Operand(v1);
9847       }
9848    }
9849
9850    unsigned target, col_format;
9851    unsigned enabled_channels = 0;
9852    aco_opcode compr_op = (aco_opcode)0;
9853
9854    slot -= FRAG_RESULT_DATA0;
9855    target = V_008DFC_SQ_EXP_MRT + slot;
9856    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
9857
9858    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
9859    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
9860
9861    switch (col_format)
9862    {
9863    case V_028714_SPI_SHADER_ZERO:
9864       enabled_channels = 0; /* writemask */
9865       target = V_008DFC_SQ_EXP_NULL;
9866       break;
9867
9868    case V_028714_SPI_SHADER_32_R:
9869       enabled_channels = 1;
9870       break;
9871
9872    case V_028714_SPI_SHADER_32_GR:
9873       enabled_channels = 0x3;
9874       break;
9875
9876    case V_028714_SPI_SHADER_32_AR:
9877       if (ctx->options->chip_class >= GFX10) {
9878          /* Special case: on GFX10, the outputs are different for 32_AR */
9879          enabled_channels = 0x3;
9880          values[1] = values[3];
9881          values[3] = Operand(v1);
9882       } else {
9883          enabled_channels = 0x9;
9884       }
9885       break;
9886
9887    case V_028714_SPI_SHADER_FP16_ABGR:
9888       enabled_channels = 0x5;
9889       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
9890       break;
9891
9892    case V_028714_SPI_SHADER_UNORM16_ABGR:
9893       enabled_channels = 0x5;
9894       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
9895       break;
9896
9897    case V_028714_SPI_SHADER_SNORM16_ABGR:
9898       enabled_channels = 0x5;
9899       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
9900       break;
9901
9902    case V_028714_SPI_SHADER_UINT16_ABGR: {
9903       enabled_channels = 0x5;
9904       compr_op = aco_opcode::v_cvt_pk_u16_u32;
9905       if (is_int8 || is_int10) {
9906          /* clamp */
9907          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
9908          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9909
9910          for (unsigned i = 0; i < 4; i++) {
9911             if ((write_mask >> i) & 1) {
9912                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
9913                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
9914                                     values[i]);
9915             }
9916          }
9917       }
9918       break;
9919    }
9920
9921    case V_028714_SPI_SHADER_SINT16_ABGR:
9922       enabled_channels = 0x5;
9923       compr_op = aco_opcode::v_cvt_pk_i16_i32;
9924       if (is_int8 || is_int10) {
9925          /* clamp */
9926          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
9927          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
9928          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9929          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
9930
9931          for (unsigned i = 0; i < 4; i++) {
9932             if ((write_mask >> i) & 1) {
9933                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
9934                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
9935                                     values[i]);
9936                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
9937                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
9938                                     values[i]);
9939             }
9940          }
9941       }
9942       break;
9943
9944    case V_028714_SPI_SHADER_32_ABGR:
9945       enabled_channels = 0xF;
9946       break;
9947
9948    default:
9949       break;
9950    }
9951
9952    if (target == V_008DFC_SQ_EXP_NULL)
9953       return false;
9954
9955    if ((bool) compr_op) {
9956       for (int i = 0; i < 2; i++) {
9957          /* check if at least one of the values to be compressed is enabled */
9958          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
9959          if (enabled) {
9960             enabled_channels |= enabled << (i*2);
9961             values[i] = bld.vop3(compr_op, bld.def(v1),
9962                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
9963                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
9964          } else {
9965             values[i] = Operand(v1);
9966          }
9967       }
9968       values[2] = Operand(v1);
9969       values[3] = Operand(v1);
9970    } else {
9971       for (int i = 0; i < 4; i++)
9972          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
9973    }
9974
9975    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9976            enabled_channels, target, (bool) compr_op);
9977    return true;
9978 }
9979
9980 static void create_fs_exports(isel_context *ctx)
9981 {
9982    bool exported = false;
9983
9984    /* Export depth, stencil and sample mask. */
9985    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
9986        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
9987        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK])
9988       exported |= export_fs_mrt_z(ctx);
9989
9990    /* Export all color render targets. */
9991    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i)
9992       if (ctx->outputs.mask[i])
9993          exported |= export_fs_mrt_color(ctx, i);
9994
9995    if (!exported)
9996       create_null_export(ctx);
9997 }
9998
9999 static void write_tcs_tess_factors(isel_context *ctx)
10000 {
10001    unsigned outer_comps;
10002    unsigned inner_comps;
10003
10004    switch (ctx->args->options->key.tcs.primitive_mode) {
10005    case GL_ISOLINES:
10006       outer_comps = 2;
10007       inner_comps = 0;
10008       break;
10009    case GL_TRIANGLES:
10010       outer_comps = 3;
10011       inner_comps = 1;
10012       break;
10013    case GL_QUADS:
10014       outer_comps = 4;
10015       inner_comps = 2;
10016       break;
10017    default:
10018       return;
10019    }
10020
10021    Builder bld(ctx->program, ctx->block);
10022
10023    bld.barrier(aco_opcode::p_memory_barrier_shared);
10024    if (unlikely(ctx->program->chip_class != GFX6 && ctx->program->workgroup_size > ctx->program->wave_size))
10025       bld.sopp(aco_opcode::s_barrier);
10026
10027    Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids);
10028    Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u));
10029
10030    Temp invocation_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), invocation_id);
10031    if_context ic_invocation_id_is_zero;
10032    begin_divergent_if_then(ctx, &ic_invocation_id_is_zero, invocation_id_is_zero);
10033    bld.reset(ctx->block);
10034
10035    Temp hs_ring_tess_factor = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_FACTOR * 16u));
10036
10037    std::pair<Temp, unsigned> lds_base = get_tcs_output_lds_offset(ctx);
10038    unsigned stride = inner_comps + outer_comps;
10039    unsigned lds_align = calculate_lds_alignment(ctx, lds_base.second);
10040    Temp tf_inner_vec;
10041    Temp tf_outer_vec;
10042    Temp out[6];
10043    assert(stride <= (sizeof(out) / sizeof(Temp)));
10044
10045    if (ctx->args->options->key.tcs.primitive_mode == GL_ISOLINES) {
10046       // LINES reversal
10047       tf_outer_vec = load_lds(ctx, 4, bld.tmp(v2), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10048       out[1] = emit_extract_vector(ctx, tf_outer_vec, 0, v1);
10049       out[0] = emit_extract_vector(ctx, tf_outer_vec, 1, v1);
10050    } else {
10051       tf_outer_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, outer_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10052       tf_inner_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, inner_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_in_loc, lds_align);
10053
10054       for (unsigned i = 0; i < outer_comps; ++i)
10055          out[i] = emit_extract_vector(ctx, tf_outer_vec, i, v1);
10056       for (unsigned i = 0; i < inner_comps; ++i)
10057          out[outer_comps + i] = emit_extract_vector(ctx, tf_inner_vec, i, v1);
10058    }
10059
10060    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
10061    Temp tf_base = get_arg(ctx, ctx->args->tess_factor_offset);
10062    Temp byte_offset = bld.v_mul_imm(bld.def(v1), rel_patch_id, stride * 4u);
10063    unsigned tf_const_offset = 0;
10064
10065    if (ctx->program->chip_class <= GFX8) {
10066       Temp rel_patch_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), rel_patch_id);
10067       if_context ic_rel_patch_id_is_zero;
10068       begin_divergent_if_then(ctx, &ic_rel_patch_id_is_zero, rel_patch_id_is_zero);
10069       bld.reset(ctx->block);
10070
10071       /* Store the dynamic HS control word. */
10072       Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u));
10073       bld.mubuf(aco_opcode::buffer_store_dword,
10074                 /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word,
10075                 /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false,
10076                 /* disable_wqm */ false, /* glc */ true);
10077       tf_const_offset += 4;
10078
10079       begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero);
10080       end_divergent_if(ctx, &ic_rel_patch_id_is_zero);
10081       bld.reset(ctx->block);
10082    }
10083
10084    assert(stride == 2 || stride == 4 || stride == 6);
10085    Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr, 4u);
10086    store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false);
10087
10088    /* Store to offchip for TES to read - only if TES reads them */
10089    if (ctx->args->options->key.tcs.tes_reads_tess_factors) {
10090       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
10091       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
10092
10093       std::pair<Temp, unsigned> vmem_offs_outer = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_out_loc);
10094       store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, false);
10095
10096       if (likely(inner_comps)) {
10097          std::pair<Temp, unsigned> vmem_offs_inner = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_in_loc);
10098          store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, false);
10099       }
10100    }
10101
10102    begin_divergent_if_else(ctx, &ic_invocation_id_is_zero);
10103    end_divergent_if(ctx, &ic_invocation_id_is_zero);
10104 }
10105
10106 static void emit_stream_output(isel_context *ctx,
10107                                Temp const *so_buffers,
10108                                Temp const *so_write_offset,
10109                                const struct radv_stream_output *output)
10110 {
10111    unsigned num_comps = util_bitcount(output->component_mask);
10112    unsigned writemask = (1 << num_comps) - 1;
10113    unsigned loc = output->location;
10114    unsigned buf = output->buffer;
10115
10116    assert(num_comps && num_comps <= 4);
10117    if (!num_comps || num_comps > 4)
10118       return;
10119
10120    unsigned start = ffs(output->component_mask) - 1;
10121
10122    Temp out[4];
10123    bool all_undef = true;
10124    assert(ctx->stage & hw_vs);
10125    for (unsigned i = 0; i < num_comps; i++) {
10126       out[i] = ctx->outputs.temps[loc * 4 + start + i];
10127       all_undef = all_undef && !out[i].id();
10128    }
10129    if (all_undef)
10130       return;
10131
10132    while (writemask) {
10133       int start, count;
10134       u_bit_scan_consecutive_range(&writemask, &start, &count);
10135       if (count == 3 && ctx->options->chip_class == GFX6) {
10136          /* GFX6 doesn't support storing vec3, split it. */
10137          writemask |= 1u << (start + 2);
10138          count = 2;
10139       }
10140
10141       unsigned offset = output->offset + start * 4;
10142
10143       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
10144       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
10145       for (int i = 0; i < count; ++i)
10146          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
10147       vec->definitions[0] = Definition(write_data);
10148       ctx->block->instructions.emplace_back(std::move(vec));
10149
10150       aco_opcode opcode;
10151       switch (count) {
10152       case 1:
10153          opcode = aco_opcode::buffer_store_dword;
10154          break;
10155       case 2:
10156          opcode = aco_opcode::buffer_store_dwordx2;
10157          break;
10158       case 3:
10159          opcode = aco_opcode::buffer_store_dwordx3;
10160          break;
10161       case 4:
10162          opcode = aco_opcode::buffer_store_dwordx4;
10163          break;
10164       default:
10165          unreachable("Unsupported dword count.");
10166       }
10167
10168       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
10169       store->operands[0] = Operand(so_buffers[buf]);
10170       store->operands[1] = Operand(so_write_offset[buf]);
10171       store->operands[2] = Operand((uint32_t) 0);
10172       store->operands[3] = Operand(write_data);
10173       if (offset > 4095) {
10174          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
10175          Builder bld(ctx->program, ctx->block);
10176          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
10177       } else {
10178          store->offset = offset;
10179       }
10180       store->offen = true;
10181       store->glc = true;
10182       store->dlc = false;
10183       store->slc = true;
10184       store->can_reorder = true;
10185       ctx->block->instructions.emplace_back(std::move(store));
10186    }
10187 }
10188
10189 static void emit_streamout(isel_context *ctx, unsigned stream)
10190 {
10191    Builder bld(ctx->program, ctx->block);
10192
10193    Temp so_buffers[4];
10194    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
10195    for (unsigned i = 0; i < 4; i++) {
10196       unsigned stride = ctx->program->info->so.strides[i];
10197       if (!stride)
10198          continue;
10199
10200       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
10201       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
10202    }
10203
10204    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10205                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
10206
10207    Temp tid = emit_mbcnt(ctx, bld.def(v1));
10208
10209    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
10210
10211    if_context ic;
10212    begin_divergent_if_then(ctx, &ic, can_emit);
10213
10214    bld.reset(ctx->block);
10215
10216    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
10217
10218    Temp so_write_offset[4];
10219
10220    for (unsigned i = 0; i < 4; i++) {
10221       unsigned stride = ctx->program->info->so.strides[i];
10222       if (!stride)
10223          continue;
10224
10225       if (stride == 1) {
10226          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
10227                                 get_arg(ctx, ctx->args->streamout_write_idx),
10228                                 get_arg(ctx, ctx->args->streamout_offset[i]));
10229          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
10230
10231          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
10232       } else {
10233          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
10234          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
10235                                  get_arg(ctx, ctx->args->streamout_offset[i]));
10236          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
10237       }
10238    }
10239
10240    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
10241       struct radv_stream_output *output =
10242          &ctx->program->info->so.outputs[i];
10243       if (stream != output->stream)
10244          continue;
10245
10246       emit_stream_output(ctx, so_buffers, so_write_offset, output);
10247    }
10248
10249    begin_divergent_if_else(ctx, &ic);
10250    end_divergent_if(ctx, &ic);
10251 }
10252
10253 } /* end namespace */
10254
10255 void fix_ls_vgpr_init_bug(isel_context *ctx, Pseudo_instruction *startpgm)
10256 {
10257    assert(ctx->shader->info.stage == MESA_SHADER_VERTEX);
10258    Builder bld(ctx->program, ctx->block);
10259    constexpr unsigned hs_idx = 1u;
10260    Builder::Result hs_thread_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10261                                               get_arg(ctx, ctx->args->merged_wave_info),
10262                                               Operand((8u << 16) | (hs_idx * 8u)));
10263    Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp());
10264
10265    /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */
10266
10267    Temp instance_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10268                                get_arg(ctx, ctx->args->rel_auto_id),
10269                                get_arg(ctx, ctx->args->ac.instance_id),
10270                                ls_has_nonzero_hs_threads);
10271    Temp rel_auto_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10272                                get_arg(ctx, ctx->args->ac.tcs_rel_ids),
10273                                get_arg(ctx, ctx->args->rel_auto_id),
10274                                ls_has_nonzero_hs_threads);
10275    Temp vertex_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10276                              get_arg(ctx, ctx->args->ac.tcs_patch_id),
10277                              get_arg(ctx, ctx->args->ac.vertex_id),
10278                              ls_has_nonzero_hs_threads);
10279
10280    ctx->arg_temps[ctx->args->ac.instance_id.arg_index] = instance_id;
10281    ctx->arg_temps[ctx->args->rel_auto_id.arg_index] = rel_auto_id;
10282    ctx->arg_temps[ctx->args->ac.vertex_id.arg_index] = vertex_id;
10283 }
10284
10285 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
10286 {
10287    /* Split all arguments except for the first (ring_offsets) and the last
10288     * (exec) so that the dead channels don't stay live throughout the program.
10289     */
10290    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
10291       if (startpgm->definitions[i].regClass().size() > 1) {
10292          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
10293                            startpgm->definitions[i].regClass().size());
10294       }
10295    }
10296 }
10297
10298 void handle_bc_optimize(isel_context *ctx)
10299 {
10300    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
10301    Builder bld(ctx->program, ctx->block);
10302    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
10303    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
10304    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
10305    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
10306    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
10307    if (uses_center && uses_centroid) {
10308       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
10309                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
10310
10311       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
10312          Temp new_coord[2];
10313          for (unsigned i = 0; i < 2; i++) {
10314             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
10315             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
10316             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10317                                     persp_centroid, persp_center, sel);
10318          }
10319          ctx->persp_centroid = bld.tmp(v2);
10320          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
10321                     Operand(new_coord[0]), Operand(new_coord[1]));
10322          emit_split_vector(ctx, ctx->persp_centroid, 2);
10323       }
10324
10325       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
10326          Temp new_coord[2];
10327          for (unsigned i = 0; i < 2; i++) {
10328             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
10329             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
10330             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10331                                     linear_centroid, linear_center, sel);
10332          }
10333          ctx->linear_centroid = bld.tmp(v2);
10334          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
10335                     Operand(new_coord[0]), Operand(new_coord[1]));
10336          emit_split_vector(ctx, ctx->linear_centroid, 2);
10337       }
10338    }
10339 }
10340
10341 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
10342 {
10343    Program *program = ctx->program;
10344
10345    unsigned float_controls = shader->info.float_controls_execution_mode;
10346
10347    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
10348       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
10349    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
10350       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
10351                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
10352
10353    program->next_fp_mode.must_flush_denorms32 =
10354       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
10355    program->next_fp_mode.must_flush_denorms16_64 =
10356       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
10357                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
10358
10359    program->next_fp_mode.care_about_round32 =
10360       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
10361
10362    program->next_fp_mode.care_about_round16_64 =
10363       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
10364                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
10365
10366    /* default to preserving fp16 and fp64 denorms, since it's free */
10367    if (program->next_fp_mode.must_flush_denorms16_64)
10368       program->next_fp_mode.denorm16_64 = 0;
10369    else
10370       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10371
10372    /* preserving fp32 denorms is expensive, so only do it if asked */
10373    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
10374       program->next_fp_mode.denorm32 = fp_denorm_keep;
10375    else
10376       program->next_fp_mode.denorm32 = 0;
10377
10378    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
10379       program->next_fp_mode.round32 = fp_round_tz;
10380    else
10381       program->next_fp_mode.round32 = fp_round_ne;
10382
10383    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
10384       program->next_fp_mode.round16_64 = fp_round_tz;
10385    else
10386       program->next_fp_mode.round16_64 = fp_round_ne;
10387
10388    ctx->block->fp_mode = program->next_fp_mode;
10389 }
10390
10391 void cleanup_cfg(Program *program)
10392 {
10393    /* create linear_succs/logical_succs */
10394    for (Block& BB : program->blocks) {
10395       for (unsigned idx : BB.linear_preds)
10396          program->blocks[idx].linear_succs.emplace_back(BB.index);
10397       for (unsigned idx : BB.logical_preds)
10398          program->blocks[idx].logical_succs.emplace_back(BB.index);
10399    }
10400 }
10401
10402 Temp merged_wave_info_to_mask(isel_context *ctx, unsigned i)
10403 {
10404    Builder bld(ctx->program, ctx->block);
10405
10406    /* The s_bfm only cares about s0.u[5:0] so we don't need either s_bfe nor s_and here */
10407    Temp count = i == 0
10408                 ? get_arg(ctx, ctx->args->merged_wave_info)
10409                 : bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc),
10410                            get_arg(ctx, ctx->args->merged_wave_info), Operand(i * 8u));
10411
10412    Temp mask = bld.sop2(aco_opcode::s_bfm_b64, bld.def(s2), count, Operand(0u));
10413    Temp cond;
10414
10415    if (ctx->program->wave_size == 64) {
10416       /* Special case for 64 active invocations, because 64 doesn't work with s_bfm */
10417       Temp active_64 = bld.sopc(aco_opcode::s_bitcmp1_b32, bld.def(s1, scc), count, Operand(6u /* log2(64) */));
10418       cond = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), mask, bld.scc(active_64));
10419    } else {
10420       /* We use s_bfm_b64 (not _b32) which works with 32, but we need to extract the lower half of the register */
10421       cond = emit_extract_vector(ctx, mask, 0, bld.lm);
10422    }
10423
10424    return cond;
10425 }
10426
10427 bool ngg_early_prim_export(isel_context *ctx)
10428 {
10429    /* TODO: Check edge flags, and if they are written, return false. (Needed for OpenGL, not for Vulkan.) */
10430    return true;
10431 }
10432
10433 void ngg_emit_sendmsg_gs_alloc_req(isel_context *ctx)
10434 {
10435    Builder bld(ctx->program, ctx->block);
10436
10437    /* It is recommended to do the GS_ALLOC_REQ as soon and as quickly as possible, so we set the maximum priority (3). */
10438    bld.sopp(aco_opcode::s_setprio, -1u, 0x3u);
10439
10440    /* Get the id of the current wave within the threadgroup (workgroup) */
10441    Builder::Result wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10442                                             get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10443
10444    /* Execute the following code only on the first wave (wave id 0),
10445     * use the SCC def to tell if the wave id is zero or not.
10446     */
10447    Temp cond = wave_id_in_tg.def(1).getTemp();
10448    if_context ic;
10449    begin_uniform_if_then(ctx, &ic, cond);
10450    begin_uniform_if_else(ctx, &ic);
10451    bld.reset(ctx->block);
10452
10453    /* Number of vertices output by VS/TES */
10454    Temp vtx_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10455                            get_arg(ctx, ctx->args->gs_tg_info), Operand(12u | (9u << 16u)));
10456    /* Number of primitives output by VS/TES */
10457    Temp prm_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10458                            get_arg(ctx, ctx->args->gs_tg_info), Operand(22u | (9u << 16u)));
10459
10460    /* Put the number of vertices and primitives into m0 for the GS_ALLOC_REQ */
10461    Temp tmp = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), prm_cnt, Operand(12u));
10462    tmp = bld.sop2(aco_opcode::s_or_b32, bld.m0(bld.def(s1)), bld.def(s1, scc), tmp, vtx_cnt);
10463
10464    /* Request the SPI to allocate space for the primitives and vertices that will be exported by the threadgroup. */
10465    bld.sopp(aco_opcode::s_sendmsg, bld.m0(tmp), -1, sendmsg_gs_alloc_req);
10466
10467    /* After the GS_ALLOC_REQ is done, reset priority to default (0). */
10468    bld.sopp(aco_opcode::s_setprio, -1u, 0x0u);
10469
10470    end_uniform_if(ctx, &ic);
10471 }
10472
10473 Temp ngg_get_prim_exp_arg(isel_context *ctx, unsigned num_vertices, const Temp vtxindex[])
10474 {
10475    Builder bld(ctx->program, ctx->block);
10476
10477    if (ctx->args->options->key.vs_common_out.as_ngg_passthrough) {
10478       return get_arg(ctx, ctx->args->gs_vtx_offset[0]);
10479    }
10480
10481    Temp gs_invocation_id = get_arg(ctx, ctx->args->ac.gs_invocation_id);
10482    Temp tmp;
10483
10484    for (unsigned i = 0; i < num_vertices; ++i) {
10485       assert(vtxindex[i].id());
10486
10487       if (i)
10488          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), vtxindex[i], Operand(10u * i), tmp);
10489       else
10490          tmp = vtxindex[i];
10491
10492       /* The initial edge flag is always false in tess eval shaders. */
10493       if (ctx->stage == ngg_vertex_gs) {
10494          Temp edgeflag = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), gs_invocation_id, Operand(8 + i), Operand(1u));
10495          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), edgeflag, Operand(10u * i + 9u), tmp);
10496       }
10497    }
10498
10499    /* TODO: Set isnull field in case of merged NGG VS+GS. */
10500
10501    return tmp;
10502 }
10503
10504 void ngg_emit_prim_export(isel_context *ctx, unsigned num_vertices_per_primitive, const Temp vtxindex[])
10505 {
10506    Builder bld(ctx->program, ctx->block);
10507    Temp prim_exp_arg = ngg_get_prim_exp_arg(ctx, num_vertices_per_primitive, vtxindex);
10508
10509    bld.exp(aco_opcode::exp, prim_exp_arg, Operand(v1), Operand(v1), Operand(v1),
10510         1 /* enabled mask */, V_008DFC_SQ_EXP_PRIM /* dest */,
10511         false /* compressed */, true/* done */, false /* valid mask */);
10512 }
10513
10514 void ngg_emit_nogs_gsthreads(isel_context *ctx)
10515 {
10516    /* Emit the things that NGG GS threads need to do, for shaders that don't have SW GS.
10517     * These must always come before VS exports.
10518     *
10519     * It is recommended to do these as early as possible. They can be at the beginning when
10520     * there is no SW GS and the shader doesn't write edge flags.
10521     */
10522
10523    if_context ic;
10524    Temp is_gs_thread = merged_wave_info_to_mask(ctx, 1);
10525    begin_divergent_if_then(ctx, &ic, is_gs_thread);
10526
10527    Builder bld(ctx->program, ctx->block);
10528    constexpr unsigned max_vertices_per_primitive = 3;
10529    unsigned num_vertices_per_primitive = max_vertices_per_primitive;
10530
10531    if (ctx->stage == ngg_vertex_gs) {
10532       /* TODO: optimize for points & lines */
10533    } else if (ctx->stage == ngg_tess_eval_gs) {
10534       if (ctx->shader->info.tess.point_mode)
10535          num_vertices_per_primitive = 1;
10536       else if (ctx->shader->info.tess.primitive_mode == GL_ISOLINES)
10537          num_vertices_per_primitive = 2;
10538    } else {
10539       unreachable("Unsupported NGG shader stage");
10540    }
10541
10542    Temp vtxindex[max_vertices_per_primitive];
10543    vtxindex[0] = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10544                           get_arg(ctx, ctx->args->gs_vtx_offset[0]));
10545    vtxindex[1] = num_vertices_per_primitive < 2 ? Temp(0, v1) :
10546                  bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
10547                           get_arg(ctx, ctx->args->gs_vtx_offset[0]), Operand(16u), Operand(16u));
10548    vtxindex[2] = num_vertices_per_primitive < 3 ? Temp(0, v1) :
10549                  bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10550                           get_arg(ctx, ctx->args->gs_vtx_offset[2]));
10551
10552    /* Export primitive data to the index buffer. */
10553    ngg_emit_prim_export(ctx, num_vertices_per_primitive, vtxindex);
10554
10555    /* Export primitive ID. */
10556    if (ctx->stage == ngg_vertex_gs && ctx->args->options->key.vs_common_out.export_prim_id) {
10557       /* Copy Primitive IDs from GS threads to the LDS address corresponding to the ES thread of the provoking vertex. */
10558       Temp prim_id = get_arg(ctx, ctx->args->ac.gs_prim_id);
10559       Temp provoking_vtx_index = vtxindex[0];
10560       Temp addr = bld.v_mul_imm(bld.def(v1), provoking_vtx_index, 4u);
10561
10562       store_lds(ctx, 4, prim_id, 0x1u, addr, 0u, 4u);
10563    }
10564
10565    begin_divergent_if_else(ctx, &ic);
10566    end_divergent_if(ctx, &ic);
10567 }
10568
10569 void ngg_emit_nogs_output(isel_context *ctx)
10570 {
10571    /* Emits NGG GS output, for stages that don't have SW GS. */
10572
10573    if_context ic;
10574    Builder bld(ctx->program, ctx->block);
10575    bool late_prim_export = !ngg_early_prim_export(ctx);
10576
10577    /* NGG streamout is currently disabled by default. */
10578    assert(!ctx->args->shader_info->so.num_outputs);
10579
10580    if (late_prim_export) {
10581       /* VS exports are output to registers in a predecessor block. Emit phis to get them into this block. */
10582       create_export_phis(ctx);
10583       /* Do what we need to do in the GS threads. */
10584       ngg_emit_nogs_gsthreads(ctx);
10585
10586       /* What comes next should be executed on ES threads. */
10587       Temp is_es_thread = merged_wave_info_to_mask(ctx, 0);
10588       begin_divergent_if_then(ctx, &ic, is_es_thread);
10589       bld.reset(ctx->block);
10590    }
10591
10592    /* Export VS outputs */
10593    ctx->block->kind |= block_kind_export_end;
10594    create_vs_exports(ctx);
10595
10596    /* Export primitive ID */
10597    if (ctx->args->options->key.vs_common_out.export_prim_id) {
10598       Temp prim_id;
10599
10600       if (ctx->stage == ngg_vertex_gs) {
10601          /* Wait for GS threads to store primitive ID in LDS. */
10602          bld.barrier(aco_opcode::p_memory_barrier_shared);
10603          bld.sopp(aco_opcode::s_barrier);
10604
10605          /* Calculate LDS address where the GS threads stored the primitive ID. */
10606          Temp wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10607                                        get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10608          Temp thread_id_in_wave = emit_mbcnt(ctx, bld.def(v1));
10609          Temp wave_id_mul = bld.v_mul_imm(bld.def(v1), as_vgpr(ctx, wave_id_in_tg), ctx->program->wave_size);
10610          Temp thread_id_in_tg = bld.vadd32(bld.def(v1), Operand(wave_id_mul), Operand(thread_id_in_wave));
10611          Temp addr = bld.v_mul_imm(bld.def(v1), thread_id_in_tg, 4u);
10612
10613          /* Load primitive ID from LDS. */
10614          prim_id = load_lds(ctx, 4, bld.tmp(v1), addr, 0u, 4u);
10615       } else if (ctx->stage == ngg_tess_eval_gs) {
10616          /* TES: Just use the patch ID as the primitive ID. */
10617          prim_id = get_arg(ctx, ctx->args->ac.tes_patch_id);
10618       } else {
10619          unreachable("unsupported NGG shader stage.");
10620       }
10621
10622       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
10623       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = prim_id;
10624
10625       export_vs_varying(ctx, VARYING_SLOT_PRIMITIVE_ID, false, nullptr);
10626    }
10627
10628    if (late_prim_export) {
10629       begin_divergent_if_else(ctx, &ic);
10630       end_divergent_if(ctx, &ic);
10631       bld.reset(ctx->block);
10632    }
10633 }
10634
10635 void select_program(Program *program,
10636                     unsigned shader_count,
10637                     struct nir_shader *const *shaders,
10638                     ac_shader_config* config,
10639                     struct radv_shader_args *args)
10640 {
10641    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
10642    if_context ic_merged_wave_info;
10643    bool ngg_no_gs = ctx.stage == ngg_vertex_gs || ctx.stage == ngg_tess_eval_gs;
10644
10645    for (unsigned i = 0; i < shader_count; i++) {
10646       nir_shader *nir = shaders[i];
10647       init_context(&ctx, nir);
10648
10649       setup_fp_mode(&ctx, nir);
10650
10651       if (!i) {
10652          /* needs to be after init_context() for FS */
10653          Pseudo_instruction *startpgm = add_startpgm(&ctx);
10654          append_logical_start(ctx.block);
10655
10656          if (unlikely(args->options->has_ls_vgpr_init_bug && ctx.stage == vertex_tess_control_hs))
10657             fix_ls_vgpr_init_bug(&ctx, startpgm);
10658
10659          split_arguments(&ctx, startpgm);
10660       }
10661
10662       if (ngg_no_gs) {
10663          ngg_emit_sendmsg_gs_alloc_req(&ctx);
10664
10665          if (ngg_early_prim_export(&ctx))
10666             ngg_emit_nogs_gsthreads(&ctx);
10667       }
10668
10669       /* In a merged VS+TCS HS, the VS implementation can be completely empty. */
10670       nir_function_impl *func = nir_shader_get_entrypoint(nir);
10671       bool empty_shader = nir_cf_list_is_empty_block(&func->body) &&
10672                           ((nir->info.stage == MESA_SHADER_VERTEX &&
10673                             (ctx.stage == vertex_tess_control_hs || ctx.stage == vertex_geometry_gs)) ||
10674                            (nir->info.stage == MESA_SHADER_TESS_EVAL &&
10675                             ctx.stage == tess_eval_geometry_gs));
10676
10677       bool check_merged_wave_info = ctx.tcs_in_out_eq ? i == 0 : ((shader_count >= 2 && !empty_shader) || ngg_no_gs);
10678       bool endif_merged_wave_info = ctx.tcs_in_out_eq ? i == 1 : check_merged_wave_info;
10679       if (check_merged_wave_info) {
10680          Temp cond = merged_wave_info_to_mask(&ctx, i);
10681          begin_divergent_if_then(&ctx, &ic_merged_wave_info, cond);
10682       }
10683
10684       if (i) {
10685          Builder bld(ctx.program, ctx.block);
10686
10687          bld.barrier(aco_opcode::p_memory_barrier_shared);
10688          bld.sopp(aco_opcode::s_barrier);
10689
10690          if (ctx.stage == vertex_geometry_gs || ctx.stage == tess_eval_geometry_gs) {
10691             ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
10692          }
10693       } else if (ctx.stage == geometry_gs)
10694          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
10695
10696       if (ctx.stage == fragment_fs)
10697          handle_bc_optimize(&ctx);
10698
10699       visit_cf_list(&ctx, &func->body);
10700
10701       if (ctx.program->info->so.num_outputs && (ctx.stage & hw_vs))
10702          emit_streamout(&ctx, 0);
10703
10704       if (ctx.stage & hw_vs) {
10705          create_vs_exports(&ctx);
10706          ctx.block->kind |= block_kind_export_end;
10707       } else if (ngg_no_gs && ngg_early_prim_export(&ctx)) {
10708          ngg_emit_nogs_output(&ctx);
10709       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
10710          Builder bld(ctx.program, ctx.block);
10711          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
10712          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
10713       } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
10714          write_tcs_tess_factors(&ctx);
10715       }
10716
10717       if (ctx.stage == fragment_fs) {
10718          create_fs_exports(&ctx);
10719          ctx.block->kind |= block_kind_export_end;
10720       }
10721
10722       if (endif_merged_wave_info) {
10723          begin_divergent_if_else(&ctx, &ic_merged_wave_info);
10724          end_divergent_if(&ctx, &ic_merged_wave_info);
10725       }
10726
10727       if (ngg_no_gs && !ngg_early_prim_export(&ctx))
10728          ngg_emit_nogs_output(&ctx);
10729
10730       ralloc_free(ctx.divergent_vals);
10731
10732       if (i == 0 && ctx.stage == vertex_tess_control_hs && ctx.tcs_in_out_eq) {
10733          /* Outputs of the previous stage are inputs to the next stage */
10734          ctx.inputs = ctx.outputs;
10735          ctx.outputs = shader_io_state();
10736       }
10737    }
10738
10739    program->config->float_mode = program->blocks[0].fp_mode.val;
10740
10741    append_logical_end(ctx.block);
10742    ctx.block->kind |= block_kind_uniform;
10743    Builder bld(ctx.program, ctx.block);
10744    if (ctx.program->wb_smem_l1_on_end)
10745       bld.smem(aco_opcode::s_dcache_wb, false);
10746    bld.sopp(aco_opcode::s_endpgm);
10747
10748    cleanup_cfg(program);
10749 }
10750
10751 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
10752                            ac_shader_config* config,
10753                            struct radv_shader_args *args)
10754 {
10755    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
10756
10757    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
10758    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
10759    program->next_fp_mode.must_flush_denorms32 = false;
10760    program->next_fp_mode.must_flush_denorms16_64 = false;
10761    program->next_fp_mode.care_about_round32 = false;
10762    program->next_fp_mode.care_about_round16_64 = false;
10763    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10764    program->next_fp_mode.denorm32 = 0;
10765    program->next_fp_mode.round32 = fp_round_ne;
10766    program->next_fp_mode.round16_64 = fp_round_ne;
10767    ctx.block->fp_mode = program->next_fp_mode;
10768
10769    add_startpgm(&ctx);
10770    append_logical_start(ctx.block);
10771
10772    Builder bld(ctx.program, ctx.block);
10773
10774    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
10775
10776    Operand stream_id(0u);
10777    if (args->shader_info->so.num_outputs)
10778       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10779                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
10780
10781    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
10782
10783    std::stack<Block> endif_blocks;
10784
10785    for (unsigned stream = 0; stream < 4; stream++) {
10786       if (stream_id.isConstant() && stream != stream_id.constantValue())
10787          continue;
10788
10789       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
10790       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
10791          continue;
10792
10793       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
10794
10795       unsigned BB_if_idx = ctx.block->index;
10796       Block BB_endif = Block();
10797       if (!stream_id.isConstant()) {
10798          /* begin IF */
10799          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
10800          append_logical_end(ctx.block);
10801          ctx.block->kind |= block_kind_uniform;
10802          bld.branch(aco_opcode::p_cbranch_z, cond);
10803
10804          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
10805
10806          ctx.block = ctx.program->create_and_insert_block();
10807          add_edge(BB_if_idx, ctx.block);
10808          bld.reset(ctx.block);
10809          append_logical_start(ctx.block);
10810       }
10811
10812       unsigned offset = 0;
10813       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
10814          if (args->shader_info->gs.output_streams[i] != stream)
10815             continue;
10816
10817          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
10818          unsigned length = util_last_bit(output_usage_mask);
10819          for (unsigned j = 0; j < length; ++j) {
10820             if (!(output_usage_mask & (1 << j)))
10821                continue;
10822
10823             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
10824             Temp voffset = vtx_offset;
10825             if (const_offset >= 4096u) {
10826                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
10827                const_offset %= 4096u;
10828             }
10829
10830             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
10831             mubuf->definitions[0] = bld.def(v1);
10832             mubuf->operands[0] = Operand(gsvs_ring);
10833             mubuf->operands[1] = Operand(voffset);
10834             mubuf->operands[2] = Operand(0u);
10835             mubuf->offen = true;
10836             mubuf->offset = const_offset;
10837             mubuf->glc = true;
10838             mubuf->slc = true;
10839             mubuf->dlc = args->options->chip_class >= GFX10;
10840             mubuf->barrier = barrier_none;
10841             mubuf->can_reorder = true;
10842
10843             ctx.outputs.mask[i] |= 1 << j;
10844             ctx.outputs.temps[i * 4u + j] = mubuf->definitions[0].getTemp();
10845
10846             bld.insert(std::move(mubuf));
10847
10848             offset++;
10849          }
10850       }
10851
10852       if (args->shader_info->so.num_outputs) {
10853          emit_streamout(&ctx, stream);
10854          bld.reset(ctx.block);
10855       }
10856
10857       if (stream == 0) {
10858          create_vs_exports(&ctx);
10859          ctx.block->kind |= block_kind_export_end;
10860       }
10861
10862       if (!stream_id.isConstant()) {
10863          append_logical_end(ctx.block);
10864
10865          /* branch from then block to endif block */
10866          bld.branch(aco_opcode::p_branch);
10867          add_edge(ctx.block->index, &BB_endif);
10868          ctx.block->kind |= block_kind_uniform;
10869
10870          /* emit else block */
10871          ctx.block = ctx.program->create_and_insert_block();
10872          add_edge(BB_if_idx, ctx.block);
10873          bld.reset(ctx.block);
10874          append_logical_start(ctx.block);
10875
10876          endif_blocks.push(std::move(BB_endif));
10877       }
10878    }
10879
10880    while (!endif_blocks.empty()) {
10881       Block BB_endif = std::move(endif_blocks.top());
10882       endif_blocks.pop();
10883
10884       Block *BB_else = ctx.block;
10885
10886       append_logical_end(BB_else);
10887       /* branch from else block to endif block */
10888       bld.branch(aco_opcode::p_branch);
10889       add_edge(BB_else->index, &BB_endif);
10890       BB_else->kind |= block_kind_uniform;
10891
10892       /** emit endif merge block */
10893       ctx.block = program->insert_block(std::move(BB_endif));
10894       bld.reset(ctx.block);
10895       append_logical_start(ctx.block);
10896    }
10897
10898    program->config->float_mode = program->blocks[0].fp_mode.val;
10899
10900    append_logical_end(ctx.block);
10901    ctx.block->kind |= block_kind_uniform;
10902    bld.sopp(aco_opcode::s_endpgm);
10903
10904    cleanup_cfg(program);
10905 }
10906 }