src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool uniform_has_then_branch;
  89    bool then_branch_divergent;
  90    Block BB_invert;
  91    Block BB_endif;
  92 };
  93
  94 static bool visit_cf_list(struct isel_context *ctx,
  95                           struct exec_list *list);
  96
  97 static void add_logical_edge(unsigned pred_idx, Block *succ)
  98 {
  99    succ->logical_preds.emplace_back(pred_idx);
 100 }
 101
 102
 103 static void add_linear_edge(unsigned pred_idx, Block *succ)
 104 {
 105    succ->linear_preds.emplace_back(pred_idx);
 106 }
 107
 108 static void add_edge(unsigned pred_idx, Block *succ)
 109 {
 110    add_logical_edge(pred_idx, succ);
 111    add_linear_edge(pred_idx, succ);
 112 }
 113
 114 static void append_logical_start(Block *b)
 115 {
 116    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 117 }
 118
 119 static void append_logical_end(Block *b)
 120 {
 121    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 122 }
 123
 124 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 125 {
 126    assert(ctx->allocated[def->index].id());
 127    return ctx->allocated[def->index];
 128 }
 129
 130 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 131                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 132 {
 133    Builder bld(ctx->program, ctx->block);
 134    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 135    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 136
 137    if (ctx->program->wave_size == 32) {
 138       return thread_id_lo;
 139    } else {
 140       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 141       return thread_id_hi;
 142    }
 143 }
 144
 145 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 146 {
 147    Builder bld(ctx->program, ctx->block);
 148
 149    if (!dst.id())
 150       dst = bld.tmp(src.regClass());
 151
 152    assert(src.size() == dst.size());
 153
 154    if (ctx->stage != fragment_fs) {
 155       if (!dst.id())
 156          return src;
 157
 158       bld.copy(Definition(dst), src);
 159       return dst;
 160    }
 161
 162    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 163    ctx->program->needs_wqm |= program_needs_wqm;
 164    return dst;
 165 }
 166
 167 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 168 {
 169    if (index.regClass() == s1)
 170       return bld.readlane(bld.def(s1), data, index);
 171
 172    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 173
 174    /* Currently not implemented on GFX6-7 */
 175    assert(ctx->options->chip_class >= GFX8);
 176
 177    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 178       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 179    }
 180
 181    /* GFX10, wave64 mode:
 182     * The bpermute instruction is limited to half-wave operation, which means that it can't
 183     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 184     * emulate it here.
 185     */
 186    if (!ctx->has_gfx10_wave64_bpermute) {
 187       ctx->has_gfx10_wave64_bpermute = true;
 188       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 189       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 190    }
 191
 192    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 193    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 194    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 195    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 196
 197    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 198                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 199 }
 200
 201 Temp as_vgpr(isel_context *ctx, Temp val)
 202 {
 203    if (val.type() == RegType::sgpr) {
 204       Builder bld(ctx->program, ctx->block);
 205       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 206    }
 207    assert(val.type() == RegType::vgpr);
 208    return val;
 209 }
 210
 211 //assumes a != 0xffffffff
 212 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 213 {
 214    assert(b != 0);
 215    Builder bld(ctx->program, ctx->block);
 216
 217    if (util_is_power_of_two_or_zero(b)) {
 218       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 219       return;
 220    }
 221
 222    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 223
 224    assert(info.multiplier <= 0xffffffff);
 225
 226    bool pre_shift = info.pre_shift != 0;
 227    bool increment = info.increment != 0;
 228    bool multiply = true;
 229    bool post_shift = info.post_shift != 0;
 230
 231    if (!pre_shift && !increment && !multiply && !post_shift) {
 232       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 233       return;
 234    }
 235
 236    Temp pre_shift_dst = a;
 237    if (pre_shift) {
 238       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 239       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 240    }
 241
 242    Temp increment_dst = pre_shift_dst;
 243    if (increment) {
 244       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 245       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 246    }
 247
 248    Temp multiply_dst = increment_dst;
 249    if (multiply) {
 250       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 251       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 252                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 253    }
 254
 255    if (post_shift) {
 256       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 257    }
 258 }
 259
 260 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 261 {
 262    Builder bld(ctx->program, ctx->block);
 263    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 264 }
 265
 266
 267 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 268 {
 269    /* no need to extract the whole vector */
 270    if (src.regClass() == dst_rc) {
 271       assert(idx == 0);
 272       return src;
 273    }
 274
 275    assert(src.bytes() > (idx * dst_rc.bytes()));
 276    Builder bld(ctx->program, ctx->block);
 277    auto it = ctx->allocated_vec.find(src.id());
 278    if (it != ctx->allocated_vec.end() && dst_rc.bytes() == it->second[idx].regClass().bytes()) {
 279       if (it->second[idx].regClass() == dst_rc) {
 280          return it->second[idx];
 281       } else {
 282          assert(!dst_rc.is_subdword());
 283          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 284          return bld.copy(bld.def(dst_rc), it->second[idx]);
 285       }
 286    }
 287
 288    if (dst_rc.is_subdword())
 289       src = as_vgpr(ctx, src);
 290
 291    if (src.bytes() == dst_rc.bytes()) {
 292       assert(idx == 0);
 293       return bld.copy(bld.def(dst_rc), src);
 294    } else {
 295       Temp dst = bld.tmp(dst_rc);
 296       emit_extract_vector(ctx, src, idx, dst);
 297       return dst;
 298    }
 299 }
 300
 301 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 302 {
 303    if (num_components == 1)
 304       return;
 305    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 306       return;
 307    RegClass rc;
 308    if (num_components > vec_src.size()) {
 309       if (vec_src.type() == RegType::sgpr) {
 310          /* should still help get_alu_src() */
 311          emit_split_vector(ctx, vec_src, vec_src.size());
 312          return;
 313       }
 314       /* sub-dword split */
 315       rc = RegClass(RegType::vgpr, vec_src.bytes() / num_components).as_subdword();
 316    } else {
 317       rc = RegClass(vec_src.type(), vec_src.size() / num_components);
 318    }
 319    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 320    split->operands[0] = Operand(vec_src);
 321    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 322    for (unsigned i = 0; i < num_components; i++) {
 323       elems[i] = {ctx->program->allocateId(), rc};
 324       split->definitions[i] = Definition(elems[i]);
 325    }
 326    ctx->block->instructions.emplace_back(std::move(split));
 327    ctx->allocated_vec.emplace(vec_src.id(), elems);
 328 }
 329
 330 /* This vector expansion uses a mask to determine which elements in the new vector
 331  * come from the original vector. The other elements are undefined. */
 332 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 333 {
 334    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 335
 336    if (vec_src == dst)
 337       return;
 338
 339    Builder bld(ctx->program, ctx->block);
 340    if (num_components == 1) {
 341       if (dst.type() == RegType::sgpr)
 342          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 343       else
 344          bld.copy(Definition(dst), vec_src);
 345       return;
 346    }
 347
 348    unsigned component_size = dst.size() / num_components;
 349    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 350
 351    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 352    vec->definitions[0] = Definition(dst);
 353    unsigned k = 0;
 354    for (unsigned i = 0; i < num_components; i++) {
 355       if (mask & (1 << i)) {
 356          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 357          if (dst.type() == RegType::sgpr)
 358             src = bld.as_uniform(src);
 359          vec->operands[i] = Operand(src);
 360       } else {
 361          vec->operands[i] = Operand(0u);
 362       }
 363       elems[i] = vec->operands[i].getTemp();
 364    }
 365    ctx->block->instructions.emplace_back(std::move(vec));
 366    ctx->allocated_vec.emplace(dst.id(), elems);
 367 }
 368
 369 /* adjust misaligned small bit size loads */
 370 void byte_align_scalar(isel_context *ctx, Temp vec, Operand offset, Temp dst)
 371 {
 372    Builder bld(ctx->program, ctx->block);
 373    Operand shift;
 374    Temp select = Temp();
 375    if (offset.isConstant()) {
 376       assert(offset.constantValue() && offset.constantValue() < 4);
 377       shift = Operand(offset.constantValue() * 8);
 378    } else {
 379       /* bit_offset = 8 * (offset & 0x3) */
 380       Temp tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), offset, Operand(3u));
 381       select = bld.tmp(s1);
 382       shift = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.scc(Definition(select)), tmp, Operand(3u));
 383    }
 384
 385    if (vec.size() == 1) {
 386       bld.sop2(aco_opcode::s_lshr_b32, Definition(dst), bld.def(s1, scc), vec, shift);
 387    } else if (vec.size() == 2) {
 388       Temp tmp = dst.size() == 2 ? dst : bld.tmp(s2);
 389       bld.sop2(aco_opcode::s_lshr_b64, Definition(tmp), bld.def(s1, scc), vec, shift);
 390       if (tmp == dst)
 391          emit_split_vector(ctx, dst, 2);
 392       else
 393          emit_extract_vector(ctx, tmp, 0, dst);
 394    } else if (vec.size() == 4) {
 395       Temp lo = bld.tmp(s2), hi = bld.tmp(s2);
 396       bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), vec);
 397       hi = bld.pseudo(aco_opcode::p_extract_vector, bld.def(s1), hi, Operand(0u));
 398       if (select != Temp())
 399          hi = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), hi, Operand(0u), select);
 400       lo = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), lo, shift);
 401       Temp mid = bld.tmp(s1);
 402       lo = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), Definition(mid), lo);
 403       hi = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), hi, shift);
 404       mid = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), hi, mid);
 405       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, mid);
 406       emit_split_vector(ctx, dst, 2);
 407    }
 408 }
 409
 410 /* this function trims subdword vectors:
 411  * if dst is vgpr - split the src and create a shrunk version according to the mask.
 412  * if dst is sgpr - split the src, but move the original to sgpr. */
 413 void trim_subdword_vector(isel_context *ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 414 {
 415    assert(vec_src.type() == RegType::vgpr);
 416    emit_split_vector(ctx, vec_src, num_components);
 417
 418    Builder bld(ctx->program, ctx->block);
 419    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 420    unsigned component_size = vec_src.bytes() / num_components;
 421    RegClass rc = RegClass(RegType::vgpr, component_size).as_subdword();
 422
 423    unsigned k = 0;
 424    for (unsigned i = 0; i < num_components; i++) {
 425       if (mask & (1 << i))
 426          elems[k++] = emit_extract_vector(ctx, vec_src, i, rc);
 427    }
 428
 429    if (dst.type() == RegType::vgpr) {
 430       assert(dst.bytes() == k * component_size);
 431       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, k, 1)};
 432       for (unsigned i = 0; i < k; i++)
 433          vec->operands[i] = Operand(elems[i]);
 434       vec->definitions[0] = Definition(dst);
 435       bld.insert(std::move(vec));
 436    } else {
 437       // TODO: alignbyte if mask doesn't start with 1?
 438       assert(mask & 1);
 439       assert(dst.size() == vec_src.size());
 440       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 441    }
 442    ctx->allocated_vec.emplace(dst.id(), elems);
 443 }
 444
 445 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 446 {
 447    Builder bld(ctx->program, ctx->block);
 448    if (!dst.id())
 449       dst = bld.tmp(bld.lm);
 450
 451    assert(val.regClass() == s1);
 452    assert(dst.regClass() == bld.lm);
 453
 454    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 455 }
 456
 457 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 458 {
 459    Builder bld(ctx->program, ctx->block);
 460    if (!dst.id())
 461       dst = bld.tmp(s1);
 462
 463    assert(val.regClass() == bld.lm);
 464    assert(dst.regClass() == s1);
 465
 466    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 467    Temp tmp = bld.tmp(s1);
 468    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 469    return emit_wqm(ctx, tmp, dst);
 470 }
 471
 472 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 473 {
 474    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 475       return get_ssa_temp(ctx, src.src.ssa);
 476
 477    if (src.src.ssa->num_components == size) {
 478       bool identity_swizzle = true;
 479       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 480          if (src.swizzle[i] != i)
 481             identity_swizzle = false;
 482       }
 483       if (identity_swizzle)
 484          return get_ssa_temp(ctx, src.src.ssa);
 485    }
 486
 487    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 488    unsigned elem_size = vec.bytes() / src.src.ssa->num_components;
 489    assert(elem_size > 0);
 490    assert(vec.bytes() % elem_size == 0);
 491
 492    if (elem_size < 4 && vec.type() == RegType::sgpr) {
 493       assert(src.src.ssa->bit_size == 8 || src.src.ssa->bit_size == 16);
 494       assert(size == 1);
 495       unsigned swizzle = src.swizzle[0];
 496       if (vec.size() > 1) {
 497          assert(src.src.ssa->bit_size == 16);
 498          vec = emit_extract_vector(ctx, vec, swizzle / 2, s1);
 499          swizzle = swizzle & 1;
 500       }
 501       if (swizzle == 0)
 502          return vec;
 503
 504       Temp dst{ctx->program->allocateId(), s1};
 505       aco_ptr<SOP2_instruction> bfe{create_instruction<SOP2_instruction>(aco_opcode::s_bfe_u32, Format::SOP2, 2, 2)};
 506       bfe->operands[0] = Operand(vec);
 507       bfe->operands[1] = Operand(uint32_t((src.src.ssa->bit_size << 16) | (src.src.ssa->bit_size * swizzle)));
 508       bfe->definitions[0] = Definition(dst);
 509       bfe->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 510       ctx->block->instructions.emplace_back(std::move(bfe));
 511       return dst;
 512    }
 513
 514    RegClass elem_rc = elem_size < 4 ? RegClass(vec.type(), elem_size).as_subdword() : RegClass(vec.type(), elem_size / 4);
 515    if (size == 1) {
 516       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 517    } else {
 518       assert(size <= 4);
 519       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 520       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 521       for (unsigned i = 0; i < size; ++i) {
 522          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 523          vec_instr->operands[i] = Operand{elems[i]};
 524       }
 525       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size / 4)};
 526       vec_instr->definitions[0] = Definition(dst);
 527       ctx->block->instructions.emplace_back(std::move(vec_instr));
 528       ctx->allocated_vec.emplace(dst.id(), elems);
 529       return dst;
 530    }
 531 }
 532
 533 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 534 {
 535    if (ptr.size() == 2)
 536       return ptr;
 537    Builder bld(ctx->program, ctx->block);
 538    if (ptr.type() == RegType::vgpr)
 539       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 540    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 541                      ptr, Operand((unsigned)ctx->options->address32_hi));
 542 }
 543
 544 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 545 {
 546    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 547    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 548    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 549    sop2->definitions[0] = Definition(dst);
 550    if (writes_scc)
 551       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 552    ctx->block->instructions.emplace_back(std::move(sop2));
 553 }
 554
 555 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 556                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 557 {
 558    Builder bld(ctx->program, ctx->block);
 559    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 560    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 561    if (src1.type() == RegType::sgpr) {
 562       if (commutative && src0.type() == RegType::vgpr) {
 563          Temp t = src0;
 564          src0 = src1;
 565          src1 = t;
 566       } else {
 567          src1 = as_vgpr(ctx, src1);
 568       }
 569    }
 570
 571    if (flush_denorms && ctx->program->chip_class < GFX9) {
 572       assert(dst.size() == 1);
 573       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 574       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 575    } else {
 576       bld.vop2(op, Definition(dst), src0, src1);
 577    }
 578 }
 579
 580 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 581                             bool flush_denorms = false)
 582 {
 583    Temp src0 = get_alu_src(ctx, instr->src[0]);
 584    Temp src1 = get_alu_src(ctx, instr->src[1]);
 585    Temp src2 = get_alu_src(ctx, instr->src[2]);
 586
 587    /* ensure that the instruction has at most 1 sgpr operand
 588     * The optimizer will inline constants for us */
 589    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 590       src0 = as_vgpr(ctx, src0);
 591    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 592       src1 = as_vgpr(ctx, src1);
 593    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 594       src2 = as_vgpr(ctx, src2);
 595
 596    Builder bld(ctx->program, ctx->block);
 597    if (flush_denorms && ctx->program->chip_class < GFX9) {
 598       assert(dst.size() == 1);
 599       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 600       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 601    } else {
 602       bld.vop3(op, Definition(dst), src0, src1, src2);
 603    }
 604 }
 605
 606 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 607 {
 608    Builder bld(ctx->program, ctx->block);
 609    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 610 }
 611
 612 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 613 {
 614    Temp src0 = get_alu_src(ctx, instr->src[0]);
 615    Temp src1 = get_alu_src(ctx, instr->src[1]);
 616    assert(src0.size() == src1.size());
 617
 618    aco_ptr<Instruction> vopc;
 619    if (src1.type() == RegType::sgpr) {
 620       if (src0.type() == RegType::vgpr) {
 621          /* to swap the operands, we might also have to change the opcode */
 622          switch (op) {
 623             case aco_opcode::v_cmp_lt_f16:
 624                op = aco_opcode::v_cmp_gt_f16;
 625                break;
 626             case aco_opcode::v_cmp_ge_f16:
 627                op = aco_opcode::v_cmp_le_f16;
 628                break;
 629             case aco_opcode::v_cmp_lt_i16:
 630                op = aco_opcode::v_cmp_gt_i16;
 631                break;
 632             case aco_opcode::v_cmp_ge_i16:
 633                op = aco_opcode::v_cmp_le_i16;
 634                break;
 635             case aco_opcode::v_cmp_lt_u16:
 636                op = aco_opcode::v_cmp_gt_u16;
 637                break;
 638             case aco_opcode::v_cmp_ge_u16:
 639                op = aco_opcode::v_cmp_le_u16;
 640                break;
 641             case aco_opcode::v_cmp_lt_f32:
 642                op = aco_opcode::v_cmp_gt_f32;
 643                break;
 644             case aco_opcode::v_cmp_ge_f32:
 645                op = aco_opcode::v_cmp_le_f32;
 646                break;
 647             case aco_opcode::v_cmp_lt_i32:
 648                op = aco_opcode::v_cmp_gt_i32;
 649                break;
 650             case aco_opcode::v_cmp_ge_i32:
 651                op = aco_opcode::v_cmp_le_i32;
 652                break;
 653             case aco_opcode::v_cmp_lt_u32:
 654                op = aco_opcode::v_cmp_gt_u32;
 655                break;
 656             case aco_opcode::v_cmp_ge_u32:
 657                op = aco_opcode::v_cmp_le_u32;
 658                break;
 659             case aco_opcode::v_cmp_lt_f64:
 660                op = aco_opcode::v_cmp_gt_f64;
 661                break;
 662             case aco_opcode::v_cmp_ge_f64:
 663                op = aco_opcode::v_cmp_le_f64;
 664                break;
 665             case aco_opcode::v_cmp_lt_i64:
 666                op = aco_opcode::v_cmp_gt_i64;
 667                break;
 668             case aco_opcode::v_cmp_ge_i64:
 669                op = aco_opcode::v_cmp_le_i64;
 670                break;
 671             case aco_opcode::v_cmp_lt_u64:
 672                op = aco_opcode::v_cmp_gt_u64;
 673                break;
 674             case aco_opcode::v_cmp_ge_u64:
 675                op = aco_opcode::v_cmp_le_u64;
 676                break;
 677             default: /* eq and ne are commutative */
 678                break;
 679          }
 680          Temp t = src0;
 681          src0 = src1;
 682          src1 = t;
 683       } else {
 684          src1 = as_vgpr(ctx, src1);
 685       }
 686    }
 687
 688    Builder bld(ctx->program, ctx->block);
 689    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 690 }
 691
 692 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 693 {
 694    Temp src0 = get_alu_src(ctx, instr->src[0]);
 695    Temp src1 = get_alu_src(ctx, instr->src[1]);
 696    Builder bld(ctx->program, ctx->block);
 697
 698    assert(dst.regClass() == bld.lm);
 699    assert(src0.type() == RegType::sgpr);
 700    assert(src1.type() == RegType::sgpr);
 701    assert(src0.regClass() == src1.regClass());
 702
 703    /* Emit the SALU comparison instruction */
 704    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 705    /* Turn the result into a per-lane bool */
 706    bool_to_vector_condition(ctx, cmp, dst);
 707 }
 708
 709 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 710                      aco_opcode v16_op, aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 711 {
 712    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : instr->src[0].src.ssa->bit_size == 32 ? s32_op : aco_opcode::num_opcodes;
 713    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : instr->src[0].src.ssa->bit_size == 32 ? v32_op : v16_op;
 714    bool use_valu = s_op == aco_opcode::num_opcodes ||
 715                    nir_dest_is_divergent(instr->dest.dest) ||
 716                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 717                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 718    aco_opcode op = use_valu ? v_op : s_op;
 719    assert(op != aco_opcode::num_opcodes);
 720    assert(dst.regClass() == ctx->program->lane_mask);
 721
 722    if (use_valu)
 723       emit_vopc_instruction(ctx, instr, op, dst);
 724    else
 725       emit_sopc_instruction(ctx, instr, op, dst);
 726 }
 727
 728 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 729 {
 730    Builder bld(ctx->program, ctx->block);
 731    Temp src0 = get_alu_src(ctx, instr->src[0]);
 732    Temp src1 = get_alu_src(ctx, instr->src[1]);
 733
 734    assert(dst.regClass() == bld.lm);
 735    assert(src0.regClass() == bld.lm);
 736    assert(src1.regClass() == bld.lm);
 737
 738    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 739 }
 740
 741 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 742 {
 743    Builder bld(ctx->program, ctx->block);
 744    Temp cond = get_alu_src(ctx, instr->src[0]);
 745    Temp then = get_alu_src(ctx, instr->src[1]);
 746    Temp els = get_alu_src(ctx, instr->src[2]);
 747
 748    assert(cond.regClass() == bld.lm);
 749
 750    if (dst.type() == RegType::vgpr) {
 751       aco_ptr<Instruction> bcsel;
 752       if (dst.size() == 1) {
 753          then = as_vgpr(ctx, then);
 754          els = as_vgpr(ctx, els);
 755
 756          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 757       } else if (dst.size() == 2) {
 758          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 759          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 760          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 761          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 762
 763          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 764          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 765
 766          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 767       } else {
 768          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 769          nir_print_instr(&instr->instr, stderr);
 770          fprintf(stderr, "\n");
 771       }
 772       return;
 773    }
 774
 775    if (instr->dest.dest.ssa.bit_size == 1) {
 776       assert(dst.regClass() == bld.lm);
 777       assert(then.regClass() == bld.lm);
 778       assert(els.regClass() == bld.lm);
 779    }
 780
 781    if (!nir_src_is_divergent(instr->src[0].src)) { /* uniform condition and values in sgpr */
 782       if (dst.regClass() == s1 || dst.regClass() == s2) {
 783          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 784          assert(dst.size() == then.size());
 785          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 786          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 787       } else {
 788          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 789          nir_print_instr(&instr->instr, stderr);
 790          fprintf(stderr, "\n");
 791       }
 792       return;
 793    }
 794
 795    /* divergent boolean bcsel
 796     * this implements bcsel on bools: dst = s0 ? s1 : s2
 797     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 798    assert(instr->dest.dest.ssa.bit_size == 1);
 799
 800    if (cond.id() != then.id())
 801       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 802
 803    if (cond.id() == els.id())
 804       bld.sop1(Builder::s_mov, Definition(dst), then);
 805    else
 806       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 807                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 808 }
 809
 810 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 811                     aco_opcode op, uint32_t undo)
 812 {
 813    /* multiply by 16777216 to handle denormals */
 814    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 815                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 816    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 817    scaled = bld.vop1(op, bld.def(v1), scaled);
 818    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 819
 820    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 821
 822    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 823 }
 824
 825 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 826 {
 827    if (ctx->block->fp_mode.denorm32 == 0) {
 828       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 829       return;
 830    }
 831
 832    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 833 }
 834
 835 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 836 {
 837    if (ctx->block->fp_mode.denorm32 == 0) {
 838       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 839       return;
 840    }
 841
 842    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 843 }
 844
 845 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 846 {
 847    if (ctx->block->fp_mode.denorm32 == 0) {
 848       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 849       return;
 850    }
 851
 852    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 853 }
 854
 855 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 856 {
 857    if (ctx->block->fp_mode.denorm32 == 0) {
 858       bld.vop1(aco_opcode::v_log_f32, dst, val);
 859       return;
 860    }
 861
 862    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 863 }
 864
 865 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 866 {
 867    if (ctx->options->chip_class >= GFX7)
 868       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 869
 870    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 871    /* TODO: create more efficient code! */
 872    if (val.type() == RegType::sgpr)
 873       val = as_vgpr(ctx, val);
 874
 875    /* Split the input value. */
 876    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 877    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 878
 879    /* Extract the exponent and compute the unbiased value. */
 880    Temp exponent = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), val_hi, Operand(20u), Operand(11u));
 881    exponent = bld.vsub32(bld.def(v1), exponent, Operand(1023u));
 882
 883    /* Extract the fractional part. */
 884    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 885    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 886
 887    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 888    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 889
 890    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 891    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 892    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 893    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 894    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 895
 896    /* Get the sign bit. */
 897    Temp sign = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x80000000u), val_hi);
 898
 899    /* Decide the operation to apply depending on the unbiased exponent. */
 900    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 901    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 902    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 903    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 904    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 905    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 906
 907    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 908 }
 909
 910 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 911 {
 912    if (ctx->options->chip_class >= GFX7)
 913       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 914
 915    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 916    Temp src0 = as_vgpr(ctx, val);
 917
 918    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 919    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 920
 921    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 922    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 923    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 924
 925    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 926    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 927    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 928    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 929
 930    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 931    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 932
 933    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 934
 935    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 936    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 937
 938    return add->definitions[0].getTemp();
 939 }
 940
 941 Temp convert_int(Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, bool is_signed, Temp dst=Temp()) {
 942    if (!dst.id()) {
 943       if (dst_bits % 32 == 0 || src.type() == RegType::sgpr)
 944          dst = bld.tmp(src.type(), DIV_ROUND_UP(dst_bits, 32u));
 945       else
 946          dst = bld.tmp(RegClass(RegType::vgpr, dst_bits / 8u).as_subdword());
 947    }
 948
 949    if (dst.bytes() == src.bytes() && dst_bits < src_bits)
 950       return bld.copy(Definition(dst), src);
 951    else if (dst.bytes() < src.bytes())
 952       return bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
 953
 954    Temp tmp = dst;
 955    if (dst_bits == 64)
 956       tmp = src_bits == 32 ? src : bld.tmp(src.type(), 1);
 957
 958    if (tmp == src) {
 959    } else if (src.regClass() == s1) {
 960       if (is_signed)
 961          bld.sop1(src_bits == 8 ? aco_opcode::s_sext_i32_i8 : aco_opcode::s_sext_i32_i16, Definition(tmp), src);
 962       else
 963          bld.sop2(aco_opcode::s_and_b32, Definition(tmp), bld.def(s1, scc), Operand(src_bits == 8 ? 0xFFu : 0xFFFFu), src);
 964    } else {
 965       assert(src_bits != 8 || src.regClass() == v1b);
 966       assert(src_bits != 16 || src.regClass() == v2b);
 967       aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
 968       sdwa->operands[0] = Operand(src);
 969       sdwa->definitions[0] = Definition(tmp);
 970       if (is_signed)
 971          sdwa->sel[0] = src_bits == 8 ? sdwa_sbyte : sdwa_sword;
 972       else
 973          sdwa->sel[0] = src_bits == 8 ? sdwa_ubyte : sdwa_uword;
 974       sdwa->dst_sel = tmp.bytes() == 2 ? sdwa_uword : sdwa_udword;
 975       bld.insert(std::move(sdwa));
 976    }
 977
 978    if (dst_bits == 64) {
 979       if (is_signed && dst.regClass() == s2) {
 980          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), tmp, Operand(31u));
 981          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, high);
 982       } else if (is_signed && dst.regClass() == v2) {
 983          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), tmp);
 984          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, high);
 985       } else {
 986          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, Operand(0u));
 987       }
 988    }
 989
 990    return dst;
 991 }
 992
 993 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 994 {
 995    if (!instr->dest.dest.is_ssa) {
 996       fprintf(stderr, "nir alu dst not in ssa: ");
 997       nir_print_instr(&instr->instr, stderr);
 998       fprintf(stderr, "\n");
 999       abort();
1000    }
1001    Builder bld(ctx->program, ctx->block);
1002    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
1003    switch(instr->op) {
1004    case nir_op_vec2:
1005    case nir_op_vec3:
1006    case nir_op_vec4: {
1007       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
1008       unsigned num = instr->dest.dest.ssa.num_components;
1009       for (unsigned i = 0; i < num; ++i)
1010          elems[i] = get_alu_src(ctx, instr->src[i]);
1011
1012       if (instr->dest.dest.ssa.bit_size >= 32 || dst.type() == RegType::vgpr) {
1013          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
1014          RegClass elem_rc = RegClass::get(RegType::vgpr, instr->dest.dest.ssa.bit_size / 8u);
1015          for (unsigned i = 0; i < num; ++i) {
1016             if (elems[i].type() == RegType::sgpr && elem_rc.is_subdword())
1017                vec->operands[i] = Operand(emit_extract_vector(ctx, elems[i], 0, elem_rc));
1018             else
1019                vec->operands[i] = Operand{elems[i]};
1020          }
1021          vec->definitions[0] = Definition(dst);
1022          ctx->block->instructions.emplace_back(std::move(vec));
1023          ctx->allocated_vec.emplace(dst.id(), elems);
1024       } else {
1025          // TODO: that is a bit suboptimal..
1026          Temp mask = bld.copy(bld.def(s1), Operand((1u << instr->dest.dest.ssa.bit_size) - 1));
1027          for (unsigned i = 0; i < num - 1; ++i)
1028             if (((i+1) * instr->dest.dest.ssa.bit_size) % 32)
1029                elems[i] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), elems[i], mask);
1030          for (unsigned i = 0; i < num; ++i) {
1031             unsigned bit = i * instr->dest.dest.ssa.bit_size;
1032             if (bit % 32 == 0) {
1033                elems[bit / 32] = elems[i];
1034             } else {
1035                elems[i] = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc),
1036                                    elems[i], Operand((i * instr->dest.dest.ssa.bit_size) % 32));
1037                elems[bit / 32] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), elems[bit / 32], elems[i]);
1038             }
1039          }
1040          if (dst.size() == 1)
1041             bld.copy(Definition(dst), elems[0]);
1042          else
1043             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), elems[0], elems[1]);
1044       }
1045       break;
1046    }
1047    case nir_op_mov: {
1048       Temp src = get_alu_src(ctx, instr->src[0]);
1049       aco_ptr<Instruction> mov;
1050       if (dst.type() == RegType::sgpr) {
1051          if (src.type() == RegType::vgpr)
1052             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
1053          else if (src.regClass() == s1)
1054             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
1055          else if (src.regClass() == s2)
1056             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
1057          else
1058             unreachable("wrong src register class for nir_op_imov");
1059       } else {
1060          if (dst.regClass() == v1)
1061             bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
1062          else if (dst.regClass() == v1b ||
1063                   dst.regClass() == v2b ||
1064                   dst.regClass() == v2)
1065             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
1066          else
1067             unreachable("wrong src register class for nir_op_imov");
1068       }
1069       break;
1070    }
1071    case nir_op_inot: {
1072       Temp src = get_alu_src(ctx, instr->src[0]);
1073       if (instr->dest.dest.ssa.bit_size == 1) {
1074          assert(src.regClass() == bld.lm);
1075          assert(dst.regClass() == bld.lm);
1076          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
1077          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
1078          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
1079       } else if (dst.regClass() == v1) {
1080          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
1081       } else if (dst.type() == RegType::sgpr) {
1082          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
1083          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
1084       } else {
1085          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1086          nir_print_instr(&instr->instr, stderr);
1087          fprintf(stderr, "\n");
1088       }
1089       break;
1090    }
1091    case nir_op_ineg: {
1092       Temp src = get_alu_src(ctx, instr->src[0]);
1093       if (dst.regClass() == v1) {
1094          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
1095       } else if (dst.regClass() == s1) {
1096          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
1097       } else if (dst.size() == 2) {
1098          Temp src0 = bld.tmp(dst.type(), 1);
1099          Temp src1 = bld.tmp(dst.type(), 1);
1100          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
1101
1102          if (dst.regClass() == s2) {
1103             Temp carry = bld.tmp(s1);
1104             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
1105             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
1106             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1107          } else {
1108             Temp lower = bld.tmp(v1);
1109             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
1110             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
1111             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1112          }
1113       } else {
1114          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1115          nir_print_instr(&instr->instr, stderr);
1116          fprintf(stderr, "\n");
1117       }
1118       break;
1119    }
1120    case nir_op_iabs: {
1121       if (dst.regClass() == s1) {
1122          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
1123       } else if (dst.regClass() == v1) {
1124          Temp src = get_alu_src(ctx, instr->src[0]);
1125          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
1126       } else {
1127          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1128          nir_print_instr(&instr->instr, stderr);
1129          fprintf(stderr, "\n");
1130       }
1131       break;
1132    }
1133    case nir_op_isign: {
1134       Temp src = get_alu_src(ctx, instr->src[0]);
1135       if (dst.regClass() == s1) {
1136          Temp tmp = bld.sop2(aco_opcode::s_max_i32, bld.def(s1), bld.def(s1, scc), src, Operand((uint32_t)-1));
1137          bld.sop2(aco_opcode::s_min_i32, Definition(dst), bld.def(s1, scc), tmp, Operand(1u));
1138       } else if (dst.regClass() == s2) {
1139          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
1140          Temp neqz;
1141          if (ctx->program->chip_class >= GFX8)
1142             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
1143          else
1144             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
1145          /* SCC gets zero-extended to 64 bit */
1146          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
1147       } else if (dst.regClass() == v1) {
1148          bld.vop3(aco_opcode::v_med3_i32, Definition(dst), Operand((uint32_t)-1), src, Operand(1u));
1149       } else if (dst.regClass() == v2) {
1150          Temp upper = emit_extract_vector(ctx, src, 1, v1);
1151          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
1152          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1153          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
1154          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
1155          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1156       } else {
1157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1158          nir_print_instr(&instr->instr, stderr);
1159          fprintf(stderr, "\n");
1160       }
1161       break;
1162    }
1163    case nir_op_imax: {
1164       if (dst.regClass() == v1) {
1165          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
1166       } else if (dst.regClass() == s1) {
1167          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
1168       } else {
1169          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1170          nir_print_instr(&instr->instr, stderr);
1171          fprintf(stderr, "\n");
1172       }
1173       break;
1174    }
1175    case nir_op_umax: {
1176       if (dst.regClass() == v1) {
1177          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
1178       } else if (dst.regClass() == s1) {
1179          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
1180       } else {
1181          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1182          nir_print_instr(&instr->instr, stderr);
1183          fprintf(stderr, "\n");
1184       }
1185       break;
1186    }
1187    case nir_op_imin: {
1188       if (dst.regClass() == v1) {
1189          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
1190       } else if (dst.regClass() == s1) {
1191          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
1192       } else {
1193          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1194          nir_print_instr(&instr->instr, stderr);
1195          fprintf(stderr, "\n");
1196       }
1197       break;
1198    }
1199    case nir_op_umin: {
1200       if (dst.regClass() == v1) {
1201          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
1202       } else if (dst.regClass() == s1) {
1203          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1204       } else {
1205          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1206          nir_print_instr(&instr->instr, stderr);
1207          fprintf(stderr, "\n");
1208       }
1209       break;
1210    }
1211    case nir_op_ior: {
1212       if (instr->dest.dest.ssa.bit_size == 1) {
1213          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1214       } else if (dst.regClass() == v1) {
1215          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1216       } else if (dst.regClass() == s1) {
1217          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1218       } else if (dst.regClass() == s2) {
1219          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1220       } else {
1221          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1222          nir_print_instr(&instr->instr, stderr);
1223          fprintf(stderr, "\n");
1224       }
1225       break;
1226    }
1227    case nir_op_iand: {
1228       if (instr->dest.dest.ssa.bit_size == 1) {
1229          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1230       } else if (dst.regClass() == v1) {
1231          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1232       } else if (dst.regClass() == s1) {
1233          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1234       } else if (dst.regClass() == s2) {
1235          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1236       } else {
1237          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1238          nir_print_instr(&instr->instr, stderr);
1239          fprintf(stderr, "\n");
1240       }
1241       break;
1242    }
1243    case nir_op_ixor: {
1244       if (instr->dest.dest.ssa.bit_size == 1) {
1245          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1246       } else if (dst.regClass() == v1) {
1247          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1248       } else if (dst.regClass() == s1) {
1249          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1250       } else if (dst.regClass() == s2) {
1251          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1252       } else {
1253          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1254          nir_print_instr(&instr->instr, stderr);
1255          fprintf(stderr, "\n");
1256       }
1257       break;
1258    }
1259    case nir_op_ushr: {
1260       if (dst.regClass() == v1) {
1261          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1262       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1263          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1264                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1265       } else if (dst.regClass() == v2) {
1266          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1267                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1268       } else if (dst.regClass() == s2) {
1269          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1270       } else if (dst.regClass() == s1) {
1271          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1272       } else {
1273          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1274          nir_print_instr(&instr->instr, stderr);
1275          fprintf(stderr, "\n");
1276       }
1277       break;
1278    }
1279    case nir_op_ishl: {
1280       if (dst.regClass() == v1) {
1281          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1282       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1283          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1284                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1285       } else if (dst.regClass() == v2) {
1286          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1287                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1288       } else if (dst.regClass() == s1) {
1289          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1290       } else if (dst.regClass() == s2) {
1291          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1292       } else {
1293          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1294          nir_print_instr(&instr->instr, stderr);
1295          fprintf(stderr, "\n");
1296       }
1297       break;
1298    }
1299    case nir_op_ishr: {
1300       if (dst.regClass() == v1) {
1301          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1302       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1303          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1304                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1305       } else if (dst.regClass() == v2) {
1306          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1307                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1308       } else if (dst.regClass() == s1) {
1309          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1310       } else if (dst.regClass() == s2) {
1311          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1312       } else {
1313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1314          nir_print_instr(&instr->instr, stderr);
1315          fprintf(stderr, "\n");
1316       }
1317       break;
1318    }
1319    case nir_op_find_lsb: {
1320       Temp src = get_alu_src(ctx, instr->src[0]);
1321       if (src.regClass() == s1) {
1322          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1323       } else if (src.regClass() == v1) {
1324          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1325       } else if (src.regClass() == s2) {
1326          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1327       } else {
1328          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1329          nir_print_instr(&instr->instr, stderr);
1330          fprintf(stderr, "\n");
1331       }
1332       break;
1333    }
1334    case nir_op_ufind_msb:
1335    case nir_op_ifind_msb: {
1336       Temp src = get_alu_src(ctx, instr->src[0]);
1337       if (src.regClass() == s1 || src.regClass() == s2) {
1338          aco_opcode op = src.regClass() == s2 ?
1339                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1340                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1341          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1342
1343          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1344                                         Operand(src.size() * 32u - 1u), msb_rev);
1345          Temp msb = sub.def(0).getTemp();
1346          Temp carry = sub.def(1).getTemp();
1347
1348          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1349       } else if (src.regClass() == v1) {
1350          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1351          Temp msb_rev = bld.tmp(v1);
1352          emit_vop1_instruction(ctx, instr, op, msb_rev);
1353          Temp msb = bld.tmp(v1);
1354          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1355          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1356       } else {
1357          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1358          nir_print_instr(&instr->instr, stderr);
1359          fprintf(stderr, "\n");
1360       }
1361       break;
1362    }
1363    case nir_op_bitfield_reverse: {
1364       if (dst.regClass() == s1) {
1365          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1366       } else if (dst.regClass() == v1) {
1367          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1368       } else {
1369          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1370          nir_print_instr(&instr->instr, stderr);
1371          fprintf(stderr, "\n");
1372       }
1373       break;
1374    }
1375    case nir_op_iadd: {
1376       if (dst.regClass() == s1) {
1377          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1378          break;
1379       }
1380
1381       Temp src0 = get_alu_src(ctx, instr->src[0]);
1382       Temp src1 = get_alu_src(ctx, instr->src[1]);
1383       if (dst.regClass() == v1) {
1384          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1385          break;
1386       }
1387
1388       assert(src0.size() == 2 && src1.size() == 2);
1389       Temp src00 = bld.tmp(src0.type(), 1);
1390       Temp src01 = bld.tmp(dst.type(), 1);
1391       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1392       Temp src10 = bld.tmp(src1.type(), 1);
1393       Temp src11 = bld.tmp(dst.type(), 1);
1394       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1395
1396       if (dst.regClass() == s2) {
1397          Temp carry = bld.tmp(s1);
1398          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1399          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1400          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1401       } else if (dst.regClass() == v2) {
1402          Temp dst0 = bld.tmp(v1);
1403          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1404          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1405          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1406       } else {
1407          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1408          nir_print_instr(&instr->instr, stderr);
1409          fprintf(stderr, "\n");
1410       }
1411       break;
1412    }
1413    case nir_op_uadd_sat: {
1414       Temp src0 = get_alu_src(ctx, instr->src[0]);
1415       Temp src1 = get_alu_src(ctx, instr->src[1]);
1416       if (dst.regClass() == s1) {
1417          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1418          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1419                   src0, src1);
1420          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1421       } else if (dst.regClass() == v1) {
1422          if (ctx->options->chip_class >= GFX9) {
1423             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1424             add->operands[0] = Operand(src0);
1425             add->operands[1] = Operand(src1);
1426             add->definitions[0] = Definition(dst);
1427             add->clamp = 1;
1428             ctx->block->instructions.emplace_back(std::move(add));
1429          } else {
1430             if (src1.regClass() != v1)
1431                std::swap(src0, src1);
1432             assert(src1.regClass() == v1);
1433             Temp tmp = bld.tmp(v1);
1434             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1435             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1436          }
1437       } else {
1438          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1439          nir_print_instr(&instr->instr, stderr);
1440          fprintf(stderr, "\n");
1441       }
1442       break;
1443    }
1444    case nir_op_uadd_carry: {
1445       Temp src0 = get_alu_src(ctx, instr->src[0]);
1446       Temp src1 = get_alu_src(ctx, instr->src[1]);
1447       if (dst.regClass() == s1) {
1448          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1449          break;
1450       }
1451       if (dst.regClass() == v1) {
1452          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1453          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1454          break;
1455       }
1456
1457       Temp src00 = bld.tmp(src0.type(), 1);
1458       Temp src01 = bld.tmp(dst.type(), 1);
1459       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1460       Temp src10 = bld.tmp(src1.type(), 1);
1461       Temp src11 = bld.tmp(dst.type(), 1);
1462       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1463       if (dst.regClass() == s2) {
1464          Temp carry = bld.tmp(s1);
1465          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1466          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1467          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1468       } else if (dst.regClass() == v2) {
1469          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1470          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1471          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1472          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1473       } else {
1474          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1475          nir_print_instr(&instr->instr, stderr);
1476          fprintf(stderr, "\n");
1477       }
1478       break;
1479    }
1480    case nir_op_isub: {
1481       if (dst.regClass() == s1) {
1482          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1483          break;
1484       }
1485
1486       Temp src0 = get_alu_src(ctx, instr->src[0]);
1487       Temp src1 = get_alu_src(ctx, instr->src[1]);
1488       if (dst.regClass() == v1) {
1489          bld.vsub32(Definition(dst), src0, src1);
1490          break;
1491       }
1492
1493       Temp src00 = bld.tmp(src0.type(), 1);
1494       Temp src01 = bld.tmp(dst.type(), 1);
1495       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1496       Temp src10 = bld.tmp(src1.type(), 1);
1497       Temp src11 = bld.tmp(dst.type(), 1);
1498       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1499       if (dst.regClass() == s2) {
1500          Temp carry = bld.tmp(s1);
1501          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1502          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1503          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1504       } else if (dst.regClass() == v2) {
1505          Temp lower = bld.tmp(v1);
1506          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1507          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1508          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1509       } else {
1510          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1511          nir_print_instr(&instr->instr, stderr);
1512          fprintf(stderr, "\n");
1513       }
1514       break;
1515    }
1516    case nir_op_usub_borrow: {
1517       Temp src0 = get_alu_src(ctx, instr->src[0]);
1518       Temp src1 = get_alu_src(ctx, instr->src[1]);
1519       if (dst.regClass() == s1) {
1520          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1521          break;
1522       } else if (dst.regClass() == v1) {
1523          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1524          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1525          break;
1526       }
1527
1528       Temp src00 = bld.tmp(src0.type(), 1);
1529       Temp src01 = bld.tmp(dst.type(), 1);
1530       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1531       Temp src10 = bld.tmp(src1.type(), 1);
1532       Temp src11 = bld.tmp(dst.type(), 1);
1533       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1534       if (dst.regClass() == s2) {
1535          Temp borrow = bld.tmp(s1);
1536          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1537          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1538          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1539       } else if (dst.regClass() == v2) {
1540          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1541          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1542          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1543          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_imul: {
1552       if (dst.regClass() == v1) {
1553          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1554                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1555       } else if (dst.regClass() == s1) {
1556          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1557       } else {
1558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1559          nir_print_instr(&instr->instr, stderr);
1560          fprintf(stderr, "\n");
1561       }
1562       break;
1563    }
1564    case nir_op_umul_high: {
1565       if (dst.regClass() == v1) {
1566          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1567       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1568          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1569       } else if (dst.regClass() == s1) {
1570          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1571                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1572          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1573       } else {
1574          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1575          nir_print_instr(&instr->instr, stderr);
1576          fprintf(stderr, "\n");
1577       }
1578       break;
1579    }
1580    case nir_op_imul_high: {
1581       if (dst.regClass() == v1) {
1582          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1583       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1584          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1585       } else if (dst.regClass() == s1) {
1586          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1587                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1588          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_fmul: {
1597       Temp src0 = get_alu_src(ctx, instr->src[0]);
1598       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1599       if (dst.regClass() == v2b) {
1600          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f16, dst, true);
1601       } else if (dst.regClass() == v1) {
1602          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1603       } else if (dst.regClass() == v2) {
1604          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), src0, src1);
1605       } else {
1606          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1607          nir_print_instr(&instr->instr, stderr);
1608          fprintf(stderr, "\n");
1609       }
1610       break;
1611    }
1612    case nir_op_fadd: {
1613       Temp src0 = get_alu_src(ctx, instr->src[0]);
1614       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1615       if (dst.regClass() == v2b) {
1616          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f16, dst, true);
1617       } else if (dst.regClass() == v1) {
1618          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1619       } else if (dst.regClass() == v2) {
1620          bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, src1);
1621       } else {
1622          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1623          nir_print_instr(&instr->instr, stderr);
1624          fprintf(stderr, "\n");
1625       }
1626       break;
1627    }
1628    case nir_op_fsub: {
1629       Temp src0 = get_alu_src(ctx, instr->src[0]);
1630       Temp src1 = get_alu_src(ctx, instr->src[1]);
1631       if (dst.regClass() == v2b) {
1632          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1633             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f16, dst, false);
1634          else
1635             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f16, dst, true);
1636       } else if (dst.regClass() == v1) {
1637          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1638             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1639          else
1640             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1641       } else if (dst.regClass() == v2) {
1642          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1643                                      as_vgpr(ctx, src0), as_vgpr(ctx, src1));
1644          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1645          sub->neg[1] = true;
1646       } else {
1647          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1648          nir_print_instr(&instr->instr, stderr);
1649          fprintf(stderr, "\n");
1650       }
1651       break;
1652    }
1653    case nir_op_fmax: {
1654       Temp src0 = get_alu_src(ctx, instr->src[0]);
1655       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1656       if (dst.regClass() == v2b) {
1657          // TODO: check fp_mode.must_flush_denorms16_64
1658          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f16, dst, true);
1659       } else if (dst.regClass() == v1) {
1660          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1661       } else if (dst.regClass() == v2) {
1662          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1663             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2), src0, src1);
1664             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1665          } else {
1666             bld.vop3(aco_opcode::v_max_f64, Definition(dst), src0, src1);
1667          }
1668       } else {
1669          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1670          nir_print_instr(&instr->instr, stderr);
1671          fprintf(stderr, "\n");
1672       }
1673       break;
1674    }
1675    case nir_op_fmin: {
1676       Temp src0 = get_alu_src(ctx, instr->src[0]);
1677       Temp src1 = as_vgpr(ctx, get_alu_src(ctx, instr->src[1]));
1678       if (dst.regClass() == v2b) {
1679          // TODO: check fp_mode.must_flush_denorms16_64
1680          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f16, dst, true);
1681       } else if (dst.regClass() == v1) {
1682          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1683       } else if (dst.regClass() == v2) {
1684          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1685             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), src0, src1);
1686             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1687          } else {
1688             bld.vop3(aco_opcode::v_min_f64, Definition(dst), src0, src1);
1689          }
1690       } else {
1691          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1692          nir_print_instr(&instr->instr, stderr);
1693          fprintf(stderr, "\n");
1694       }
1695       break;
1696    }
1697    case nir_op_fmax3: {
1698       if (dst.regClass() == v2b) {
1699          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f16, dst, false);
1700       } else if (dst.regClass() == v1) {
1701          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1702       } else {
1703          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1704          nir_print_instr(&instr->instr, stderr);
1705          fprintf(stderr, "\n");
1706       }
1707       break;
1708    }
1709    case nir_op_fmin3: {
1710       if (dst.regClass() == v2b) {
1711          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f16, dst, false);
1712       } else if (dst.regClass() == v1) {
1713          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1714       } else {
1715          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1716          nir_print_instr(&instr->instr, stderr);
1717          fprintf(stderr, "\n");
1718       }
1719       break;
1720    }
1721    case nir_op_fmed3: {
1722       if (dst.regClass() == v2b) {
1723          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f16, dst, false);
1724       } else if (dst.regClass() == v1) {
1725          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1726       } else {
1727          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1728          nir_print_instr(&instr->instr, stderr);
1729          fprintf(stderr, "\n");
1730       }
1731       break;
1732    }
1733    case nir_op_umax3: {
1734       if (dst.size() == 1) {
1735          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1736       } else {
1737          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1738          nir_print_instr(&instr->instr, stderr);
1739          fprintf(stderr, "\n");
1740       }
1741       break;
1742    }
1743    case nir_op_umin3: {
1744       if (dst.size() == 1) {
1745          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1746       } else {
1747          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1748          nir_print_instr(&instr->instr, stderr);
1749          fprintf(stderr, "\n");
1750       }
1751       break;
1752    }
1753    case nir_op_umed3: {
1754       if (dst.size() == 1) {
1755          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1756       } else {
1757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1758          nir_print_instr(&instr->instr, stderr);
1759          fprintf(stderr, "\n");
1760       }
1761       break;
1762    }
1763    case nir_op_imax3: {
1764       if (dst.size() == 1) {
1765          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1766       } else {
1767          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1768          nir_print_instr(&instr->instr, stderr);
1769          fprintf(stderr, "\n");
1770       }
1771       break;
1772    }
1773    case nir_op_imin3: {
1774       if (dst.size() == 1) {
1775          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1776       } else {
1777          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1778          nir_print_instr(&instr->instr, stderr);
1779          fprintf(stderr, "\n");
1780       }
1781       break;
1782    }
1783    case nir_op_imed3: {
1784       if (dst.size() == 1) {
1785          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1786       } else {
1787          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1788          nir_print_instr(&instr->instr, stderr);
1789          fprintf(stderr, "\n");
1790       }
1791       break;
1792    }
1793    case nir_op_cube_face_coord: {
1794       Temp in = get_alu_src(ctx, instr->src[0], 3);
1795       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1796                       emit_extract_vector(ctx, in, 1, v1),
1797                       emit_extract_vector(ctx, in, 2, v1) };
1798       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1799       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1800       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1801       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1802       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1803       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1804       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1805       break;
1806    }
1807    case nir_op_cube_face_index: {
1808       Temp in = get_alu_src(ctx, instr->src[0], 3);
1809       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1810                       emit_extract_vector(ctx, in, 1, v1),
1811                       emit_extract_vector(ctx, in, 2, v1) };
1812       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1813       break;
1814    }
1815    case nir_op_bcsel: {
1816       emit_bcsel(ctx, instr, dst);
1817       break;
1818    }
1819    case nir_op_frsq: {
1820       Temp src = get_alu_src(ctx, instr->src[0]);
1821       if (dst.regClass() == v2b) {
1822          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f16, dst);
1823       } else if (dst.regClass() == v1) {
1824          emit_rsq(ctx, bld, Definition(dst), src);
1825       } else if (dst.regClass() == v2) {
1826          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1827       } else {
1828          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1829          nir_print_instr(&instr->instr, stderr);
1830          fprintf(stderr, "\n");
1831       }
1832       break;
1833    }
1834    case nir_op_fneg: {
1835       Temp src = get_alu_src(ctx, instr->src[0]);
1836       if (dst.regClass() == v2b) {
1837          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x8000u), as_vgpr(ctx, src));
1838       } else if (dst.regClass() == v1) {
1839          if (ctx->block->fp_mode.must_flush_denorms32)
1840             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1841          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1842       } else if (dst.regClass() == v2) {
1843          if (ctx->block->fp_mode.must_flush_denorms16_64)
1844             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1845          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1846          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1847          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1848          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1849       } else {
1850          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1851          nir_print_instr(&instr->instr, stderr);
1852          fprintf(stderr, "\n");
1853       }
1854       break;
1855    }
1856    case nir_op_fabs: {
1857       Temp src = get_alu_src(ctx, instr->src[0]);
1858       if (dst.regClass() == v2b) {
1859          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFu), as_vgpr(ctx, src));
1860       } else if (dst.regClass() == v1) {
1861          if (ctx->block->fp_mode.must_flush_denorms32)
1862             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1863          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1864       } else if (dst.regClass() == v2) {
1865          if (ctx->block->fp_mode.must_flush_denorms16_64)
1866             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1867          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1868          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1869          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1870          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1871       } else {
1872          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1873          nir_print_instr(&instr->instr, stderr);
1874          fprintf(stderr, "\n");
1875       }
1876       break;
1877    }
1878    case nir_op_fsat: {
1879       Temp src = get_alu_src(ctx, instr->src[0]);
1880       if (dst.regClass() == v2b) {
1881          bld.vop3(aco_opcode::v_med3_f16, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1882       } else if (dst.regClass() == v1) {
1883          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1884          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1885          // TODO: confirm that this holds under any circumstances
1886       } else if (dst.regClass() == v2) {
1887          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1888          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1889          vop3->clamp = true;
1890       } else {
1891          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1892          nir_print_instr(&instr->instr, stderr);
1893          fprintf(stderr, "\n");
1894       }
1895       break;
1896    }
1897    case nir_op_flog2: {
1898       Temp src = get_alu_src(ctx, instr->src[0]);
1899       if (dst.regClass() == v2b) {
1900          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f16, dst);
1901       } else if (dst.regClass() == v1) {
1902          emit_log2(ctx, bld, Definition(dst), src);
1903       } else {
1904          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1905          nir_print_instr(&instr->instr, stderr);
1906          fprintf(stderr, "\n");
1907       }
1908       break;
1909    }
1910    case nir_op_frcp: {
1911       Temp src = get_alu_src(ctx, instr->src[0]);
1912       if (dst.regClass() == v2b) {
1913          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f16, dst);
1914       } else if (dst.regClass() == v1) {
1915          emit_rcp(ctx, bld, Definition(dst), src);
1916       } else if (dst.regClass() == v2) {
1917          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1918       } else {
1919          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1920          nir_print_instr(&instr->instr, stderr);
1921          fprintf(stderr, "\n");
1922       }
1923       break;
1924    }
1925    case nir_op_fexp2: {
1926       if (dst.regClass() == v2b) {
1927          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f16, dst);
1928       } else if (dst.regClass() == v1) {
1929          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1930       } else {
1931          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1932          nir_print_instr(&instr->instr, stderr);
1933          fprintf(stderr, "\n");
1934       }
1935       break;
1936    }
1937    case nir_op_fsqrt: {
1938       Temp src = get_alu_src(ctx, instr->src[0]);
1939       if (dst.regClass() == v2b) {
1940          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f16, dst);
1941       } else if (dst.regClass() == v1) {
1942          emit_sqrt(ctx, bld, Definition(dst), src);
1943       } else if (dst.regClass() == v2) {
1944          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1945       } else {
1946          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1947          nir_print_instr(&instr->instr, stderr);
1948          fprintf(stderr, "\n");
1949       }
1950       break;
1951    }
1952    case nir_op_ffract: {
1953       if (dst.regClass() == v2b) {
1954          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f16, dst);
1955       } else if (dst.regClass() == v1) {
1956          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1957       } else if (dst.regClass() == v2) {
1958          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1959       } else {
1960          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1961          nir_print_instr(&instr->instr, stderr);
1962          fprintf(stderr, "\n");
1963       }
1964       break;
1965    }
1966    case nir_op_ffloor: {
1967       Temp src = get_alu_src(ctx, instr->src[0]);
1968       if (dst.regClass() == v2b) {
1969          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f16, dst);
1970       } else if (dst.regClass() == v1) {
1971          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1972       } else if (dst.regClass() == v2) {
1973          emit_floor_f64(ctx, bld, Definition(dst), src);
1974       } else {
1975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1976          nir_print_instr(&instr->instr, stderr);
1977          fprintf(stderr, "\n");
1978       }
1979       break;
1980    }
1981    case nir_op_fceil: {
1982       Temp src0 = get_alu_src(ctx, instr->src[0]);
1983       if (dst.regClass() == v2b) {
1984          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f16, dst);
1985       } else if (dst.regClass() == v1) {
1986          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1987       } else if (dst.regClass() == v2) {
1988          if (ctx->options->chip_class >= GFX7) {
1989             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1990          } else {
1991             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1992             /* trunc = trunc(src0)
1993              * if (src0 > 0.0 && src0 != trunc)
1994              *    trunc += 1.0
1995              */
1996             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1997             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1998             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1999             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
2000             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
2001             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
2002             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
2003          }
2004       } else {
2005          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2006          nir_print_instr(&instr->instr, stderr);
2007          fprintf(stderr, "\n");
2008       }
2009       break;
2010    }
2011    case nir_op_ftrunc: {
2012       Temp src = get_alu_src(ctx, instr->src[0]);
2013       if (dst.regClass() == v2b) {
2014          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f16, dst);
2015       } else if (dst.regClass() == v1) {
2016          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
2017       } else if (dst.regClass() == v2) {
2018          emit_trunc_f64(ctx, bld, Definition(dst), src);
2019       } else {
2020          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2021          nir_print_instr(&instr->instr, stderr);
2022          fprintf(stderr, "\n");
2023       }
2024       break;
2025    }
2026    case nir_op_fround_even: {
2027       Temp src0 = get_alu_src(ctx, instr->src[0]);
2028       if (dst.regClass() == v2b) {
2029          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f16, dst);
2030       } else if (dst.regClass() == v1) {
2031          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
2032       } else if (dst.regClass() == v2) {
2033          if (ctx->options->chip_class >= GFX7) {
2034             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
2035          } else {
2036             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
2037             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
2038             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
2039
2040             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
2041             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
2042             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
2043             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
2044             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
2045             tmp = sub->definitions[0].getTemp();
2046
2047             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
2048             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
2049             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2050             Temp cond = vop3->definitions[0].getTemp();
2051
2052             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
2053             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
2054             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
2055             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
2056
2057             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
2058          }
2059       } else {
2060          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2061          nir_print_instr(&instr->instr, stderr);
2062          fprintf(stderr, "\n");
2063       }
2064       break;
2065    }
2066    case nir_op_fsin:
2067    case nir_op_fcos: {
2068       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2069       aco_ptr<Instruction> norm;
2070       Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
2071       if (dst.regClass() == v2b) {
2072          Temp tmp = bld.vop2(aco_opcode::v_mul_f16, bld.def(v1), half_pi, src);
2073          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f16 : aco_opcode::v_cos_f16;
2074          bld.vop1(opcode, Definition(dst), tmp);
2075       } else if (dst.regClass() == v1) {
2076          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
2077
2078          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
2079          if (ctx->options->chip_class < GFX9)
2080             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
2081
2082          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
2083          bld.vop1(opcode, Definition(dst), tmp);
2084       } else {
2085          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2086          nir_print_instr(&instr->instr, stderr);
2087          fprintf(stderr, "\n");
2088       }
2089       break;
2090    }
2091    case nir_op_ldexp: {
2092       Temp src0 = get_alu_src(ctx, instr->src[0]);
2093       Temp src1 = get_alu_src(ctx, instr->src[1]);
2094       if (dst.regClass() == v2b) {
2095          emit_vop2_instruction(ctx, instr, aco_opcode::v_ldexp_f16, dst, false);
2096       } else if (dst.regClass() == v1) {
2097          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst), as_vgpr(ctx, src0), src1);
2098       } else if (dst.regClass() == v2) {
2099          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst), as_vgpr(ctx, src0), src1);
2100       } else {
2101          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2102          nir_print_instr(&instr->instr, stderr);
2103          fprintf(stderr, "\n");
2104       }
2105       break;
2106    }
2107    case nir_op_frexp_sig: {
2108       Temp src = get_alu_src(ctx, instr->src[0]);
2109       if (dst.regClass() == v2b) {
2110          bld.vop1(aco_opcode::v_frexp_mant_f16, Definition(dst), src);
2111       } else if (dst.regClass() == v1) {
2112          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst), src);
2113       } else if (dst.regClass() == v2) {
2114          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst), src);
2115       } else {
2116          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2117          nir_print_instr(&instr->instr, stderr);
2118          fprintf(stderr, "\n");
2119       }
2120       break;
2121    }
2122    case nir_op_frexp_exp: {
2123       Temp src = get_alu_src(ctx, instr->src[0]);
2124       if (instr->src[0].src.ssa->bit_size == 16) {
2125          Temp tmp = bld.vop1(aco_opcode::v_frexp_exp_i16_f16, bld.def(v1), src);
2126          tmp = bld.pseudo(aco_opcode::p_extract_vector, bld.def(v1b), tmp, Operand(0u));
2127          convert_int(bld, tmp, 8, 32, true, dst);
2128       } else if (instr->src[0].src.ssa->bit_size == 32) {
2129          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst), src);
2130       } else if (instr->src[0].src.ssa->bit_size == 64) {
2131          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst), src);
2132       } else {
2133          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2134          nir_print_instr(&instr->instr, stderr);
2135          fprintf(stderr, "\n");
2136       }
2137       break;
2138    }
2139    case nir_op_fsign: {
2140       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
2141       if (dst.regClass() == v2b) {
2142          Temp one = bld.copy(bld.def(v1), Operand(0x3c00u));
2143          Temp minus_one = bld.copy(bld.def(v1), Operand(0xbc00u));
2144          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f16, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2145          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), one, src, cond);
2146          cond = bld.vopc(aco_opcode::v_cmp_le_f16, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2147          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), minus_one, src, cond);
2148       } else if (dst.regClass() == v1) {
2149          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2150          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
2151          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2152          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
2153       } else if (dst.regClass() == v2) {
2154          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2155          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
2156          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
2157
2158          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
2159          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
2160          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
2161
2162          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2163       } else {
2164          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2165          nir_print_instr(&instr->instr, stderr);
2166          fprintf(stderr, "\n");
2167       }
2168       break;
2169    }
2170    case nir_op_f2f16:
2171    case nir_op_f2f16_rtne: {
2172       Temp src = get_alu_src(ctx, instr->src[0]);
2173       if (instr->src[0].src.ssa->bit_size == 64)
2174          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2175       bld.vop1(aco_opcode::v_cvt_f16_f32, Definition(dst), src);
2176       break;
2177    }
2178    case nir_op_f2f16_rtz: {
2179       Temp src = get_alu_src(ctx, instr->src[0]);
2180       if (instr->src[0].src.ssa->bit_size == 64)
2181          src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src);
2182       bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src, Operand(0u));
2183       break;
2184    }
2185    case nir_op_f2f32: {
2186       if (instr->src[0].src.ssa->bit_size == 16) {
2187          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f16, dst);
2188       } else if (instr->src[0].src.ssa->bit_size == 64) {
2189          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
2190       } else {
2191          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2192          nir_print_instr(&instr->instr, stderr);
2193          fprintf(stderr, "\n");
2194       }
2195       break;
2196    }
2197    case nir_op_f2f64: {
2198       Temp src = get_alu_src(ctx, instr->src[0]);
2199       if (instr->src[0].src.ssa->bit_size == 16)
2200          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2201       bld.vop1(aco_opcode::v_cvt_f64_f32, Definition(dst), src);
2202       break;
2203    }
2204    case nir_op_i2f16: {
2205       assert(dst.regClass() == v2b);
2206       Temp src = get_alu_src(ctx, instr->src[0]);
2207       if (instr->src[0].src.ssa->bit_size == 8)
2208          src = convert_int(bld, src, 8, 16, true);
2209       bld.vop1(aco_opcode::v_cvt_f16_i16, Definition(dst), src);
2210       break;
2211    }
2212    case nir_op_i2f32: {
2213       assert(dst.size() == 1);
2214       Temp src = get_alu_src(ctx, instr->src[0]);
2215       if (instr->src[0].src.ssa->bit_size <= 16)
2216          src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2217       bld.vop1(aco_opcode::v_cvt_f32_i32, Definition(dst), src);
2218       break;
2219    }
2220    case nir_op_i2f64: {
2221       if (instr->src[0].src.ssa->bit_size <= 32) {
2222          Temp src = get_alu_src(ctx, instr->src[0]);
2223          if (instr->src[0].src.ssa->bit_size <= 16)
2224             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2225          bld.vop1(aco_opcode::v_cvt_f64_i32, Definition(dst), src);
2226       } else if (instr->src[0].src.ssa->bit_size == 64) {
2227          Temp src = get_alu_src(ctx, instr->src[0]);
2228          RegClass rc = RegClass(src.type(), 1);
2229          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2230          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2231          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2232          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
2233          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2234          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2235
2236       } else {
2237          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2238          nir_print_instr(&instr->instr, stderr);
2239          fprintf(stderr, "\n");
2240       }
2241       break;
2242    }
2243    case nir_op_u2f16: {
2244       assert(dst.regClass() == v2b);
2245       Temp src = get_alu_src(ctx, instr->src[0]);
2246       if (instr->src[0].src.ssa->bit_size == 8)
2247          src = convert_int(bld, src, 8, 16, false);
2248       bld.vop1(aco_opcode::v_cvt_f16_u16, Definition(dst), src);
2249       break;
2250    }
2251    case nir_op_u2f32: {
2252       assert(dst.size() == 1);
2253       Temp src = get_alu_src(ctx, instr->src[0]);
2254       if (instr->src[0].src.ssa->bit_size == 8) {
2255          //TODO: we should use v_cvt_f32_ubyte1/v_cvt_f32_ubyte2/etc depending on the register assignment
2256          bld.vop1(aco_opcode::v_cvt_f32_ubyte0, Definition(dst), src);
2257       } else {
2258          if (instr->src[0].src.ssa->bit_size == 16)
2259             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true);
2260          bld.vop1(aco_opcode::v_cvt_f32_u32, Definition(dst), src);
2261       }
2262       break;
2263    }
2264    case nir_op_u2f64: {
2265       if (instr->src[0].src.ssa->bit_size <= 32) {
2266          Temp src = get_alu_src(ctx, instr->src[0]);
2267          if (instr->src[0].src.ssa->bit_size <= 16)
2268             src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, false);
2269          bld.vop1(aco_opcode::v_cvt_f64_u32, Definition(dst), src);
2270       } else if (instr->src[0].src.ssa->bit_size == 64) {
2271          Temp src = get_alu_src(ctx, instr->src[0]);
2272          RegClass rc = RegClass(src.type(), 1);
2273          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
2274          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
2275          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
2276          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
2277          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
2278          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
2279       } else {
2280          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2281          nir_print_instr(&instr->instr, stderr);
2282          fprintf(stderr, "\n");
2283       }
2284       break;
2285    }
2286    case nir_op_f2i8:
2287    case nir_op_f2i16: {
2288       Temp src = get_alu_src(ctx, instr->src[0]);
2289       if (instr->src[0].src.ssa->bit_size == 16)
2290          src = bld.vop1(aco_opcode::v_cvt_i16_f16, bld.def(v1), src);
2291       else if (instr->src[0].src.ssa->bit_size == 32)
2292          src = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src);
2293       else
2294          src = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src);
2295
2296       if (dst.type() == RegType::vgpr)
2297          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
2298       else
2299          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2300       break;
2301    }
2302    case nir_op_f2u8:
2303    case nir_op_f2u16: {
2304       Temp src = get_alu_src(ctx, instr->src[0]);
2305       if (instr->src[0].src.ssa->bit_size == 16)
2306          src = bld.vop1(aco_opcode::v_cvt_u16_f16, bld.def(v1), src);
2307       else if (instr->src[0].src.ssa->bit_size == 32)
2308          src = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src);
2309       else
2310          src = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src);
2311
2312       if (dst.type() == RegType::vgpr)
2313          bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u));
2314       else
2315          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
2316       break;
2317    }
2318    case nir_op_f2i32: {
2319       Temp src = get_alu_src(ctx, instr->src[0]);
2320       if (instr->src[0].src.ssa->bit_size == 16) {
2321          Temp tmp = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2322          if (dst.type() == RegType::vgpr) {
2323             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), tmp);
2324          } else {
2325             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2326                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), tmp));
2327          }
2328       } else if (instr->src[0].src.ssa->bit_size == 32) {
2329          if (dst.type() == RegType::vgpr)
2330             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
2331          else
2332             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2333                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
2334
2335       } else if (instr->src[0].src.ssa->bit_size == 64) {
2336          if (dst.type() == RegType::vgpr)
2337             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
2338          else
2339             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2340                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
2341
2342       } else {
2343          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2344          nir_print_instr(&instr->instr, stderr);
2345          fprintf(stderr, "\n");
2346       }
2347       break;
2348    }
2349    case nir_op_f2u32: {
2350       Temp src = get_alu_src(ctx, instr->src[0]);
2351       if (instr->src[0].src.ssa->bit_size == 16) {
2352          Temp tmp = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2353          if (dst.type() == RegType::vgpr) {
2354             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), tmp);
2355          } else {
2356             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2357                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), tmp));
2358          }
2359       } else if (instr->src[0].src.ssa->bit_size == 32) {
2360          if (dst.type() == RegType::vgpr)
2361             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2362          else
2363             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2364                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2365
2366       } else if (instr->src[0].src.ssa->bit_size == 64) {
2367          if (dst.type() == RegType::vgpr)
2368             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2369          else
2370             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2371                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2372
2373       } else {
2374          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2375          nir_print_instr(&instr->instr, stderr);
2376          fprintf(stderr, "\n");
2377       }
2378       break;
2379    }
2380    case nir_op_f2i64: {
2381       Temp src = get_alu_src(ctx, instr->src[0]);
2382       if (instr->src[0].src.ssa->bit_size == 16)
2383          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2384
2385       if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::vgpr) {
2386          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2387          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2388          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2389          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2390          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2391          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2392          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2393          Temp new_exponent = bld.tmp(v1);
2394          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2395          if (ctx->program->chip_class >= GFX8)
2396             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2397          else
2398             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2399          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2400          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2401          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2402          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2403          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2404          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2405          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2406          Temp new_lower = bld.tmp(v1);
2407          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2408          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2409          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2410
2411       } else if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::sgpr) {
2412          if (src.type() == RegType::vgpr)
2413             src = bld.as_uniform(src);
2414          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2415          exponent = bld.sop2(aco_opcode::s_sub_i32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2416          exponent = bld.sop2(aco_opcode::s_max_i32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2417          exponent = bld.sop2(aco_opcode::s_min_i32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2418          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2419          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2420          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2421          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2422          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2423          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2424          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2425          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2426          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2427          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2428          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2429          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2430          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2431          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2432          Temp borrow = bld.tmp(s1);
2433          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2434          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2435          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2436
2437       } else if (instr->src[0].src.ssa->bit_size == 64) {
2438          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2439          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2440          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2441          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2442          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2443          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2444          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2445          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2446          if (dst.type() == RegType::sgpr) {
2447             lower = bld.as_uniform(lower);
2448             upper = bld.as_uniform(upper);
2449          }
2450          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2451
2452       } else {
2453          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2454          nir_print_instr(&instr->instr, stderr);
2455          fprintf(stderr, "\n");
2456       }
2457       break;
2458    }
2459    case nir_op_f2u64: {
2460       Temp src = get_alu_src(ctx, instr->src[0]);
2461       if (instr->src[0].src.ssa->bit_size == 16)
2462          src = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src);
2463
2464       if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::vgpr) {
2465          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2466          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2467          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2468          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2469          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2470          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2471          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2472          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2473          Temp new_exponent = bld.tmp(v1);
2474          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2475          if (ctx->program->chip_class >= GFX8)
2476             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2477          else
2478             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2479          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2480          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2481          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2482          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2483          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2484          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2485          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2486
2487       } else if (instr->src[0].src.ssa->bit_size <= 32 && dst.type() == RegType::sgpr) {
2488          if (src.type() == RegType::vgpr)
2489             src = bld.as_uniform(src);
2490          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2491          exponent = bld.sop2(aco_opcode::s_sub_i32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2492          exponent = bld.sop2(aco_opcode::s_max_i32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2493          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2494          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2495          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2496          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2497          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2498          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2499          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2500          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2501          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2502          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2503          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2504          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2505          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2506          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2507          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2508
2509       } else if (instr->src[0].src.ssa->bit_size == 64) {
2510          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2511          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2512          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2513          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2514          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2515          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2516          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2517          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2518          if (dst.type() == RegType::sgpr) {
2519             lower = bld.as_uniform(lower);
2520             upper = bld.as_uniform(upper);
2521          }
2522          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2523
2524       } else {
2525          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2526          nir_print_instr(&instr->instr, stderr);
2527          fprintf(stderr, "\n");
2528       }
2529       break;
2530    }
2531    case nir_op_b2f16: {
2532       Temp src = get_alu_src(ctx, instr->src[0]);
2533       assert(src.regClass() == bld.lm);
2534
2535       if (dst.regClass() == s1) {
2536          src = bool_to_scalar_condition(ctx, src);
2537          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3c00u), src);
2538       } else if (dst.regClass() == v2b) {
2539          Temp one = bld.copy(bld.def(v1), Operand(0x3c00u));
2540          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), one, src);
2541       } else {
2542          unreachable("Wrong destination register class for nir_op_b2f16.");
2543       }
2544       break;
2545    }
2546    case nir_op_b2f32: {
2547       Temp src = get_alu_src(ctx, instr->src[0]);
2548       assert(src.regClass() == bld.lm);
2549
2550       if (dst.regClass() == s1) {
2551          src = bool_to_scalar_condition(ctx, src);
2552          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2553       } else if (dst.regClass() == v1) {
2554          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2555       } else {
2556          unreachable("Wrong destination register class for nir_op_b2f32.");
2557       }
2558       break;
2559    }
2560    case nir_op_b2f64: {
2561       Temp src = get_alu_src(ctx, instr->src[0]);
2562       assert(src.regClass() == bld.lm);
2563
2564       if (dst.regClass() == s2) {
2565          src = bool_to_scalar_condition(ctx, src);
2566          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2567       } else if (dst.regClass() == v2) {
2568          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2569          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2570          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2571       } else {
2572          unreachable("Wrong destination register class for nir_op_b2f64.");
2573       }
2574       break;
2575    }
2576    case nir_op_i2i8:
2577    case nir_op_i2i16:
2578    case nir_op_i2i32:
2579    case nir_op_i2i64: {
2580       convert_int(bld, get_alu_src(ctx, instr->src[0]),
2581                   instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, true, dst);
2582       break;
2583    }
2584    case nir_op_u2u8:
2585    case nir_op_u2u16:
2586    case nir_op_u2u32:
2587    case nir_op_u2u64: {
2588       convert_int(bld, get_alu_src(ctx, instr->src[0]),
2589                   instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, false, dst);
2590       break;
2591    }
2592    case nir_op_b2b32:
2593    case nir_op_b2i32: {
2594       Temp src = get_alu_src(ctx, instr->src[0]);
2595       assert(src.regClass() == bld.lm);
2596
2597       if (dst.regClass() == s1) {
2598          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2599          bool_to_scalar_condition(ctx, src, dst);
2600       } else if (dst.regClass() == v1) {
2601          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2602       } else {
2603          unreachable("Invalid register class for b2i32");
2604       }
2605       break;
2606    }
2607    case nir_op_b2b1:
2608    case nir_op_i2b1: {
2609       Temp src = get_alu_src(ctx, instr->src[0]);
2610       assert(dst.regClass() == bld.lm);
2611
2612       if (src.type() == RegType::vgpr) {
2613          assert(src.regClass() == v1 || src.regClass() == v2);
2614          assert(dst.regClass() == bld.lm);
2615          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2616                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2617       } else {
2618          assert(src.regClass() == s1 || src.regClass() == s2);
2619          Temp tmp;
2620          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2621             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2622          } else {
2623             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2624                            bld.scc(bld.def(s1)), Operand(0u), src);
2625          }
2626          bool_to_vector_condition(ctx, tmp, dst);
2627       }
2628       break;
2629    }
2630    case nir_op_pack_64_2x32_split: {
2631       Temp src0 = get_alu_src(ctx, instr->src[0]);
2632       Temp src1 = get_alu_src(ctx, instr->src[1]);
2633
2634       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2635       break;
2636    }
2637    case nir_op_unpack_64_2x32_split_x:
2638       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2639       break;
2640    case nir_op_unpack_64_2x32_split_y:
2641       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2642       break;
2643    case nir_op_unpack_32_2x16_split_x:
2644       if (dst.type() == RegType::vgpr) {
2645          bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2646       } else {
2647          bld.copy(Definition(dst), get_alu_src(ctx, instr->src[0]));
2648       }
2649       break;
2650    case nir_op_unpack_32_2x16_split_y:
2651       if (dst.type() == RegType::vgpr) {
2652          bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2653       } else {
2654          bld.sop2(aco_opcode::s_bfe_u32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]), Operand(uint32_t(16 << 16 | 16)));
2655       }
2656       break;
2657    case nir_op_pack_32_2x16_split: {
2658       Temp src0 = get_alu_src(ctx, instr->src[0]);
2659       Temp src1 = get_alu_src(ctx, instr->src[1]);
2660       if (dst.regClass() == v1) {
2661          src0 = emit_extract_vector(ctx, src0, 0, v2b);
2662          src1 = emit_extract_vector(ctx, src1, 0, v2b);
2663          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2664       } else {
2665          src0 = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), src0, Operand(0xFFFFu));
2666          src1 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), src1, Operand(16u));
2667          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), src0, src1);
2668       }
2669       break;
2670    }
2671    case nir_op_pack_half_2x16: {
2672       Temp src = get_alu_src(ctx, instr->src[0], 2);
2673
2674       if (dst.regClass() == v1) {
2675          Temp src0 = bld.tmp(v1);
2676          Temp src1 = bld.tmp(v1);
2677          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2678          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2679             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2680          else
2681             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2682                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2683                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2684       } else {
2685          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2686          nir_print_instr(&instr->instr, stderr);
2687          fprintf(stderr, "\n");
2688       }
2689       break;
2690    }
2691    case nir_op_unpack_half_2x16_split_x: {
2692       if (dst.regClass() == v1) {
2693          Builder bld(ctx->program, ctx->block);
2694          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2695       } else {
2696          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2697          nir_print_instr(&instr->instr, stderr);
2698          fprintf(stderr, "\n");
2699       }
2700       break;
2701    }
2702    case nir_op_unpack_half_2x16_split_y: {
2703       if (dst.regClass() == v1) {
2704          Builder bld(ctx->program, ctx->block);
2705          /* TODO: use SDWA here */
2706          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2707                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2708       } else {
2709          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2710          nir_print_instr(&instr->instr, stderr);
2711          fprintf(stderr, "\n");
2712       }
2713       break;
2714    }
2715    case nir_op_fquantize2f16: {
2716       Temp src = get_alu_src(ctx, instr->src[0]);
2717       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2718       Temp f32, cmp_res;
2719
2720       if (ctx->program->chip_class >= GFX8) {
2721          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2722          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2723          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2724       } else {
2725          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2726           * so compare the result and flush to 0 if it's smaller.
2727           */
2728          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2729          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2730          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2731          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2732          cmp_res = vop3->definitions[0].getTemp();
2733       }
2734
2735       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2736          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2737          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2738       } else {
2739          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2740       }
2741       break;
2742    }
2743    case nir_op_bfm: {
2744       Temp bits = get_alu_src(ctx, instr->src[0]);
2745       Temp offset = get_alu_src(ctx, instr->src[1]);
2746
2747       if (dst.regClass() == s1) {
2748          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2749       } else if (dst.regClass() == v1) {
2750          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2751       } else {
2752          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2753          nir_print_instr(&instr->instr, stderr);
2754          fprintf(stderr, "\n");
2755       }
2756       break;
2757    }
2758    case nir_op_bitfield_select: {
2759       /* (mask & insert) | (~mask & base) */
2760       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2761       Temp insert = get_alu_src(ctx, instr->src[1]);
2762       Temp base = get_alu_src(ctx, instr->src[2]);
2763
2764       /* dst = (insert & bitmask) | (base & ~bitmask) */
2765       if (dst.regClass() == s1) {
2766          aco_ptr<Instruction> sop2;
2767          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2768          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2769          Operand lhs;
2770          if (const_insert && const_bitmask) {
2771             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2772          } else {
2773             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2774             lhs = Operand(insert);
2775          }
2776
2777          Operand rhs;
2778          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2779          if (const_base && const_bitmask) {
2780             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2781          } else {
2782             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2783             rhs = Operand(base);
2784          }
2785
2786          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2787
2788       } else if (dst.regClass() == v1) {
2789          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2790             base = as_vgpr(ctx, base);
2791          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2792             insert = as_vgpr(ctx, insert);
2793
2794          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2795
2796       } else {
2797          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2798          nir_print_instr(&instr->instr, stderr);
2799          fprintf(stderr, "\n");
2800       }
2801       break;
2802    }
2803    case nir_op_ubfe:
2804    case nir_op_ibfe: {
2805       Temp base = get_alu_src(ctx, instr->src[0]);
2806       Temp offset = get_alu_src(ctx, instr->src[1]);
2807       Temp bits = get_alu_src(ctx, instr->src[2]);
2808
2809       if (dst.type() == RegType::sgpr) {
2810          Operand extract;
2811          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2812          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2813          if (const_offset && const_bits) {
2814             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2815             extract = Operand(const_extract);
2816          } else {
2817             Operand width;
2818             if (const_bits) {
2819                width = Operand(const_bits->u32 << 16);
2820             } else {
2821                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2822             }
2823             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2824          }
2825
2826          aco_opcode opcode;
2827          if (dst.regClass() == s1) {
2828             if (instr->op == nir_op_ubfe)
2829                opcode = aco_opcode::s_bfe_u32;
2830             else
2831                opcode = aco_opcode::s_bfe_i32;
2832          } else if (dst.regClass() == s2) {
2833             if (instr->op == nir_op_ubfe)
2834                opcode = aco_opcode::s_bfe_u64;
2835             else
2836                opcode = aco_opcode::s_bfe_i64;
2837          } else {
2838             unreachable("Unsupported BFE bit size");
2839          }
2840
2841          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2842
2843       } else {
2844          aco_opcode opcode;
2845          if (dst.regClass() == v1) {
2846             if (instr->op == nir_op_ubfe)
2847                opcode = aco_opcode::v_bfe_u32;
2848             else
2849                opcode = aco_opcode::v_bfe_i32;
2850          } else {
2851             unreachable("Unsupported BFE bit size");
2852          }
2853
2854          emit_vop3a_instruction(ctx, instr, opcode, dst);
2855       }
2856       break;
2857    }
2858    case nir_op_bit_count: {
2859       Temp src = get_alu_src(ctx, instr->src[0]);
2860       if (src.regClass() == s1) {
2861          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2862       } else if (src.regClass() == v1) {
2863          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2864       } else if (src.regClass() == v2) {
2865          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2866                   emit_extract_vector(ctx, src, 1, v1),
2867                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2868                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2869       } else if (src.regClass() == s2) {
2870          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2871       } else {
2872          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2873          nir_print_instr(&instr->instr, stderr);
2874          fprintf(stderr, "\n");
2875       }
2876       break;
2877    }
2878    case nir_op_flt: {
2879       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f16, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2880       break;
2881    }
2882    case nir_op_fge: {
2883       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f16, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2884       break;
2885    }
2886    case nir_op_feq: {
2887       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f16, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2888       break;
2889    }
2890    case nir_op_fne: {
2891       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f16, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2892       break;
2893    }
2894    case nir_op_ilt: {
2895       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i16, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2896       break;
2897    }
2898    case nir_op_ige: {
2899       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i16, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2900       break;
2901    }
2902    case nir_op_ieq: {
2903       if (instr->src[0].src.ssa->bit_size == 1)
2904          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2905       else
2906          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i16, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2907                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2908       break;
2909    }
2910    case nir_op_ine: {
2911       if (instr->src[0].src.ssa->bit_size == 1)
2912          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2913       else
2914          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i16, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2915                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2916       break;
2917    }
2918    case nir_op_ult: {
2919       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u16, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2920       break;
2921    }
2922    case nir_op_uge: {
2923       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u16, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2924       break;
2925    }
2926    case nir_op_fddx:
2927    case nir_op_fddy:
2928    case nir_op_fddx_fine:
2929    case nir_op_fddy_fine:
2930    case nir_op_fddx_coarse:
2931    case nir_op_fddy_coarse: {
2932       Temp src = get_alu_src(ctx, instr->src[0]);
2933       uint16_t dpp_ctrl1, dpp_ctrl2;
2934       if (instr->op == nir_op_fddx_fine) {
2935          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2936          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2937       } else if (instr->op == nir_op_fddy_fine) {
2938          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2939          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2940       } else {
2941          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2942          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2943             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2944          else
2945             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2946       }
2947
2948       Temp tmp;
2949       if (ctx->program->chip_class >= GFX8) {
2950          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2951          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2952       } else {
2953          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2954          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2955          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2956       }
2957       emit_wqm(ctx, tmp, dst, true);
2958       break;
2959    }
2960    default:
2961       fprintf(stderr, "Unknown NIR ALU instr: ");
2962       nir_print_instr(&instr->instr, stderr);
2963       fprintf(stderr, "\n");
2964    }
2965 }
2966
2967 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2968 {
2969    Temp dst = get_ssa_temp(ctx, &instr->def);
2970
2971    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2972    // which get truncated the lsb if double and msb if int
2973    // for now, we only use s_mov_b64 with 64bit inline constants
2974    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2975    assert(dst.type() == RegType::sgpr);
2976
2977    Builder bld(ctx->program, ctx->block);
2978
2979    if (instr->def.bit_size == 1) {
2980       assert(dst.regClass() == bld.lm);
2981       int val = instr->value[0].b ? -1 : 0;
2982       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2983       bld.sop1(Builder::s_mov, Definition(dst), op);
2984    } else if (instr->def.bit_size == 8) {
2985       /* ensure that the value is correctly represented in the low byte of the register */
2986       bld.sopk(aco_opcode::s_movk_i32, Definition(dst), instr->value[0].u8);
2987    } else if (instr->def.bit_size == 16) {
2988       /* ensure that the value is correctly represented in the low half of the register */
2989       bld.sopk(aco_opcode::s_movk_i32, Definition(dst), instr->value[0].u16);
2990    } else if (dst.size() == 1) {
2991       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2992    } else {
2993       assert(dst.size() != 1);
2994       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2995       if (instr->def.bit_size == 64)
2996          for (unsigned i = 0; i < dst.size(); i++)
2997             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2998       else {
2999          for (unsigned i = 0; i < dst.size(); i++)
3000             vec->operands[i] = Operand{instr->value[i].u32};
3001       }
3002       vec->definitions[0] = Definition(dst);
3003       ctx->block->instructions.emplace_back(std::move(vec));
3004    }
3005 }
3006
3007 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
3008 {
3009    uint32_t new_mask = 0;
3010    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
3011       if (mask & (1u << i))
3012          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
3013    return new_mask;
3014 }
3015
3016 void byte_align_vector(isel_context *ctx, Temp vec, Operand offset, Temp dst)
3017 {
3018    Builder bld(ctx->program, ctx->block);
3019    if (offset.isTemp()) {
3020       Temp tmp[3] = {vec, vec, vec};
3021
3022       if (vec.size() == 3) {
3023          tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1);
3024          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), vec);
3025       } else if (vec.size() == 2) {
3026          tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = tmp[1];
3027          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), vec);
3028       }
3029       for (unsigned i = 0; i < dst.size(); i++)
3030          tmp[i] = bld.vop3(aco_opcode::v_alignbyte_b32, bld.def(v1), tmp[i + 1], tmp[i], offset);
3031
3032       vec = tmp[0];
3033       if (dst.size() == 2)
3034          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), tmp[0], tmp[1]);
3035
3036       offset = Operand(0u);
3037    }
3038
3039    if (vec.bytes() == dst.bytes() && offset.constantValue() == 0)
3040       bld.copy(Definition(dst), vec);
3041    else
3042       trim_subdword_vector(ctx, vec, dst, vec.bytes(), ((1 << dst.bytes()) - 1) << offset.constantValue());
3043 }
3044
3045 struct LoadEmitInfo {
3046    Operand offset;
3047    Temp dst;
3048    unsigned num_components;
3049    unsigned component_size;
3050    Temp resource = Temp(0, s1);
3051    unsigned component_stride = 0;
3052    unsigned const_offset = 0;
3053    unsigned align_mul = 0;
3054    unsigned align_offset = 0;
3055
3056    bool glc = false;
3057    unsigned swizzle_component_size = 0;
3058    barrier_interaction barrier = barrier_none;
3059    bool can_reorder = true;
3060    Temp soffset = Temp(0, s1);
3061 };
3062
3063 using LoadCallback = Temp(*)(
3064    Builder& bld, const LoadEmitInfo* info, Temp offset, unsigned bytes_needed,
3065    unsigned align, unsigned const_offset, Temp dst_hint);
3066
3067 template <LoadCallback callback, bool byte_align_loads, bool supports_8bit_16bit_loads, unsigned max_const_offset_plus_one>
3068 void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info)
3069 {
3070    unsigned load_size = info->num_components * info->component_size;
3071    unsigned component_size = info->component_size;
3072
3073    unsigned num_vals = 0;
3074    Temp vals[info->dst.bytes()];
3075
3076    unsigned const_offset = info->const_offset;
3077
3078    unsigned align_mul = info->align_mul ? info->align_mul : component_size;
3079    unsigned align_offset = (info->align_offset + const_offset) % align_mul;
3080
3081    unsigned bytes_read = 0;
3082    while (bytes_read < load_size) {
3083       unsigned bytes_needed = load_size - bytes_read;
3084
3085       /* add buffer for unaligned loads */
3086       int byte_align = align_mul % 4 == 0 ? align_offset % 4 : -1;
3087
3088       if (byte_align) {
3089          if ((bytes_needed > 2 || !supports_8bit_16bit_loads) && byte_align_loads) {
3090             if (info->component_stride) {
3091                assert(supports_8bit_16bit_loads && "unimplemented");
3092                bytes_needed = 2;
3093                byte_align = 0;
3094             } else {
3095                bytes_needed += byte_align == -1 ? 4 - info->align_mul : byte_align;
3096                bytes_needed = align(bytes_needed, 4);
3097             }
3098          } else {
3099             byte_align = 0;
3100          }
3101       }
3102
3103       if (info->swizzle_component_size)
3104          bytes_needed = MIN2(bytes_needed, info->swizzle_component_size);
3105       if (info->component_stride)
3106          bytes_needed = MIN2(bytes_needed, info->component_size);
3107
3108       bool need_to_align_offset = byte_align && (align_mul % 4 || align_offset % 4);
3109
3110       /* reduce constant offset */
3111       Operand offset = info->offset;
3112       unsigned reduced_const_offset = const_offset;
3113       bool remove_const_offset_completely = need_to_align_offset;
3114       if (const_offset && (remove_const_offset_completely || const_offset >= max_const_offset_plus_one)) {
3115          unsigned to_add = const_offset;
3116          if (remove_const_offset_completely) {
3117             reduced_const_offset = 0;
3118          } else {
3119             to_add = const_offset / max_const_offset_plus_one * max_const_offset_plus_one;
3120             reduced_const_offset %= max_const_offset_plus_one;
3121          }
3122          Temp offset_tmp = offset.isTemp() ? offset.getTemp() : Temp();
3123          if (offset.isConstant()) {
3124             offset = Operand(offset.constantValue() + to_add);
3125          } else if (offset_tmp.regClass() == s1) {
3126             offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3127                               offset_tmp, Operand(to_add));
3128          } else if (offset_tmp.regClass() == v1) {
3129             offset = bld.vadd32(bld.def(v1), offset_tmp, Operand(to_add));
3130          } else {
3131             Temp lo = bld.tmp(offset_tmp.type(), 1);
3132             Temp hi = bld.tmp(offset_tmp.type(), 1);
3133             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), offset_tmp);
3134
3135             if (offset_tmp.regClass() == s2) {
3136                Temp carry = bld.tmp(s1);
3137                lo = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), lo, Operand(to_add));
3138                hi = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), hi, carry);
3139                offset = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), lo, hi);
3140             } else {
3141                Temp new_lo = bld.tmp(v1);
3142                Temp carry = bld.vadd32(Definition(new_lo), lo, Operand(to_add), true).def(1).getTemp();
3143                hi = bld.vadd32(bld.def(v1), hi, Operand(0u), false, carry);
3144                offset = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_lo, hi);
3145             }
3146          }
3147       }
3148
3149       /* align offset down if needed */
3150       Operand aligned_offset = offset;
3151       if (need_to_align_offset) {
3152          Temp offset_tmp = offset.isTemp() ? offset.getTemp() : Temp();
3153          if (offset.isConstant()) {
3154             aligned_offset = Operand(offset.constantValue() & 0xfffffffcu);
3155          } else if (offset_tmp.regClass() == s1) {
3156             aligned_offset = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfffffffcu), offset_tmp);
3157          } else if (offset_tmp.regClass() == s2) {
3158             aligned_offset = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand((uint64_t)0xfffffffffffffffcllu), offset_tmp);
3159          } else if (offset_tmp.regClass() == v1) {
3160             aligned_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xfffffffcu), offset_tmp);
3161          } else if (offset_tmp.regClass() == v2) {
3162             Temp hi = bld.tmp(v1), lo = bld.tmp(v1);
3163             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), offset_tmp);
3164             lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xfffffffcu), lo);
3165             aligned_offset = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), lo, hi);
3166          }
3167       }
3168       Temp aligned_offset_tmp = aligned_offset.isTemp() ? aligned_offset.getTemp() :
3169                                 bld.copy(bld.def(s1), aligned_offset);
3170
3171       unsigned align = align_offset ? 1 << (ffs(align_offset) - 1) : align_mul;
3172       Temp val = callback(bld, info, aligned_offset_tmp, bytes_needed, align,
3173                           reduced_const_offset, byte_align ? Temp() : info->dst);
3174
3175       /* shift result right if needed */
3176       if (byte_align) {
3177          Operand align((uint32_t)byte_align);
3178          if (byte_align == -1) {
3179             if (offset.isConstant())
3180                align = Operand(offset.constantValue() % 4u);
3181             else if (offset.size() == 2)
3182                align = Operand(emit_extract_vector(ctx, offset.getTemp(), 0, RegClass(offset.getTemp().type(), 1)));
3183             else
3184                align = offset;
3185          }
3186
3187          if (align.isTemp() || align.constantValue()) {
3188             assert(val.bytes() >= load_size && "unimplemented");
3189             Temp new_val = bld.tmp(RegClass::get(val.type(), load_size));
3190             if (val.type() == RegType::sgpr)
3191                byte_align_scalar(ctx, val, align, new_val);
3192             else
3193                byte_align_vector(ctx, val, align, new_val);
3194             val = new_val;
3195          }
3196       }
3197
3198       /* add result to list and advance */
3199       if (info->component_stride) {
3200          assert(val.bytes() == info->component_size && "unimplemented");
3201          const_offset += info->component_stride;
3202          align_offset = (align_offset + info->component_stride) % align_mul;
3203       } else {
3204          const_offset += val.bytes();
3205          align_offset = (align_offset + val.bytes()) % align_mul;
3206       }
3207       bytes_read += val.bytes();
3208       vals[num_vals++] = val;
3209    }
3210
3211    /* the callback wrote directly to dst */
3212    if (vals[0] == info->dst) {
3213       assert(num_vals == 1);
3214       emit_split_vector(ctx, info->dst, info->num_components);
3215       return;
3216    }
3217
3218    /* create array of components */
3219    unsigned components_split = 0;
3220    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
3221    bool has_vgprs = false;
3222    for (unsigned i = 0; i < num_vals;) {
3223       Temp tmp[num_vals];
3224       unsigned num_tmps = 0;
3225       unsigned tmp_size = 0;
3226       RegType reg_type = RegType::sgpr;
3227       while ((!tmp_size || (tmp_size % component_size)) && i < num_vals) {
3228          if (vals[i].type() == RegType::vgpr)
3229             reg_type = RegType::vgpr;
3230          tmp_size += vals[i].bytes();
3231          tmp[num_tmps++] = vals[i++];
3232       }
3233       if (num_tmps > 1) {
3234          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3235             aco_opcode::p_create_vector, Format::PSEUDO, num_tmps, 1)};
3236          for (unsigned i = 0; i < num_vals; i++)
3237             vec->operands[i] = Operand(tmp[i]);
3238          tmp[0] = bld.tmp(RegClass::get(reg_type, tmp_size));
3239          vec->definitions[0] = Definition(tmp[0]);
3240          bld.insert(std::move(vec));
3241       }
3242
3243       if (tmp[0].bytes() % component_size) {
3244          /* trim tmp[0] */
3245          assert(i == num_vals);
3246          RegClass new_rc = RegClass::get(reg_type, tmp[0].bytes() / component_size * component_size);
3247          tmp[0] = bld.pseudo(aco_opcode::p_extract_vector, bld.def(new_rc), tmp[0], Operand(0u));
3248       }
3249
3250       RegClass elem_rc = RegClass::get(reg_type, component_size);
3251
3252       unsigned start = components_split;
3253
3254       if (tmp_size == elem_rc.bytes()) {
3255          allocated_vec[components_split++] = tmp[0];
3256       } else {
3257          assert(tmp_size % elem_rc.bytes() == 0);
3258          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(
3259             aco_opcode::p_split_vector, Format::PSEUDO, 1, tmp_size / elem_rc.bytes())};
3260          for (unsigned i = 0; i < split->definitions.size(); i++) {
3261             Temp component = bld.tmp(elem_rc);
3262             allocated_vec[components_split++] = component;
3263             split->definitions[i] = Definition(component);
3264          }
3265          split->operands[0] = Operand(tmp[0]);
3266          bld.insert(std::move(split));
3267       }
3268
3269       /* try to p_as_uniform early so we can create more optimizable code and
3270        * also update allocated_vec */
3271       for (unsigned j = start; j < components_split; j++) {
3272          if (allocated_vec[j].bytes() % 4 == 0 && info->dst.type() == RegType::sgpr)
3273             allocated_vec[j] = bld.as_uniform(allocated_vec[j]);
3274          has_vgprs |= allocated_vec[j].type() == RegType::vgpr;
3275       }
3276    }
3277
3278    /* concatenate components and p_as_uniform() result if needed */
3279    if (info->dst.type() == RegType::vgpr || !has_vgprs)
3280       ctx->allocated_vec.emplace(info->dst.id(), allocated_vec);
3281
3282    int padding_bytes = MAX2((int)info->dst.bytes() - int(allocated_vec[0].bytes() * info->num_components), 0);
3283
3284    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3285       aco_opcode::p_create_vector, Format::PSEUDO, info->num_components + !!padding_bytes, 1)};
3286    for (unsigned i = 0; i < info->num_components; i++)
3287       vec->operands[i] = Operand(allocated_vec[i]);
3288    if (padding_bytes)
3289       vec->operands[info->num_components] = Operand(RegClass::get(RegType::vgpr, padding_bytes));
3290    if (info->dst.type() == RegType::sgpr && has_vgprs) {
3291       Temp tmp = bld.tmp(RegType::vgpr, info->dst.size());
3292       vec->definitions[0] = Definition(tmp);
3293       bld.insert(std::move(vec));
3294       bld.pseudo(aco_opcode::p_as_uniform, Definition(info->dst), tmp);
3295    } else {
3296       vec->definitions[0] = Definition(info->dst);
3297       bld.insert(std::move(vec));
3298    }
3299 }
3300
3301 Operand load_lds_size_m0(Builder& bld)
3302 {
3303    /* TODO: m0 does not need to be initialized on GFX9+ */
3304    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
3305 }
3306
3307 Temp lds_load_callback(Builder& bld, const LoadEmitInfo *info,
3308                        Temp offset, unsigned bytes_needed,
3309                        unsigned align, unsigned const_offset,
3310                        Temp dst_hint)
3311 {
3312    offset = offset.regClass() == s1 ? bld.copy(bld.def(v1), offset) : offset;
3313
3314    Operand m = load_lds_size_m0(bld);
3315
3316    bool large_ds_read = bld.program->chip_class >= GFX7;
3317    bool usable_read2 = bld.program->chip_class >= GFX7;
3318
3319    bool read2 = false;
3320    unsigned size = 0;
3321    aco_opcode op;
3322    //TODO: use ds_read_u8_d16_hi/ds_read_u16_d16_hi if beneficial
3323    if (bytes_needed >= 16 && align % 16 == 0 && large_ds_read) {
3324       size = 16;
3325       op = aco_opcode::ds_read_b128;
3326    } else if (bytes_needed >= 16 && align % 8 == 0 && const_offset % 8 == 0 && usable_read2) {
3327       size = 16;
3328       read2 = true;
3329       op = aco_opcode::ds_read2_b64;
3330    } else if (bytes_needed >= 12 && align % 16 == 0 && large_ds_read) {
3331       size = 12;
3332       op = aco_opcode::ds_read_b96;
3333    } else if (bytes_needed >= 8 && align % 8 == 0) {
3334       size = 8;
3335       op = aco_opcode::ds_read_b64;
3336    } else if (bytes_needed >= 8 && align % 4 == 0 && const_offset % 4 == 0) {
3337       size = 8;
3338       read2 = true;
3339       op = aco_opcode::ds_read2_b32;
3340    } else if (bytes_needed >= 4 && align % 4 == 0) {
3341       size = 4;
3342       op = aco_opcode::ds_read_b32;
3343    } else if (bytes_needed >= 2 && align % 2 == 0) {
3344       size = 2;
3345       op = aco_opcode::ds_read_u16;
3346    } else {
3347       size = 1;
3348       op = aco_opcode::ds_read_u8;
3349    }
3350
3351    unsigned max_offset_plus_one = read2 ? 254 * (size / 2u) + 1 : 65536;
3352    if (const_offset >= max_offset_plus_one) {
3353       offset = bld.vadd32(bld.def(v1), offset, Operand(const_offset / max_offset_plus_one));
3354       const_offset %= max_offset_plus_one;
3355    }
3356
3357    if (read2)
3358       const_offset /= (size / 2u);
3359
3360    RegClass rc = RegClass(RegType::vgpr, DIV_ROUND_UP(size, 4));
3361    Temp val = rc == info->dst.regClass() && dst_hint.id() ? dst_hint : bld.tmp(rc);
3362    if (read2)
3363       bld.ds(op, Definition(val), offset, m, const_offset, const_offset + 1);
3364    else
3365       bld.ds(op, Definition(val), offset, m, const_offset);
3366
3367    if (size < 4)
3368       val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, size)), val, Operand(0u));
3369
3370    return val;
3371 }
3372
3373 static auto emit_lds_load = emit_load<lds_load_callback, false, true, UINT32_MAX>;
3374
3375 Temp smem_load_callback(Builder& bld, const LoadEmitInfo *info,
3376                         Temp offset, unsigned bytes_needed,
3377                         unsigned align, unsigned const_offset,
3378                         Temp dst_hint)
3379 {
3380    unsigned size = 0;
3381    aco_opcode op;
3382    if (bytes_needed <= 4) {
3383       size = 1;
3384       op = info->resource.id() ? aco_opcode::s_buffer_load_dword : aco_opcode::s_load_dword;
3385    } else if (bytes_needed <= 8) {
3386       size = 2;
3387       op = info->resource.id() ? aco_opcode::s_buffer_load_dwordx2 : aco_opcode::s_load_dwordx2;
3388    } else if (bytes_needed <= 16) {
3389       size = 4;
3390       op = info->resource.id() ? aco_opcode::s_buffer_load_dwordx4 : aco_opcode::s_load_dwordx4;
3391    } else if (bytes_needed <= 32) {
3392       size = 8;
3393       op = info->resource.id() ? aco_opcode::s_buffer_load_dwordx8 : aco_opcode::s_load_dwordx8;
3394    } else {
3395       size = 16;
3396       op = info->resource.id() ? aco_opcode::s_buffer_load_dwordx16 : aco_opcode::s_load_dwordx16;
3397    }
3398    aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3399    if (info->resource.id()) {
3400       load->operands[0] = Operand(info->resource);
3401       load->operands[1] = Operand(offset);
3402    } else {
3403       load->operands[0] = Operand(offset);
3404       load->operands[1] = Operand(0u);
3405    }
3406    RegClass rc(RegType::sgpr, size);
3407    Temp val = dst_hint.id() && dst_hint.regClass() == rc ? dst_hint : bld.tmp(rc);
3408    load->definitions[0] = Definition(val);
3409    load->glc = info->glc;
3410    load->dlc = info->glc && bld.program->chip_class >= GFX10;
3411    load->barrier = info->barrier;
3412    load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3413    bld.insert(std::move(load));
3414    return val;
3415 }
3416
3417 static auto emit_smem_load = emit_load<smem_load_callback, true, false, 1024>;
3418
3419 Temp mubuf_load_callback(Builder& bld, const LoadEmitInfo *info,
3420                          Temp offset, unsigned bytes_needed,
3421                          unsigned align_, unsigned const_offset,
3422                          Temp dst_hint)
3423 {
3424    Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3425    Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3426
3427    if (info->soffset.id()) {
3428       if (soffset.isTemp())
3429          vaddr = bld.copy(bld.def(v1), soffset);
3430       soffset = Operand(info->soffset);
3431    }
3432
3433    unsigned bytes_size = 0;
3434    aco_opcode op;
3435    if (bytes_needed == 1) {
3436       bytes_size = 1;
3437       op = aco_opcode::buffer_load_ubyte;
3438    } else if (bytes_needed == 2) {
3439       bytes_size = 2;
3440       op = aco_opcode::buffer_load_ushort;
3441    } else if (bytes_needed <= 4) {
3442       bytes_size = 4;
3443       op = aco_opcode::buffer_load_dword;
3444    } else if (bytes_needed <= 8) {
3445       bytes_size = 8;
3446       op = aco_opcode::buffer_load_dwordx2;
3447    } else if (bytes_needed <= 12 && bld.program->chip_class > GFX6) {
3448       bytes_size = 12;
3449       op = aco_opcode::buffer_load_dwordx3;
3450    } else {
3451       bytes_size = 16;
3452       op = aco_opcode::buffer_load_dwordx4;
3453    }
3454    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3455    mubuf->operands[0] = Operand(info->resource);
3456    mubuf->operands[1] = vaddr;
3457    mubuf->operands[2] = soffset;
3458    mubuf->offen = (offset.type() == RegType::vgpr);
3459    mubuf->glc = info->glc;
3460    mubuf->dlc = info->glc && bld.program->chip_class >= GFX10;
3461    mubuf->barrier = info->barrier;
3462    mubuf->can_reorder = info->can_reorder;
3463    mubuf->offset = const_offset;
3464    RegClass rc = RegClass::get(RegType::vgpr, align(bytes_size, 4));
3465    Temp val = dst_hint.id() && rc == dst_hint.regClass() ? dst_hint : bld.tmp(rc);
3466    mubuf->definitions[0] = Definition(val);
3467    bld.insert(std::move(mubuf));
3468
3469    if (bytes_size < 4)
3470       val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, bytes_size)), val, Operand(0u));
3471
3472    return val;
3473 }
3474
3475 static auto emit_mubuf_load = emit_load<mubuf_load_callback, true, true, 4096>;
3476
3477 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
3478 {
3479    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3480                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3481
3482    if (addr.type() == RegType::vgpr)
3483       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
3484    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
3485 }
3486
3487 Temp global_load_callback(Builder& bld, const LoadEmitInfo *info,
3488                           Temp offset, unsigned bytes_needed,
3489                           unsigned align_, unsigned const_offset,
3490                           Temp dst_hint)
3491 {
3492    unsigned bytes_size = 0;
3493    bool mubuf = bld.program->chip_class == GFX6;
3494    bool global = bld.program->chip_class >= GFX9;
3495    aco_opcode op;
3496    if (bytes_needed == 1) {
3497       bytes_size = 1;
3498       op = mubuf ? aco_opcode::buffer_load_ubyte : global ? aco_opcode::global_load_ubyte : aco_opcode::flat_load_ubyte;
3499    } else if (bytes_needed == 2) {
3500       bytes_size = 2;
3501       op = mubuf ? aco_opcode::buffer_load_ushort : global ? aco_opcode::global_load_ushort : aco_opcode::flat_load_ushort;
3502    } else if (bytes_needed <= 4) {
3503       bytes_size = 4;
3504       op = mubuf ? aco_opcode::buffer_load_dword : global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
3505    } else if (bytes_needed <= 8) {
3506       bytes_size = 8;
3507       op = mubuf ? aco_opcode::buffer_load_dwordx2 : global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
3508    } else if (bytes_needed <= 12 && !mubuf) {
3509       bytes_size = 12;
3510       op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
3511    } else {
3512       bytes_size = 16;
3513       op = mubuf ? aco_opcode::buffer_load_dwordx4 : global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
3514    }
3515    RegClass rc = RegClass::get(RegType::vgpr, align(bytes_size, 4));
3516    Temp val = dst_hint.id() && rc == dst_hint.regClass() ? dst_hint : bld.tmp(rc);
3517    if (mubuf) {
3518       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3519       mubuf->operands[0] = Operand(get_gfx6_global_rsrc(bld, offset));
3520       mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3521       mubuf->operands[2] = Operand(0u);
3522       mubuf->glc = info->glc;
3523       mubuf->dlc = false;
3524       mubuf->offset = 0;
3525       mubuf->addr64 = offset.type() == RegType::vgpr;
3526       mubuf->disable_wqm = false;
3527       mubuf->barrier = info->barrier;
3528       mubuf->definitions[0] = Definition(val);
3529       bld.insert(std::move(mubuf));
3530    } else {
3531       offset = offset.regClass() == s2 ? bld.copy(bld.def(v2), offset) : offset;
3532
3533       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
3534       flat->operands[0] = Operand(offset);
3535       flat->operands[1] = Operand(s1);
3536       flat->glc = info->glc;
3537       flat->dlc = info->glc && bld.program->chip_class >= GFX10;
3538       flat->barrier = info->barrier;
3539       flat->offset = 0u;
3540       flat->definitions[0] = Definition(val);
3541       bld.insert(std::move(flat));
3542    }
3543
3544    if (bytes_size < 4)
3545       val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, bytes_size)), val, Operand(0u));
3546
3547    return val;
3548 }
3549
3550 static auto emit_global_load = emit_load<global_load_callback, true, true, 1>;
3551
3552 Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
3553               Temp address, unsigned base_offset, unsigned align)
3554 {
3555    assert(util_is_power_of_two_nonzero(align));
3556
3557    Builder bld(ctx->program, ctx->block);
3558
3559    unsigned num_components = dst.bytes() / elem_size_bytes;
3560    LoadEmitInfo info = {Operand(as_vgpr(ctx, address)), dst, num_components, elem_size_bytes};
3561    info.align_mul = align;
3562    info.align_offset = 0;
3563    info.barrier = barrier_shared;
3564    info.can_reorder = false;
3565    info.const_offset = base_offset;
3566    emit_lds_load(ctx, bld, &info);
3567
3568    return dst;
3569 }
3570
3571 void split_store_data(isel_context *ctx, RegType dst_type, unsigned count, Temp *dst, unsigned *offsets, Temp src)
3572 {
3573    if (!count)
3574       return;
3575
3576    Builder bld(ctx->program, ctx->block);
3577
3578    ASSERTED bool is_subdword = false;
3579    for (unsigned i = 0; i < count; i++)
3580       is_subdword |= offsets[i] % 4;
3581    is_subdword |= (src.bytes() - offsets[count - 1]) % 4;
3582    assert(!is_subdword || dst_type == RegType::vgpr);
3583
3584    /* count == 1 fast path */
3585    if (count == 1) {
3586       if (dst_type == RegType::sgpr)
3587          dst[0] = bld.as_uniform(src);
3588       else
3589          dst[0] = as_vgpr(ctx, src);
3590       return;
3591    }
3592
3593    for (unsigned i = 0; i < count - 1; i++)
3594       dst[i] = bld.tmp(RegClass::get(dst_type, offsets[i + 1] - offsets[i]));
3595    dst[count - 1] = bld.tmp(RegClass::get(dst_type, src.bytes() - offsets[count - 1]));
3596
3597    if (is_subdword && src.type() == RegType::sgpr) {
3598       src = as_vgpr(ctx, src);
3599    } else {
3600       /* use allocated_vec if possible */
3601       auto it = ctx->allocated_vec.find(src.id());
3602       if (it != ctx->allocated_vec.end()) {
3603          unsigned total_size = 0;
3604          for (unsigned i = 0; it->second[i].bytes() && (i < NIR_MAX_VEC_COMPONENTS); i++)
3605             total_size += it->second[i].bytes();
3606          if (total_size != src.bytes())
3607             goto split;
3608
3609          unsigned elem_size = it->second[0].bytes();
3610
3611          for (unsigned i = 0; i < count; i++) {
3612             if (offsets[i] % elem_size || dst[i].bytes() % elem_size)
3613                goto split;
3614          }
3615
3616          for (unsigned i = 0; i < count; i++) {
3617             unsigned start_idx = offsets[i] / elem_size;
3618             unsigned op_count = dst[i].bytes() / elem_size;
3619             if (op_count == 1) {
3620                if (dst_type == RegType::sgpr)
3621                   dst[i] = bld.as_uniform(it->second[start_idx]);
3622                else
3623                   dst[i] = as_vgpr(ctx, it->second[start_idx]);
3624                continue;
3625             }
3626
3627             aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, op_count, 1)};
3628             for (unsigned j = 0; j < op_count; j++) {
3629                Temp tmp = it->second[start_idx + j];
3630                if (dst_type == RegType::sgpr)
3631                   tmp = bld.as_uniform(tmp);
3632                vec->operands[j] = Operand(tmp);
3633             }
3634             vec->definitions[0] = Definition(dst[i]);
3635             bld.insert(std::move(vec));
3636          }
3637          return;
3638       }
3639    }
3640
3641    if (dst_type == RegType::sgpr)
3642       src = bld.as_uniform(src);
3643
3644    split:
3645    /* just split it */
3646    aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, count)};
3647    split->operands[0] = Operand(src);
3648    for (unsigned i = 0; i < count; i++)
3649       split->definitions[i] = Definition(dst[i]);
3650    bld.insert(std::move(split));
3651 }
3652
3653 bool scan_write_mask(uint32_t mask, uint32_t todo_mask,
3654                      int *start, int *count)
3655 {
3656    unsigned start_elem = ffs(todo_mask) - 1;
3657    bool skip = !(mask & (1 << start_elem));
3658    if (skip)
3659       mask = ~mask & todo_mask;
3660
3661    mask &= todo_mask;
3662
3663    u_bit_scan_consecutive_range(&mask, start, count);
3664
3665    return !skip;
3666 }
3667
3668 void advance_write_mask(uint32_t *todo_mask, int start, int count)
3669 {
3670    *todo_mask &= ~u_bit_consecutive(0, count) << start;
3671 }
3672
3673 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
3674                Temp address, unsigned base_offset, unsigned align)
3675 {
3676    assert(util_is_power_of_two_nonzero(align));
3677    assert(util_is_power_of_two_nonzero(elem_size_bytes) && elem_size_bytes <= 8);
3678
3679    Builder bld(ctx->program, ctx->block);
3680    bool large_ds_write = ctx->options->chip_class >= GFX7;
3681    bool usable_write2 = ctx->options->chip_class >= GFX7;
3682
3683    unsigned write_count = 0;
3684    Temp write_datas[32];
3685    unsigned offsets[32];
3686    aco_opcode opcodes[32];
3687
3688    wrmask = widen_mask(wrmask, elem_size_bytes);
3689
3690    uint32_t todo = u_bit_consecutive(0, data.bytes());
3691    while (todo) {
3692       int offset, bytes;
3693       if (!scan_write_mask(wrmask, todo, &offset, &bytes)) {
3694          offsets[write_count] = offset;
3695          opcodes[write_count] = aco_opcode::num_opcodes;
3696          write_count++;
3697          advance_write_mask(&todo, offset, bytes);
3698          continue;
3699       }
3700
3701       bool aligned2 = offset % 2 == 0 && align % 2 == 0;
3702       bool aligned4 = offset % 4 == 0 && align % 4 == 0;
3703       bool aligned8 = offset % 8 == 0 && align % 8 == 0;
3704       bool aligned16 = offset % 16 == 0 && align % 16 == 0;
3705
3706       //TODO: use ds_write_b8_d16_hi/ds_write_b16_d16_hi if beneficial
3707       aco_opcode op = aco_opcode::num_opcodes;
3708       if (bytes >= 16 && aligned16 && large_ds_write) {
3709          op = aco_opcode::ds_write_b128;
3710          bytes = 16;
3711       } else if (bytes >= 12 && aligned16 && large_ds_write) {
3712          op = aco_opcode::ds_write_b96;
3713          bytes = 12;
3714       } else if (bytes >= 8 && aligned8) {
3715          op = aco_opcode::ds_write_b64;
3716          bytes = 8;
3717       } else if (bytes >= 4 && aligned4) {
3718          op = aco_opcode::ds_write_b32;
3719          bytes = 4;
3720       } else if (bytes >= 2 && aligned2) {
3721          op = aco_opcode::ds_write_b16;
3722          bytes = 2;
3723       } else if (bytes >= 1) {
3724          op = aco_opcode::ds_write_b8;
3725          bytes = 1;
3726       } else {
3727          assert(false);
3728       }
3729
3730       offsets[write_count] = offset;
3731       opcodes[write_count] = op;
3732       write_count++;
3733       advance_write_mask(&todo, offset, bytes);
3734    }
3735
3736    Operand m = load_lds_size_m0(bld);
3737
3738    split_store_data(ctx, RegType::vgpr, write_count, write_datas, offsets, data);
3739
3740    for (unsigned i = 0; i < write_count; i++) {
3741       aco_opcode op = opcodes[i];
3742       if (op == aco_opcode::num_opcodes)
3743          continue;
3744
3745       Temp data = write_datas[i];
3746
3747       unsigned second = write_count;
3748       if (usable_write2 && (op == aco_opcode::ds_write_b32 || op == aco_opcode::ds_write_b64)) {
3749          for (second = i + 1; second < write_count; second++) {
3750             if (opcodes[second] == op && (offsets[second] - offsets[i]) % data.bytes() == 0) {
3751                op = data.bytes() == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
3752                opcodes[second] = aco_opcode::num_opcodes;
3753                break;
3754             }
3755          }
3756       }
3757
3758       bool write2 = op == aco_opcode::ds_write2_b32 || op == aco_opcode::ds_write2_b64;
3759       unsigned write2_off = (offsets[second] - offsets[i]) / data.bytes();
3760
3761       unsigned inline_offset = base_offset + offsets[i];
3762       unsigned max_offset = write2 ? (255 - write2_off) * data.bytes() : 65535;
3763       Temp address_offset = address;
3764       if (inline_offset > max_offset) {
3765          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3766          inline_offset = offsets[i];
3767       }
3768       assert(inline_offset <= max_offset); /* offsets[i] shouldn't be large enough for this to happen */
3769
3770       if (write2) {
3771          Temp second_data = write_datas[second];
3772          inline_offset /= data.bytes();
3773          bld.ds(op, address_offset, data, second_data, m, inline_offset, inline_offset + write2_off);
3774       } else {
3775          bld.ds(op, address_offset, data, m, inline_offset);
3776       }
3777    }
3778 }
3779
3780 unsigned calculate_lds_alignment(isel_context *ctx, unsigned const_offset)
3781 {
3782    unsigned align = 16;
3783    if (const_offset)
3784       align = std::min(align, 1u << (ffs(const_offset) - 1));
3785
3786    return align;
3787 }
3788
3789
3790 aco_opcode get_buffer_store_op(bool smem, unsigned bytes)
3791 {
3792    switch (bytes) {
3793    case 1:
3794       assert(!smem);
3795       return aco_opcode::buffer_store_byte;
3796    case 2:
3797       assert(!smem);
3798       return aco_opcode::buffer_store_short;
3799    case 4:
3800       return smem ? aco_opcode::s_buffer_store_dword : aco_opcode::buffer_store_dword;
3801    case 8:
3802       return smem ? aco_opcode::s_buffer_store_dwordx2 : aco_opcode::buffer_store_dwordx2;
3803    case 12:
3804       assert(!smem);
3805       return aco_opcode::buffer_store_dwordx3;
3806    case 16:
3807       return smem ? aco_opcode::s_buffer_store_dwordx4 : aco_opcode::buffer_store_dwordx4;
3808    }
3809    unreachable("Unexpected store size");
3810    return aco_opcode::num_opcodes;
3811 }
3812
3813 void split_buffer_store(isel_context *ctx, nir_intrinsic_instr *instr, bool smem, RegType dst_type,
3814                         Temp data, unsigned writemask, int swizzle_element_size,
3815                         unsigned *write_count, Temp *write_datas, unsigned *offsets)
3816 {
3817    unsigned write_count_with_skips = 0;
3818    bool skips[16];
3819
3820    /* determine how to split the data */
3821    unsigned todo = u_bit_consecutive(0, data.bytes());
3822    while (todo) {
3823       int offset, bytes;
3824       skips[write_count_with_skips] = !scan_write_mask(writemask, todo, &offset, &bytes);
3825       offsets[write_count_with_skips] = offset;
3826       if (skips[write_count_with_skips]) {
3827          advance_write_mask(&todo, offset, bytes);
3828          write_count_with_skips++;
3829          continue;
3830       }
3831
3832       /* only supported sizes are 1, 2, 4, 8, 12 and 16 bytes and can't be
3833        * larger than swizzle_element_size */
3834       bytes = MIN2(bytes, swizzle_element_size);
3835       if (bytes % 4)
3836          bytes = bytes > 4 ? bytes & ~0x3 : MIN2(bytes, 2);
3837
3838       /* SMEM and GFX6 VMEM can't emit 12-byte stores */
3839       if ((ctx->program->chip_class == GFX6 || smem) && bytes == 12)
3840          bytes = 8;
3841
3842       /* dword or larger stores have to be dword-aligned */
3843       unsigned align_mul = instr ? nir_intrinsic_align_mul(instr) : 4;
3844       unsigned align_offset = instr ? nir_intrinsic_align_mul(instr) : 0;
3845       bool dword_aligned = (align_offset + offset) % 4 == 0 && align_mul % 4 == 0;
3846       if (bytes >= 4 && !dword_aligned)
3847          bytes = MIN2(bytes, 2);
3848
3849       advance_write_mask(&todo, offset, bytes);
3850       write_count_with_skips++;
3851    }
3852
3853    /* actually split data */
3854    split_store_data(ctx, dst_type, write_count_with_skips, write_datas, offsets, data);
3855
3856    /* remove skips */
3857    for (unsigned i = 0; i < write_count_with_skips; i++) {
3858       if (skips[i])
3859          continue;
3860       write_datas[*write_count] = write_datas[i];
3861       offsets[*write_count] = offsets[i];
3862       (*write_count)++;
3863    }
3864 }
3865
3866 Temp create_vec_from_array(isel_context *ctx, Temp arr[], unsigned cnt, RegType reg_type, unsigned elem_size_bytes,
3867                            unsigned split_cnt = 0u, Temp dst = Temp())
3868 {
3869    Builder bld(ctx->program, ctx->block);
3870    unsigned dword_size = elem_size_bytes / 4;
3871
3872    if (!dst.id())
3873       dst = bld.tmp(RegClass(reg_type, cnt * dword_size));
3874
3875    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
3876    aco_ptr<Pseudo_instruction> instr {create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, cnt, 1)};
3877    instr->definitions[0] = Definition(dst);
3878
3879    for (unsigned i = 0; i < cnt; ++i) {
3880       if (arr[i].id()) {
3881          assert(arr[i].size() == dword_size);
3882          allocated_vec[i] = arr[i];
3883          instr->operands[i] = Operand(arr[i]);
3884       } else {
3885          Temp zero = bld.copy(bld.def(RegClass(reg_type, dword_size)), Operand(0u, dword_size == 2));
3886          allocated_vec[i] = zero;
3887          instr->operands[i] = Operand(zero);
3888       }
3889    }
3890
3891    bld.insert(std::move(instr));
3892
3893    if (split_cnt)
3894       emit_split_vector(ctx, dst, split_cnt);
3895    else
3896       ctx->allocated_vec.emplace(dst.id(), allocated_vec); /* emit_split_vector already does this */
3897
3898    return dst;
3899 }
3900
3901 inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, unsigned const_offset)
3902 {
3903    if (const_offset >= 4096) {
3904       unsigned excess_const_offset = const_offset / 4096u * 4096u;
3905       const_offset %= 4096u;
3906
3907       if (!voffset.id())
3908          voffset = bld.copy(bld.def(v1), Operand(excess_const_offset));
3909       else if (unlikely(voffset.regClass() == s1))
3910          voffset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), Operand(excess_const_offset), Operand(voffset));
3911       else if (likely(voffset.regClass() == v1))
3912          voffset = bld.vadd32(bld.def(v1), Operand(voffset), Operand(excess_const_offset));
3913       else
3914          unreachable("Unsupported register class of voffset");
3915    }
3916
3917    return const_offset;
3918 }
3919
3920 void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata,
3921                              unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false)
3922 {
3923    assert(vdata.id());
3924    assert(vdata.size() != 3 || ctx->program->chip_class != GFX6);
3925    assert(vdata.size() >= 1 && vdata.size() <= 4);
3926
3927    Builder bld(ctx->program, ctx->block);
3928    aco_opcode op = get_buffer_store_op(false, vdata.bytes());
3929    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3930
3931    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3932    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3933    Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset,
3934                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3935                                  /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc);
3936
3937    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3938 }
3939
3940 void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset,
3941                                    unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask,
3942                                    bool allow_combining = true, bool reorder = true, bool slc = false)
3943 {
3944    Builder bld(ctx->program, ctx->block);
3945    assert(elem_size_bytes == 2 || elem_size_bytes == 4 || elem_size_bytes == 8);
3946    assert(write_mask);
3947    write_mask = widen_mask(write_mask, elem_size_bytes);
3948
3949    unsigned write_count = 0;
3950    Temp write_datas[32];
3951    unsigned offsets[32];
3952    split_buffer_store(ctx, NULL, false, RegType::vgpr, src, write_mask,
3953                       allow_combining ? 16 : 4, &write_count, write_datas, offsets);
3954
3955    for (unsigned i = 0; i < write_count; i++) {
3956       unsigned const_offset = offsets[i] + base_const_offset;
3957       emit_single_mubuf_store(ctx, descriptor, voffset, soffset, write_datas[i], const_offset, reorder, slc);
3958    }
3959 }
3960
3961 void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, Temp soffset,
3962                      unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components,
3963                      unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true)
3964 {
3965    assert(elem_size_bytes == 2 || elem_size_bytes == 4 || elem_size_bytes == 8);
3966    assert((num_components * elem_size_bytes) == dst.bytes());
3967    assert(!!stride != allow_combining);
3968
3969    Builder bld(ctx->program, ctx->block);
3970
3971    LoadEmitInfo info = {Operand(voffset), dst, num_components, elem_size_bytes, descriptor};
3972    info.component_stride = allow_combining ? 0 : stride;
3973    info.glc = true;
3974    info.swizzle_component_size = allow_combining ? 0 : 4;
3975    info.align_mul = MIN2(elem_size_bytes, 4);
3976    info.align_offset = 0;
3977    info.soffset = soffset;
3978    info.const_offset = base_const_offset;
3979    emit_mubuf_load(ctx, bld, &info);
3980 }
3981
3982 std::pair<Temp, unsigned> offset_add_from_nir(isel_context *ctx, const std::pair<Temp, unsigned> &base_offset, nir_src *off_src, unsigned stride = 1u)
3983 {
3984    Builder bld(ctx->program, ctx->block);
3985    Temp offset = base_offset.first;
3986    unsigned const_offset = base_offset.second;
3987
3988    if (!nir_src_is_const(*off_src)) {
3989       Temp indirect_offset_arg = get_ssa_temp(ctx, off_src->ssa);
3990       Temp with_stride;
3991
3992       /* Calculate indirect offset with stride */
3993       if (likely(indirect_offset_arg.regClass() == v1))
3994          with_stride = bld.v_mul24_imm(bld.def(v1), indirect_offset_arg, stride);
3995       else if (indirect_offset_arg.regClass() == s1)
3996          with_stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), indirect_offset_arg);
3997       else
3998          unreachable("Unsupported register class of indirect offset");
3999
4000       /* Add to the supplied base offset */
4001       if (offset.id() == 0)
4002          offset = with_stride;
4003       else if (unlikely(offset.regClass() == s1 && with_stride.regClass() == s1))
4004          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), with_stride, offset);
4005       else if (offset.size() == 1 && with_stride.size() == 1)
4006          offset = bld.vadd32(bld.def(v1), with_stride, offset);
4007       else
4008          unreachable("Unsupported register class of indirect offset");
4009    } else {
4010       unsigned const_offset_arg = nir_src_as_uint(*off_src);
4011       const_offset += const_offset_arg * stride;
4012    }
4013
4014    return std::make_pair(offset, const_offset);
4015 }
4016
4017 std::pair<Temp, unsigned> offset_add(isel_context *ctx, const std::pair<Temp, unsigned> &off1, const std::pair<Temp, unsigned> &off2)
4018 {
4019    Builder bld(ctx->program, ctx->block);
4020    Temp offset;
4021
4022    if (off1.first.id() && off2.first.id()) {
4023       if (unlikely(off1.first.regClass() == s1 && off2.first.regClass() == s1))
4024          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), off1.first, off2.first);
4025       else if (off1.first.size() == 1 && off2.first.size() == 1)
4026          offset = bld.vadd32(bld.def(v1), off1.first, off2.first);
4027       else
4028          unreachable("Unsupported register class of indirect offset");
4029    } else {
4030       offset = off1.first.id() ? off1.first : off2.first;
4031    }
4032
4033    return std::make_pair(offset, off1.second + off2.second);
4034 }
4035
4036 std::pair<Temp, unsigned> offset_mul(isel_context *ctx, const std::pair<Temp, unsigned> &offs, unsigned multiplier)
4037 {
4038    Builder bld(ctx->program, ctx->block);
4039    unsigned const_offset = offs.second * multiplier;
4040
4041    if (!offs.first.id())
4042       return std::make_pair(offs.first, const_offset);
4043
4044    Temp offset = unlikely(offs.first.regClass() == s1)
4045                  ? bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(multiplier), offs.first)
4046                  : bld.v_mul24_imm(bld.def(v1), offs.first, multiplier);
4047
4048    return std::make_pair(offset, const_offset);
4049 }
4050
4051 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride, unsigned component_stride)
4052 {
4053    Builder bld(ctx->program, ctx->block);
4054
4055    /* base is the driver_location, which is already multiplied by 4, so is in dwords */
4056    unsigned const_offset = nir_intrinsic_base(instr) * base_stride;
4057    /* component is in bytes */
4058    const_offset += nir_intrinsic_component(instr) * component_stride;
4059
4060    /* offset should be interpreted in relation to the base, so the instruction effectively reads/writes another input/output when it has an offset */
4061    nir_src *off_src = nir_get_io_offset_src(instr);
4062    return offset_add_from_nir(ctx, std::make_pair(Temp(), const_offset), off_src, 4u * base_stride);
4063 }
4064
4065 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned stride = 1u)
4066 {
4067    return get_intrinsic_io_basic_offset(ctx, instr, stride, stride);
4068 }
4069
4070 Temp get_tess_rel_patch_id(isel_context *ctx)
4071 {
4072    Builder bld(ctx->program, ctx->block);
4073
4074    switch (ctx->shader->info.stage) {
4075    case MESA_SHADER_TESS_CTRL:
4076       return bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffu),
4077                       get_arg(ctx, ctx->args->ac.tcs_rel_ids));
4078    case MESA_SHADER_TESS_EVAL:
4079       return get_arg(ctx, ctx->args->tes_rel_patch_id);
4080    default:
4081       unreachable("Unsupported stage in get_tess_rel_patch_id");
4082    }
4083 }
4084
4085 std::pair<Temp, unsigned> get_tcs_per_vertex_input_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr)
4086 {
4087    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4088    Builder bld(ctx->program, ctx->block);
4089
4090    uint32_t tcs_in_patch_stride = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 4;
4091    uint32_t tcs_in_vertex_stride = ctx->tcs_num_inputs * 4;
4092
4093    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr);
4094
4095    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
4096    offs = offset_add_from_nir(ctx, offs, vertex_index_src, tcs_in_vertex_stride);
4097
4098    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
4099    Temp tcs_in_current_patch_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, tcs_in_patch_stride);
4100    offs = offset_add(ctx, offs, std::make_pair(tcs_in_current_patch_offset, 0));
4101
4102    return offset_mul(ctx, offs, 4u);
4103 }
4104
4105 std::pair<Temp, unsigned> get_tcs_output_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, bool per_vertex = false)
4106 {
4107    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4108    Builder bld(ctx->program, ctx->block);
4109
4110    uint32_t input_patch_size = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 16;
4111    uint32_t output_vertex_size = ctx->tcs_num_outputs * 16;
4112    uint32_t pervertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
4113    uint32_t output_patch_stride = pervertex_output_patch_size + ctx->tcs_num_patch_outputs * 16;
4114
4115    std::pair<Temp, unsigned> offs = instr
4116                                     ? get_intrinsic_io_basic_offset(ctx, instr, 4u)
4117                                     : std::make_pair(Temp(), 0u);
4118
4119    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
4120    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, output_patch_stride);
4121
4122    if (per_vertex) {
4123       assert(instr);
4124
4125       nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
4126       offs = offset_add_from_nir(ctx, offs, vertex_index_src, output_vertex_size);
4127
4128       uint32_t output_patch0_offset = (input_patch_size * ctx->tcs_num_patches);
4129       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_offset));
4130    } else {
4131       uint32_t output_patch0_patch_data_offset = (input_patch_size * ctx->tcs_num_patches + pervertex_output_patch_size);
4132       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_patch_data_offset));
4133    }
4134
4135    return offs;
4136 }
4137
4138 std::pair<Temp, unsigned> get_tcs_per_vertex_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr)
4139 {
4140    Builder bld(ctx->program, ctx->block);
4141
4142    unsigned vertices_per_patch = ctx->shader->info.tess.tcs_vertices_out;
4143    unsigned attr_stride = vertices_per_patch * ctx->tcs_num_patches;
4144
4145    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u);
4146
4147    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
4148    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, vertices_per_patch * 16u);
4149    offs = offset_add(ctx, offs, std::make_pair(patch_off, 0u));
4150
4151    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
4152    offs = offset_add_from_nir(ctx, offs, vertex_index_src, 16u);
4153
4154    return offs;
4155 }
4156
4157 std::pair<Temp, unsigned> get_tcs_per_patch_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, unsigned const_base_offset = 0u)
4158 {
4159    Builder bld(ctx->program, ctx->block);
4160
4161    unsigned output_vertex_size = ctx->tcs_num_outputs * 16;
4162    unsigned per_vertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
4163    unsigned per_patch_data_offset = per_vertex_output_patch_size * ctx->tcs_num_patches;
4164    unsigned attr_stride = ctx->tcs_num_patches;
4165
4166    std::pair<Temp, unsigned> offs = instr
4167                                     ? get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u)
4168                                     : std::make_pair(Temp(), 0u);
4169
4170    if (const_base_offset)
4171       offs.second += const_base_offset * attr_stride;
4172
4173    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
4174    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, 16u);
4175    offs = offset_add(ctx, offs, std::make_pair(patch_off, per_patch_data_offset));
4176
4177    return offs;
4178 }
4179
4180 bool tcs_driver_location_matches_api_mask(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex, uint64_t mask, bool *indirect)
4181 {
4182    assert(per_vertex || ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4183
4184    if (mask == 0)
4185       return false;
4186
4187    unsigned drv_loc = nir_intrinsic_base(instr);
4188    nir_src *off_src = nir_get_io_offset_src(instr);
4189
4190    if (!nir_src_is_const(*off_src)) {
4191       *indirect = true;
4192       return false;
4193    }
4194
4195    *indirect = false;
4196    uint64_t slot = per_vertex
4197                    ? ctx->output_drv_loc_to_var_slot[ctx->shader->info.stage][drv_loc / 4]
4198                    : (ctx->output_tcs_patch_drv_loc_to_var_slot[drv_loc / 4] - VARYING_SLOT_PATCH0);
4199    return (((uint64_t) 1) << slot) & mask;
4200 }
4201
4202 bool store_output_to_temps(isel_context *ctx, nir_intrinsic_instr *instr)
4203 {
4204    unsigned write_mask = nir_intrinsic_write_mask(instr);
4205    unsigned component = nir_intrinsic_component(instr);
4206    unsigned idx = nir_intrinsic_base(instr) + component;
4207
4208    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
4209    if (off_instr->type != nir_instr_type_load_const)
4210       return false;
4211
4212    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
4213    idx += nir_src_as_uint(instr->src[1]) * 4u;
4214
4215    if (instr->src[0].ssa->bit_size == 64)
4216       write_mask = widen_mask(write_mask, 2);
4217
4218    RegClass rc = instr->src[0].ssa->bit_size == 16 ? v2b : v1;
4219
4220    for (unsigned i = 0; i < 8; ++i) {
4221       if (write_mask & (1 << i)) {
4222          ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
4223          ctx->outputs.temps[idx] = emit_extract_vector(ctx, src, i, rc);
4224       }
4225       idx++;
4226    }
4227
4228    return true;
4229 }
4230
4231 bool load_input_from_temps(isel_context *ctx, nir_intrinsic_instr *instr, Temp dst)
4232 {
4233    /* Only TCS per-vertex inputs are supported by this function.
4234     * Per-vertex inputs only match between the VS/TCS invocation id when the number of invocations is the same.
4235     */
4236    if (ctx->shader->info.stage != MESA_SHADER_TESS_CTRL || !ctx->tcs_in_out_eq)
4237       return false;
4238
4239    nir_src *off_src = nir_get_io_offset_src(instr);
4240    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
4241    nir_instr *vertex_index_instr = vertex_index_src->ssa->parent_instr;
4242    bool can_use_temps = nir_src_is_const(*off_src) &&
4243                         vertex_index_instr->type == nir_instr_type_intrinsic &&
4244                         nir_instr_as_intrinsic(vertex_index_instr)->intrinsic == nir_intrinsic_load_invocation_id;
4245
4246    if (!can_use_temps)
4247       return false;
4248
4249    unsigned idx = nir_intrinsic_base(instr) + nir_intrinsic_component(instr) + 4 * nir_src_as_uint(*off_src);
4250    Temp *src = &ctx->inputs.temps[idx];
4251    create_vec_from_array(ctx, src, dst.size(), dst.regClass().type(), 4u, 0, dst);
4252
4253    return true;
4254 }
4255
4256 void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr)
4257 {
4258    Builder bld(ctx->program, ctx->block);
4259
4260    if (ctx->tcs_in_out_eq && store_output_to_temps(ctx, instr)) {
4261       /* When the TCS only reads this output directly and for the same vertices as its invocation id, it is unnecessary to store the VS output to LDS. */
4262       bool indirect_write;
4263       bool temp_only_input = tcs_driver_location_matches_api_mask(ctx, instr, true, ctx->tcs_temp_only_inputs, &indirect_write);
4264       if (temp_only_input && !indirect_write)
4265          return;
4266    }
4267
4268    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, 4u);
4269    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
4270    unsigned write_mask = nir_intrinsic_write_mask(instr);
4271    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
4272
4273    if (ctx->stage == vertex_es || ctx->stage == tess_eval_es) {
4274       /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */
4275       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
4276       Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset);
4277       store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true);
4278    } else {
4279       Temp lds_base;
4280
4281       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4282          /* GFX9+: ES stage is merged into GS, data is passed between them using LDS. */
4283          unsigned itemsize = ctx->stage == vertex_geometry_gs
4284                              ? ctx->program->info->vs.es_info.esgs_itemsize
4285                              : ctx->program->info->tes.es_info.esgs_itemsize;
4286          Temp thread_id = emit_mbcnt(ctx, bld.def(v1));
4287          Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
4288          Temp vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), thread_id,
4289                                bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
4290          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
4291       } else if (ctx->stage == vertex_ls || ctx->stage == vertex_tess_control_hs) {
4292          /* GFX6-8: VS runs on LS stage when tessellation is used, but LS shares LDS space with HS.
4293           * GFX9+: LS is merged into HS, but still uses the same LDS layout.
4294           */
4295          Temp vertex_idx = get_arg(ctx, ctx->args->rel_auto_id);
4296          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, ctx->tcs_num_inputs * 16u);
4297       } else {
4298          unreachable("Invalid LS or ES stage");
4299       }
4300
4301       offs = offset_add(ctx, offs, std::make_pair(lds_base, 0u));
4302       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4303       store_lds(ctx, elem_size_bytes, src, write_mask, offs.first, offs.second, lds_align);
4304    }
4305 }
4306
4307 bool tcs_output_is_tess_factor(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
4308 {
4309    if (per_vertex)
4310       return false;
4311
4312    unsigned off = nir_intrinsic_base(instr) * 4u;
4313    return off == ctx->tcs_tess_lvl_out_loc ||
4314           off == ctx->tcs_tess_lvl_in_loc;
4315
4316 }
4317
4318 bool tcs_output_is_read_by_tes(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
4319 {
4320    uint64_t mask = per_vertex
4321                    ? ctx->program->info->tcs.tes_inputs_read
4322                    : ctx->program->info->tcs.tes_patch_inputs_read;
4323
4324    bool indirect_write = false;
4325    bool output_read_by_tes = tcs_driver_location_matches_api_mask(ctx, instr, per_vertex, mask, &indirect_write);
4326    return indirect_write || output_read_by_tes;
4327 }
4328
4329 bool tcs_output_is_read_by_tcs(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
4330 {
4331    uint64_t mask = per_vertex
4332                    ? ctx->shader->info.outputs_read
4333                    : ctx->shader->info.patch_outputs_read;
4334
4335    bool indirect_write = false;
4336    bool output_read = tcs_driver_location_matches_api_mask(ctx, instr, per_vertex, mask, &indirect_write);
4337    return indirect_write || output_read;
4338 }
4339
4340 void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
4341 {
4342    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
4343    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4344
4345    Builder bld(ctx->program, ctx->block);
4346
4347    Temp store_val = get_ssa_temp(ctx, instr->src[0].ssa);
4348    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4349    unsigned write_mask = nir_intrinsic_write_mask(instr);
4350
4351    bool is_tess_factor = tcs_output_is_tess_factor(ctx, instr, per_vertex);
4352    bool write_to_vmem = !is_tess_factor && tcs_output_is_read_by_tes(ctx, instr, per_vertex);
4353    bool write_to_lds = is_tess_factor || tcs_output_is_read_by_tcs(ctx, instr, per_vertex);
4354
4355    if (write_to_vmem) {
4356       std::pair<Temp, unsigned> vmem_offs = per_vertex
4357                                             ? get_tcs_per_vertex_output_vmem_offset(ctx, instr)
4358                                             : get_tcs_per_patch_output_vmem_offset(ctx, instr);
4359
4360       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4361       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
4362       store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, false);
4363    }
4364
4365    if (write_to_lds) {
4366       std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
4367       unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
4368       store_lds(ctx, elem_size_bytes, store_val, write_mask, lds_offs.first, lds_offs.second, lds_align);
4369    }
4370 }
4371
4372 void visit_load_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
4373 {
4374    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
4375    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4376
4377    Builder bld(ctx->program, ctx->block);
4378
4379    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4380    std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
4381    unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
4382    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4383
4384    load_lds(ctx, elem_size_bytes, dst, lds_offs.first, lds_offs.second, lds_align);
4385 }
4386
4387 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
4388 {
4389    if (ctx->stage == vertex_vs ||
4390        ctx->stage == tess_eval_vs ||
4391        ctx->stage == fragment_fs ||
4392        ctx->stage == ngg_vertex_gs ||
4393        ctx->stage == ngg_tess_eval_gs ||
4394        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
4395       bool stored_to_temps = store_output_to_temps(ctx, instr);
4396       if (!stored_to_temps) {
4397          fprintf(stderr, "Unimplemented output offset instruction:\n");
4398          nir_print_instr(instr->src[1].ssa->parent_instr, stderr);
4399          fprintf(stderr, "\n");
4400          abort();
4401       }
4402    } else if (ctx->stage == vertex_es ||
4403               ctx->stage == vertex_ls ||
4404               ctx->stage == tess_eval_es ||
4405               (ctx->stage == vertex_tess_control_hs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
4406               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
4407               (ctx->stage == tess_eval_geometry_gs && ctx->shader->info.stage == MESA_SHADER_TESS_EVAL)) {
4408       visit_store_ls_or_es_output(ctx, instr);
4409    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
4410       visit_store_tcs_output(ctx, instr, false);
4411    } else {
4412       unreachable("Shader stage not implemented");
4413    }
4414 }
4415
4416 void visit_load_output(isel_context *ctx, nir_intrinsic_instr *instr)
4417 {
4418    visit_load_tcs_output(ctx, instr, false);
4419 }
4420
4421 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
4422 {
4423    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
4424    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
4425
4426    Builder bld(ctx->program, ctx->block);
4427
4428    if (dst.regClass() == v2b) {
4429       if (ctx->program->has_16bank_lds) {
4430          assert(ctx->options->chip_class <= GFX8);
4431          Builder::Result interp_p1 =
4432             bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1),
4433                        Operand(2u) /* P0 */, bld.m0(prim_mask), idx, component);
4434          interp_p1 = bld.vintrp(aco_opcode::v_interp_p1lv_f16, bld.def(v2b),
4435                                 coord1, bld.m0(prim_mask), interp_p1, idx, component);
4436          bld.vintrp(aco_opcode::v_interp_p2_legacy_f16, Definition(dst), coord2,
4437                  bld.m0(prim_mask), interp_p1, idx, component);
4438       } else {
4439          aco_opcode interp_p2_op = aco_opcode::v_interp_p2_f16;
4440
4441          if (ctx->options->chip_class == GFX8)
4442             interp_p2_op = aco_opcode::v_interp_p2_legacy_f16;
4443
4444          Builder::Result interp_p1 =
4445             bld.vintrp(aco_opcode::v_interp_p1ll_f16, bld.def(v1),
4446                        coord1, bld.m0(prim_mask), idx, component);
4447          bld.vintrp(interp_p2_op, Definition(dst), coord2, bld.m0(prim_mask),
4448                     interp_p1, idx, component);
4449       }
4450    } else {
4451       Builder::Result interp_p1 =
4452          bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1,
4453                     bld.m0(prim_mask), idx, component);
4454
4455       if (ctx->program->has_16bank_lds)
4456          interp_p1.instr->operands[0].setLateKill(true);
4457
4458       bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2,
4459                  bld.m0(prim_mask), interp_p1, idx, component);
4460    }
4461 }
4462
4463 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
4464 {
4465    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
4466    for (unsigned i = 0; i < num_components; i++)
4467       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
4468    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
4469       assert(num_components == 4);
4470       Builder bld(ctx->program, ctx->block);
4471       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
4472    }
4473
4474    for (Operand& op : vec->operands)
4475       op = op.isUndefined() ? Operand(0u) : op;
4476
4477    vec->definitions[0] = Definition(dst);
4478    ctx->block->instructions.emplace_back(std::move(vec));
4479    emit_split_vector(ctx, dst, num_components);
4480    return;
4481 }
4482
4483 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
4484 {
4485    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4486    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
4487    unsigned idx = nir_intrinsic_base(instr);
4488    unsigned component = nir_intrinsic_component(instr);
4489    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
4490
4491    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
4492    if (offset) {
4493       assert(offset->u32 == 0);
4494    } else {
4495       /* the lower 15bit of the prim_mask contain the offset into LDS
4496        * while the upper bits contain the number of prims */
4497       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
4498       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
4499       Builder bld(ctx->program, ctx->block);
4500       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
4501       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
4502       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
4503       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
4504       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
4505    }
4506
4507    if (instr->dest.ssa.num_components == 1) {
4508       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
4509    } else {
4510       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
4511       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
4512       {
4513          Temp tmp = {ctx->program->allocateId(), v1};
4514          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
4515          vec->operands[i] = Operand(tmp);
4516       }
4517       vec->definitions[0] = Definition(dst);
4518       ctx->block->instructions.emplace_back(std::move(vec));
4519    }
4520 }
4521
4522 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
4523                              unsigned offset, unsigned stride, unsigned channels)
4524 {
4525    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
4526    if (vtx_info->chan_byte_size != 4 && channels == 3)
4527       return false;
4528    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
4529           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
4530 }
4531
4532 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
4533                               unsigned offset, unsigned stride, unsigned *channels)
4534 {
4535    if (!vtx_info->chan_byte_size) {
4536       *channels = vtx_info->num_channels;
4537       return vtx_info->chan_format;
4538    }
4539
4540    unsigned num_channels = *channels;
4541    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
4542       unsigned new_channels = num_channels + 1;
4543       /* first, assume more loads is worse and try using a larger data format */
4544       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
4545          new_channels++;
4546          /* don't make the attribute potentially out-of-bounds */
4547          if (offset + new_channels * vtx_info->chan_byte_size > stride)
4548             new_channels = 5;
4549       }
4550
4551       if (new_channels == 5) {
4552          /* then try decreasing load size (at the cost of more loads) */
4553          new_channels = *channels;
4554          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
4555             new_channels--;
4556       }
4557
4558       if (new_channels < *channels)
4559          *channels = new_channels;
4560       num_channels = new_channels;
4561    }
4562
4563    switch (vtx_info->chan_format) {
4564    case V_008F0C_BUF_DATA_FORMAT_8:
4565       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
4566                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
4567    case V_008F0C_BUF_DATA_FORMAT_16:
4568       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
4569                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
4570    case V_008F0C_BUF_DATA_FORMAT_32:
4571       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
4572                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
4573    }
4574    unreachable("shouldn't reach here");
4575    return V_008F0C_BUF_DATA_FORMAT_INVALID;
4576 }
4577
4578 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
4579  * so we may need to fix it up. */
4580 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
4581 {
4582    Builder bld(ctx->program, ctx->block);
4583
4584    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
4585       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
4586
4587    /* For the integer-like cases, do a natural sign extension.
4588     *
4589     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
4590     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
4591     * exponent.
4592     */
4593    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
4594    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
4595
4596    /* Convert back to the right type. */
4597    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
4598       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4599       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
4600       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
4601    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
4602       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
4603    }
4604
4605    return alpha;
4606 }
4607
4608 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
4609 {
4610    Builder bld(ctx->program, ctx->block);
4611    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4612    if (ctx->shader->info.stage == MESA_SHADER_VERTEX) {
4613
4614       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
4615       if (off_instr->type != nir_instr_type_load_const) {
4616          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4617          nir_print_instr(off_instr, stderr);
4618          fprintf(stderr, "\n");
4619       }
4620       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
4621
4622       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
4623
4624       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
4625       unsigned component = nir_intrinsic_component(instr);
4626       unsigned bitsize = instr->dest.ssa.bit_size;
4627       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
4628       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
4629       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
4630       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
4631
4632       unsigned dfmt = attrib_format & 0xf;
4633       unsigned nfmt = (attrib_format >> 4) & 0x7;
4634       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
4635
4636       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
4637       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
4638       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
4639       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
4640       if (post_shuffle)
4641          num_channels = MAX2(num_channels, 3);
4642
4643       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
4644       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
4645
4646       Temp index;
4647       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
4648          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
4649          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
4650          if (divisor) {
4651             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
4652             if (divisor != 1) {
4653                Temp divided = bld.tmp(v1);
4654                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
4655                index = bld.vadd32(bld.def(v1), start_instance, divided);
4656             } else {
4657                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
4658             }
4659          } else {
4660             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
4661          }
4662       } else {
4663          index = bld.vadd32(bld.def(v1),
4664                             get_arg(ctx, ctx->args->ac.base_vertex),
4665                             get_arg(ctx, ctx->args->ac.vertex_id));
4666       }
4667
4668       Temp channels[num_channels];
4669       unsigned channel_start = 0;
4670       bool direct_fetch = false;
4671
4672       /* skip unused channels at the start */
4673       if (vtx_info->chan_byte_size && !post_shuffle) {
4674          channel_start = ffs(mask) - 1;
4675          for (unsigned i = 0; i < channel_start; i++)
4676             channels[i] = Temp(0, s1);
4677       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
4678          num_channels = 3 - (ffs(mask) - 1);
4679       }
4680
4681       /* load channels */
4682       while (channel_start < num_channels) {
4683          unsigned fetch_component = num_channels - channel_start;
4684          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
4685          bool expanded = false;
4686
4687          /* use MUBUF when possible to avoid possible alignment issues */
4688          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
4689          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
4690                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
4691                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
4692                           vtx_info->chan_byte_size == 4;
4693          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
4694          if (!use_mubuf) {
4695             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_component);
4696          } else {
4697             if (fetch_component == 3 && ctx->options->chip_class == GFX6) {
4698                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
4699                fetch_component = 4;
4700                expanded = true;
4701             }
4702          }
4703
4704          unsigned fetch_bytes = fetch_component * bitsize / 8;
4705
4706          Temp fetch_index = index;
4707          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
4708             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
4709             fetch_offset = fetch_offset % attrib_stride;
4710          }
4711
4712          Operand soffset(0u);
4713          if (fetch_offset >= 4096) {
4714             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
4715             fetch_offset %= 4096;
4716          }
4717
4718          aco_opcode opcode;
4719          switch (fetch_bytes) {
4720          case 2:
4721             assert(!use_mubuf && bitsize == 16);
4722             opcode = aco_opcode::tbuffer_load_format_d16_x;
4723             break;
4724          case 4:
4725             if (bitsize == 16) {
4726                assert(!use_mubuf);
4727                opcode = aco_opcode::tbuffer_load_format_d16_xy;
4728             } else {
4729                opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
4730             }
4731             break;
4732          case 6:
4733             assert(!use_mubuf && bitsize == 16);
4734             opcode = aco_opcode::tbuffer_load_format_d16_xyz;
4735             break;
4736          case 8:
4737             if (bitsize == 16) {
4738                assert(!use_mubuf);
4739                opcode = aco_opcode::tbuffer_load_format_d16_xyzw;
4740             } else {
4741                opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
4742             }
4743             break;
4744          case 12:
4745             assert(ctx->options->chip_class >= GFX7 ||
4746                    (!use_mubuf && ctx->options->chip_class == GFX6));
4747             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
4748             break;
4749          case 16:
4750             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
4751             break;
4752          default:
4753             unreachable("Unimplemented load_input vector size");
4754          }
4755
4756          Temp fetch_dst;
4757          if (channel_start == 0 && fetch_bytes == dst.bytes() && !post_shuffle &&
4758              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
4759                            num_channels <= 3)) {
4760             direct_fetch = true;
4761             fetch_dst = dst;
4762          } else {
4763             fetch_dst = bld.tmp(RegClass::get(RegType::vgpr, fetch_bytes));
4764          }
4765
4766          if (use_mubuf) {
4767             Instruction *mubuf = bld.mubuf(opcode,
4768                                            Definition(fetch_dst), list, fetch_index, soffset,
4769                                            fetch_offset, false, true).instr;
4770             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
4771          } else {
4772             Instruction *mtbuf = bld.mtbuf(opcode,
4773                                            Definition(fetch_dst), list, fetch_index, soffset,
4774                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
4775             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
4776          }
4777
4778          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
4779
4780          if (fetch_component == 1) {
4781             channels[channel_start] = fetch_dst;
4782          } else {
4783             for (unsigned i = 0; i < MIN2(fetch_component, num_channels - channel_start); i++)
4784                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i,
4785                                                                  bitsize == 16 ? v2b : v1);
4786          }
4787
4788          channel_start += fetch_component;
4789       }
4790
4791       if (!direct_fetch) {
4792          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
4793                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
4794
4795          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
4796          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
4797          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
4798
4799          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4800          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4801          unsigned num_temp = 0;
4802          for (unsigned i = 0; i < dst.size(); i++) {
4803             unsigned idx = i + component;
4804             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
4805                Temp channel = channels[swizzle[idx]];
4806                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
4807                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
4808                vec->operands[i] = Operand(channel);
4809
4810                num_temp++;
4811                elems[i] = channel;
4812             } else if (is_float && idx == 3) {
4813                vec->operands[i] = Operand(0x3f800000u);
4814             } else if (!is_float && idx == 3) {
4815                vec->operands[i] = Operand(1u);
4816             } else {
4817                vec->operands[i] = Operand(0u);
4818             }
4819          }
4820          vec->definitions[0] = Definition(dst);
4821          ctx->block->instructions.emplace_back(std::move(vec));
4822          emit_split_vector(ctx, dst, dst.size());
4823
4824          if (num_temp == dst.size())
4825             ctx->allocated_vec.emplace(dst.id(), elems);
4826       }
4827    } else if (ctx->shader->info.stage == MESA_SHADER_FRAGMENT) {
4828       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
4829       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
4830       if (off_instr->type != nir_instr_type_load_const ||
4831           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
4832          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
4833          nir_print_instr(off_instr, stderr);
4834          fprintf(stderr, "\n");
4835       }
4836
4837       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
4838       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
4839       if (offset) {
4840          assert(offset->u32 == 0);
4841       } else {
4842          /* the lower 15bit of the prim_mask contain the offset into LDS
4843           * while the upper bits contain the number of prims */
4844          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
4845          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
4846          Builder bld(ctx->program, ctx->block);
4847          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
4848          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
4849          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
4850          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
4851          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
4852       }
4853
4854       unsigned idx = nir_intrinsic_base(instr);
4855       unsigned component = nir_intrinsic_component(instr);
4856       unsigned vertex_id = 2; /* P0 */
4857
4858       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
4859          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
4860          switch (src0->u32) {
4861          case 0:
4862             vertex_id = 2; /* P0 */
4863             break;
4864          case 1:
4865             vertex_id = 0; /* P10 */
4866             break;
4867          case 2:
4868             vertex_id = 1; /* P20 */
4869             break;
4870          default:
4871             unreachable("invalid vertex index");
4872          }
4873       }
4874
4875       if (dst.size() == 1) {
4876          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
4877       } else {
4878          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
4879          for (unsigned i = 0; i < dst.size(); i++)
4880             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
4881          vec->definitions[0] = Definition(dst);
4882          bld.insert(std::move(vec));
4883       }
4884
4885    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_EVAL) {
4886       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4887       Temp soffset = get_arg(ctx, ctx->args->oc_lds);
4888       std::pair<Temp, unsigned> offs = get_tcs_per_patch_output_vmem_offset(ctx, instr);
4889       unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8u;
4890
4891       load_vmem_mubuf(ctx, dst, ring, offs.first, soffset, offs.second, elem_size_bytes, instr->dest.ssa.num_components);
4892    } else {
4893       unreachable("Shader stage not implemented");
4894    }
4895 }
4896
4897 std::pair<Temp, unsigned> get_gs_per_vertex_input_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride = 1u)
4898 {
4899    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4900
4901    Builder bld(ctx->program, ctx->block);
4902    nir_src *vertex_src = nir_get_io_vertex_index_src(instr);
4903    Temp vertex_offset;
4904
4905    if (!nir_src_is_const(*vertex_src)) {
4906       /* better code could be created, but this case probably doesn't happen
4907        * much in practice */
4908       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, vertex_src->ssa));
4909       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
4910          Temp elem;
4911
4912          if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4913             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
4914             if (i % 2u)
4915                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
4916          } else {
4917             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
4918          }
4919
4920          if (vertex_offset.id()) {
4921             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)),
4922                                  Operand(i), indirect_vertex);
4923             vertex_offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), vertex_offset, elem, cond);
4924          } else {
4925             vertex_offset = elem;
4926          }
4927       }
4928
4929       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4930          vertex_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), vertex_offset);
4931    } else {
4932       unsigned vertex = nir_src_as_uint(*vertex_src);
4933       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
4934          vertex_offset = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
4935                                   get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
4936                                   Operand((vertex % 2u) * 16u), Operand(16u));
4937       else
4938          vertex_offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
4939    }
4940
4941    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, base_stride);
4942    offs = offset_add(ctx, offs, std::make_pair(vertex_offset, 0u));
4943    return offset_mul(ctx, offs, 4u);
4944 }
4945
4946 void visit_load_gs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4947 {
4948    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
4949
4950    Builder bld(ctx->program, ctx->block);
4951    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4952    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4953
4954    if (ctx->stage == geometry_gs) {
4955       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr, ctx->program->wave_size);
4956       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
4957       load_vmem_mubuf(ctx, dst, ring, offs.first, Temp(), offs.second, elem_size_bytes, instr->dest.ssa.num_components, 4u * ctx->program->wave_size, false, true);
4958    } else if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
4959       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr);
4960       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4961       load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4962    } else {
4963       unreachable("Unsupported GS stage.");
4964    }
4965 }
4966
4967 void visit_load_tcs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4968 {
4969    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4970
4971    Builder bld(ctx->program, ctx->block);
4972    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4973
4974    if (load_input_from_temps(ctx, instr, dst))
4975       return;
4976
4977    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_input_lds_offset(ctx, instr);
4978    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4979    unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4980
4981    load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4982 }
4983
4984 void visit_load_tes_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4985 {
4986    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4987
4988    Builder bld(ctx->program, ctx->block);
4989
4990    Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4991    Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
4992    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4993
4994    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4995    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_output_vmem_offset(ctx, instr);
4996
4997    load_vmem_mubuf(ctx, dst, ring, offs.first, oc_lds, offs.second, elem_size_bytes, instr->dest.ssa.num_components, 0u, true, true);
4998 }
4999
5000 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
5001 {
5002    switch (ctx->shader->info.stage) {
5003    case MESA_SHADER_GEOMETRY:
5004       visit_load_gs_per_vertex_input(ctx, instr);
5005       break;
5006    case MESA_SHADER_TESS_CTRL:
5007       visit_load_tcs_per_vertex_input(ctx, instr);
5008       break;
5009    case MESA_SHADER_TESS_EVAL:
5010       visit_load_tes_per_vertex_input(ctx, instr);
5011       break;
5012    default:
5013       unreachable("Unimplemented shader stage");
5014    }
5015 }
5016
5017 void visit_load_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
5018 {
5019    visit_load_tcs_output(ctx, instr, true);
5020 }
5021
5022 void visit_store_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
5023 {
5024    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
5025    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
5026
5027    visit_store_tcs_output(ctx, instr, true);
5028 }
5029
5030 void visit_load_tess_coord(isel_context *ctx, nir_intrinsic_instr *instr)
5031 {
5032    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
5033
5034    Builder bld(ctx->program, ctx->block);
5035    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5036
5037    Operand tes_u(get_arg(ctx, ctx->args->tes_u));
5038    Operand tes_v(get_arg(ctx, ctx->args->tes_v));
5039    Operand tes_w(0u);
5040
5041    if (ctx->shader->info.tess.primitive_mode == GL_TRIANGLES) {
5042       Temp tmp = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), tes_u, tes_v);
5043       tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0x3f800000u /* 1.0f */), tmp);
5044       tes_w = Operand(tmp);
5045    }
5046
5047    Temp tess_coord = bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tes_u, tes_v, tes_w);
5048    emit_split_vector(ctx, tess_coord, 3);
5049 }
5050
5051 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
5052 {
5053    if (ctx->program->info->need_indirect_descriptor_sets) {
5054       Builder bld(ctx->program, ctx->block);
5055       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
5056       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
5057       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
5058    }
5059
5060    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
5061 }
5062
5063
5064 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
5065 {
5066    Builder bld(ctx->program, ctx->block);
5067    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
5068    if (!nir_dest_is_divergent(instr->dest))
5069       index = bld.as_uniform(index);
5070    unsigned desc_set = nir_intrinsic_desc_set(instr);
5071    unsigned binding = nir_intrinsic_binding(instr);
5072
5073    Temp desc_ptr;
5074    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
5075    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
5076    unsigned offset = layout->binding[binding].offset;
5077    unsigned stride;
5078    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
5079        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
5080       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
5081       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
5082       offset = pipeline_layout->push_constant_size + 16 * idx;
5083       stride = 16;
5084    } else {
5085       desc_ptr = load_desc_ptr(ctx, desc_set);
5086       stride = layout->binding[binding].size;
5087    }
5088
5089    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
5090    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
5091    if (stride != 1) {
5092       if (nir_const_index) {
5093          const_index = const_index * stride;
5094       } else if (index.type() == RegType::vgpr) {
5095          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
5096          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
5097       } else {
5098          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
5099       }
5100    }
5101    if (offset) {
5102       if (nir_const_index) {
5103          const_index = const_index + offset;
5104       } else if (index.type() == RegType::vgpr) {
5105          index = bld.vadd32(bld.def(v1), Operand(offset), index);
5106       } else {
5107          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
5108       }
5109    }
5110
5111    if (nir_const_index && const_index == 0) {
5112       index = desc_ptr;
5113    } else if (index.type() == RegType::vgpr) {
5114       index = bld.vadd32(bld.def(v1),
5115                          nir_const_index ? Operand(const_index) : Operand(index),
5116                          Operand(desc_ptr));
5117    } else {
5118       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
5119                        nir_const_index ? Operand(const_index) : Operand(index),
5120                        Operand(desc_ptr));
5121    }
5122
5123    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
5124 }
5125
5126 void load_buffer(isel_context *ctx, unsigned num_components, unsigned component_size,
5127                  Temp dst, Temp rsrc, Temp offset, unsigned align_mul, unsigned align_offset,
5128                  bool glc=false, bool readonly=true)
5129 {
5130    Builder bld(ctx->program, ctx->block);
5131
5132    bool use_smem = dst.type() != RegType::vgpr && ((ctx->options->chip_class >= GFX8 && component_size >= 4) || readonly);
5133    if (use_smem)
5134       offset = bld.as_uniform(offset);
5135
5136    LoadEmitInfo info = {Operand(offset), dst, num_components, component_size, rsrc};
5137    info.glc = glc;
5138    info.barrier = readonly ? barrier_none : barrier_buffer;
5139    info.can_reorder = readonly;
5140    info.align_mul = align_mul;
5141    info.align_offset = align_offset;
5142    if (use_smem)
5143       emit_smem_load(ctx, bld, &info);
5144    else
5145       emit_mubuf_load(ctx, bld, &info);
5146 }
5147
5148 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
5149 {
5150    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5151    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
5152
5153    Builder bld(ctx->program, ctx->block);
5154
5155    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
5156    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
5157    unsigned binding = nir_intrinsic_binding(idx_instr);
5158    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
5159
5160    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
5161       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5162                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5163                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5164                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
5165       if (ctx->options->chip_class >= GFX10) {
5166          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5167                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5168                       S_008F0C_RESOURCE_LEVEL(1);
5169       } else {
5170          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5171                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5172       }
5173       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
5174                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
5175                                      Operand(0xFFFFFFFFu),
5176                                      Operand(desc_type));
5177       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5178                         rsrc, upper_dwords);
5179    } else {
5180       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
5181       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5182    }
5183    unsigned size = instr->dest.ssa.bit_size / 8;
5184    load_buffer(ctx, instr->num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa),
5185                nir_intrinsic_align_mul(instr), nir_intrinsic_align_offset(instr));
5186 }
5187
5188 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
5189 {
5190    Builder bld(ctx->program, ctx->block);
5191    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5192    unsigned offset = nir_intrinsic_base(instr);
5193    unsigned count = instr->dest.ssa.num_components;
5194    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
5195
5196    if (index_cv && instr->dest.ssa.bit_size == 32) {
5197       unsigned start = (offset + index_cv->u32) / 4u;
5198       start -= ctx->args->ac.base_inline_push_consts;
5199       if (start + count <= ctx->args->ac.num_inline_push_consts) {
5200          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5201          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5202          for (unsigned i = 0; i < count; ++i) {
5203             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
5204             vec->operands[i] = Operand{elems[i]};
5205          }
5206          vec->definitions[0] = Definition(dst);
5207          ctx->block->instructions.emplace_back(std::move(vec));
5208          ctx->allocated_vec.emplace(dst.id(), elems);
5209          return;
5210       }
5211    }
5212
5213    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
5214    if (offset != 0) // TODO check if index != 0 as well
5215       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
5216    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
5217    Temp vec = dst;
5218    bool trim = false;
5219    bool aligned = true;
5220
5221    if (instr->dest.ssa.bit_size == 8) {
5222       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
5223       bool fits_in_dword = count == 1 || (index_cv && ((offset + index_cv->u32) % 4 + count) <= 4);
5224       if (!aligned)
5225          vec = fits_in_dword ? bld.tmp(s1) : bld.tmp(s2);
5226    } else if (instr->dest.ssa.bit_size == 16) {
5227       aligned = index_cv && (offset + index_cv->u32) % 4 == 0;
5228       if (!aligned)
5229          vec = count == 4 ? bld.tmp(s4) : count > 1 ? bld.tmp(s2) : bld.tmp(s1);
5230    }
5231
5232    aco_opcode op;
5233
5234    switch (vec.size()) {
5235    case 1:
5236       op = aco_opcode::s_load_dword;
5237       break;
5238    case 2:
5239       op = aco_opcode::s_load_dwordx2;
5240       break;
5241    case 3:
5242       vec = bld.tmp(s4);
5243       trim = true;
5244    case 4:
5245       op = aco_opcode::s_load_dwordx4;
5246       break;
5247    case 6:
5248       vec = bld.tmp(s8);
5249       trim = true;
5250    case 8:
5251       op = aco_opcode::s_load_dwordx8;
5252       break;
5253    default:
5254       unreachable("unimplemented or forbidden load_push_constant.");
5255    }
5256
5257    bld.smem(op, Definition(vec), ptr, index);
5258
5259    if (!aligned) {
5260       Operand byte_offset = index_cv ? Operand((offset + index_cv->u32) % 4) : Operand(index);
5261       byte_align_scalar(ctx, vec, byte_offset, dst);
5262       return;
5263    }
5264
5265    if (trim) {
5266       emit_split_vector(ctx, vec, 4);
5267       RegClass rc = dst.size() == 3 ? s1 : s2;
5268       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5269                  emit_extract_vector(ctx, vec, 0, rc),
5270                  emit_extract_vector(ctx, vec, 1, rc),
5271                  emit_extract_vector(ctx, vec, 2, rc));
5272
5273    }
5274    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
5275 }
5276
5277 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
5278 {
5279    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5280
5281    Builder bld(ctx->program, ctx->block);
5282
5283    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
5284                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
5285                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
5286                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
5287    if (ctx->options->chip_class >= GFX10) {
5288       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5289                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5290                    S_008F0C_RESOURCE_LEVEL(1);
5291    } else {
5292       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5293                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5294    }
5295
5296    unsigned base = nir_intrinsic_base(instr);
5297    unsigned range = nir_intrinsic_range(instr);
5298
5299    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5300    if (base && offset.type() == RegType::sgpr)
5301       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
5302    else if (base && offset.type() == RegType::vgpr)
5303       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
5304
5305    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5306                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
5307                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
5308                           Operand(desc_type));
5309    unsigned size = instr->dest.ssa.bit_size / 8;
5310    // TODO: get alignment information for subdword constants
5311    load_buffer(ctx, instr->num_components, size, dst, rsrc, offset, size, 0);
5312 }
5313
5314 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
5315 {
5316    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
5317       ctx->cf_info.exec_potentially_empty_discard = true;
5318
5319    ctx->program->needs_exact = true;
5320
5321    // TODO: optimize uniform conditions
5322    Builder bld(ctx->program, ctx->block);
5323    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5324    assert(src.regClass() == bld.lm);
5325    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5326    bld.pseudo(aco_opcode::p_discard_if, src);
5327    ctx->block->kind |= block_kind_uses_discard_if;
5328    return;
5329 }
5330
5331 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
5332 {
5333    Builder bld(ctx->program, ctx->block);
5334
5335    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
5336       ctx->cf_info.exec_potentially_empty_discard = true;
5337
5338    bool divergent = ctx->cf_info.parent_if.is_divergent ||
5339                     ctx->cf_info.parent_loop.has_divergent_continue;
5340
5341    if (ctx->block->loop_nest_depth &&
5342        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
5343       /* we handle discards the same way as jump instructions */
5344       append_logical_end(ctx->block);
5345
5346       /* in loops, discard behaves like break */
5347       Block *linear_target = ctx->cf_info.parent_loop.exit;
5348       ctx->block->kind |= block_kind_discard;
5349
5350       if (!divergent) {
5351          /* uniform discard - loop ends here */
5352          assert(nir_instr_is_last(&instr->instr));
5353          ctx->block->kind |= block_kind_uniform;
5354          ctx->cf_info.has_branch = true;
5355          bld.branch(aco_opcode::p_branch);
5356          add_linear_edge(ctx->block->index, linear_target);
5357          return;
5358       }
5359
5360       /* we add a break right behind the discard() instructions */
5361       ctx->block->kind |= block_kind_break;
5362       unsigned idx = ctx->block->index;
5363
5364       ctx->cf_info.parent_loop.has_divergent_branch = true;
5365       ctx->cf_info.nir_to_aco[instr->instr.block->index] = idx;
5366
5367       /* remove critical edges from linear CFG */
5368       bld.branch(aco_opcode::p_branch);
5369       Block* break_block = ctx->program->create_and_insert_block();
5370       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5371       break_block->kind |= block_kind_uniform;
5372       add_linear_edge(idx, break_block);
5373       add_linear_edge(break_block->index, linear_target);
5374       bld.reset(break_block);
5375       bld.branch(aco_opcode::p_branch);
5376
5377       Block* continue_block = ctx->program->create_and_insert_block();
5378       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
5379       add_linear_edge(idx, continue_block);
5380       append_logical_start(continue_block);
5381       ctx->block = continue_block;
5382
5383       return;
5384    }
5385
5386    /* it can currently happen that NIR doesn't remove the unreachable code */
5387    if (!nir_instr_is_last(&instr->instr)) {
5388       ctx->program->needs_exact = true;
5389       /* save exec somewhere temporarily so that it doesn't get
5390        * overwritten before the discard from outer exec masks */
5391       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
5392       bld.pseudo(aco_opcode::p_discard_if, cond);
5393       ctx->block->kind |= block_kind_uses_discard_if;
5394       return;
5395    }
5396
5397    /* This condition is incorrect for uniformly branched discards in a loop
5398     * predicated by a divergent condition, but the above code catches that case
5399     * and the discard would end up turning into a discard_if.
5400     * For example:
5401     * if (divergent) {
5402     *    while (...) {
5403     *       if (uniform) {
5404     *          discard;
5405     *       }
5406     *    }
5407     * }
5408     */
5409    if (!ctx->cf_info.parent_if.is_divergent) {
5410       /* program just ends here */
5411       ctx->block->kind |= block_kind_uniform;
5412       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
5413               0 /* enabled mask */, 9 /* dest */,
5414               false /* compressed */, true/* done */, true /* valid mask */);
5415       bld.sopp(aco_opcode::s_endpgm);
5416       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
5417    } else {
5418       ctx->block->kind |= block_kind_discard;
5419       /* branch and linear edge is added by visit_if() */
5420    }
5421 }
5422
5423 enum aco_descriptor_type {
5424    ACO_DESC_IMAGE,
5425    ACO_DESC_FMASK,
5426    ACO_DESC_SAMPLER,
5427    ACO_DESC_BUFFER,
5428    ACO_DESC_PLANE_0,
5429    ACO_DESC_PLANE_1,
5430    ACO_DESC_PLANE_2,
5431 };
5432
5433 static bool
5434 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
5435    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
5436       return false;
5437    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
5438    return dim == ac_image_cube ||
5439           dim == ac_image_1darray ||
5440           dim == ac_image_2darray ||
5441           dim == ac_image_2darraymsaa;
5442 }
5443
5444 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
5445                       enum aco_descriptor_type desc_type,
5446                       const nir_tex_instr *tex_instr, bool image, bool write)
5447 {
5448 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
5449    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
5450    if (it != ctx->tex_desc.end())
5451       return it->second;
5452 */
5453    Temp index = Temp();
5454    bool index_set = false;
5455    unsigned constant_index = 0;
5456    unsigned descriptor_set;
5457    unsigned base_index;
5458    Builder bld(ctx->program, ctx->block);
5459
5460    if (!deref_instr) {
5461       assert(tex_instr && !image);
5462       descriptor_set = 0;
5463       base_index = tex_instr->sampler_index;
5464    } else {
5465       while(deref_instr->deref_type != nir_deref_type_var) {
5466          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
5467          if (!array_size)
5468             array_size = 1;
5469
5470          assert(deref_instr->deref_type == nir_deref_type_array);
5471          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
5472          if (const_value) {
5473             constant_index += array_size * const_value->u32;
5474          } else {
5475             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
5476             if (indirect.type() == RegType::vgpr)
5477                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
5478
5479             if (array_size != 1)
5480                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
5481
5482             if (!index_set) {
5483                index = indirect;
5484                index_set = true;
5485             } else {
5486                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
5487             }
5488          }
5489
5490          deref_instr = nir_src_as_deref(deref_instr->parent);
5491       }
5492       descriptor_set = deref_instr->var->data.descriptor_set;
5493       base_index = deref_instr->var->data.binding;
5494    }
5495
5496    Temp list = load_desc_ptr(ctx, descriptor_set);
5497    list = convert_pointer_to_64_bit(ctx, list);
5498
5499    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
5500    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
5501    unsigned offset = binding->offset;
5502    unsigned stride = binding->size;
5503    aco_opcode opcode;
5504    RegClass type;
5505
5506    assert(base_index < layout->binding_count);
5507
5508    switch (desc_type) {
5509    case ACO_DESC_IMAGE:
5510       type = s8;
5511       opcode = aco_opcode::s_load_dwordx8;
5512       break;
5513    case ACO_DESC_FMASK:
5514       type = s8;
5515       opcode = aco_opcode::s_load_dwordx8;
5516       offset += 32;
5517       break;
5518    case ACO_DESC_SAMPLER:
5519       type = s4;
5520       opcode = aco_opcode::s_load_dwordx4;
5521       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
5522          offset += radv_combined_image_descriptor_sampler_offset(binding);
5523       break;
5524    case ACO_DESC_BUFFER:
5525       type = s4;
5526       opcode = aco_opcode::s_load_dwordx4;
5527       break;
5528    case ACO_DESC_PLANE_0:
5529    case ACO_DESC_PLANE_1:
5530       type = s8;
5531       opcode = aco_opcode::s_load_dwordx8;
5532       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
5533       break;
5534    case ACO_DESC_PLANE_2:
5535       type = s4;
5536       opcode = aco_opcode::s_load_dwordx4;
5537       offset += 64;
5538       break;
5539    default:
5540       unreachable("invalid desc_type\n");
5541    }
5542
5543    offset += constant_index * stride;
5544
5545    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
5546       (!index_set || binding->immutable_samplers_equal)) {
5547       if (binding->immutable_samplers_equal)
5548          constant_index = 0;
5549
5550       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
5551       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5552                         Operand(samplers[constant_index * 4 + 0]),
5553                         Operand(samplers[constant_index * 4 + 1]),
5554                         Operand(samplers[constant_index * 4 + 2]),
5555                         Operand(samplers[constant_index * 4 + 3]));
5556    }
5557
5558    Operand off;
5559    if (!index_set) {
5560       off = bld.copy(bld.def(s1), Operand(offset));
5561    } else {
5562       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
5563                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
5564    }
5565
5566    Temp res = bld.smem(opcode, bld.def(type), list, off);
5567
5568    if (desc_type == ACO_DESC_PLANE_2) {
5569       Temp components[8];
5570       for (unsigned i = 0; i < 8; i++)
5571          components[i] = bld.tmp(s1);
5572       bld.pseudo(aco_opcode::p_split_vector,
5573                  Definition(components[0]),
5574                  Definition(components[1]),
5575                  Definition(components[2]),
5576                  Definition(components[3]),
5577                  res);
5578
5579       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
5580       bld.pseudo(aco_opcode::p_split_vector,
5581                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
5582                  Definition(components[4]),
5583                  Definition(components[5]),
5584                  Definition(components[6]),
5585                  Definition(components[7]),
5586                  desc2);
5587
5588       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
5589                        components[0], components[1], components[2], components[3],
5590                        components[4], components[5], components[6], components[7]);
5591    }
5592
5593    return res;
5594 }
5595
5596 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
5597 {
5598    switch (dim) {
5599    case GLSL_SAMPLER_DIM_BUF:
5600       return 1;
5601    case GLSL_SAMPLER_DIM_1D:
5602       return array ? 2 : 1;
5603    case GLSL_SAMPLER_DIM_2D:
5604       return array ? 3 : 2;
5605    case GLSL_SAMPLER_DIM_MS:
5606       return array ? 4 : 3;
5607    case GLSL_SAMPLER_DIM_3D:
5608    case GLSL_SAMPLER_DIM_CUBE:
5609       return 3;
5610    case GLSL_SAMPLER_DIM_RECT:
5611    case GLSL_SAMPLER_DIM_SUBPASS:
5612       return 2;
5613    case GLSL_SAMPLER_DIM_SUBPASS_MS:
5614       return 3;
5615    default:
5616       break;
5617    }
5618    return 0;
5619 }
5620
5621
5622 /* Adjust the sample index according to FMASK.
5623  *
5624  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
5625  * which is the identity mapping. Each nibble says which physical sample
5626  * should be fetched to get that sample.
5627  *
5628  * For example, 0x11111100 means there are only 2 samples stored and
5629  * the second sample covers 3/4 of the pixel. When reading samples 0
5630  * and 1, return physical sample 0 (determined by the first two 0s
5631  * in FMASK), otherwise return physical sample 1.
5632  *
5633  * The sample index should be adjusted as follows:
5634  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
5635  */
5636 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
5637 {
5638    Builder bld(ctx->program, ctx->block);
5639    Temp fmask = bld.tmp(v1);
5640    unsigned dim = ctx->options->chip_class >= GFX10
5641                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
5642                   : 0;
5643
5644    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
5645                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
5646    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
5647    load->operands[0] = Operand(fmask_desc_ptr);
5648    load->operands[1] = Operand(s4); /* no sampler */
5649    load->operands[2] = Operand(coord);
5650    load->definitions[0] = Definition(fmask);
5651    load->glc = false;
5652    load->dlc = false;
5653    load->dmask = 0x1;
5654    load->unrm = true;
5655    load->da = da;
5656    load->dim = dim;
5657    load->can_reorder = true; /* fmask images shouldn't be modified */
5658    ctx->block->instructions.emplace_back(std::move(load));
5659
5660    Operand sample_index4;
5661    if (sample_index.isConstant()) {
5662       if (sample_index.constantValue() < 16) {
5663          sample_index4 = Operand(sample_index.constantValue() << 2);
5664       } else {
5665          sample_index4 = Operand(0u);
5666       }
5667    } else if (sample_index.regClass() == s1) {
5668       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
5669    } else {
5670       assert(sample_index.regClass() == v1);
5671       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
5672    }
5673
5674    Temp final_sample;
5675    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
5676       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
5677    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
5678       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
5679    else
5680       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
5681
5682    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
5683     * resource descriptor is 0 (invalid),
5684     */
5685    Temp compare = bld.tmp(bld.lm);
5686    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
5687                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
5688
5689    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
5690
5691    /* Replace the MSAA sample index. */
5692    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
5693 }
5694
5695 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
5696 {
5697
5698    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
5699    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5700    bool is_array = glsl_sampler_type_is_array(type);
5701    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5702    assert(!add_frag_pos && "Input attachments should be lowered.");
5703    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
5704    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
5705    int count = image_type_to_components_count(dim, is_array);
5706    std::vector<Temp> coords(count);
5707    Builder bld(ctx->program, ctx->block);
5708
5709    if (is_ms) {
5710       count--;
5711       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
5712       /* get sample index */
5713       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
5714          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
5715          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
5716          std::vector<Temp> fmask_load_address;
5717          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
5718             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
5719
5720          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
5721          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
5722       } else {
5723          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
5724       }
5725    }
5726
5727    if (gfx9_1d) {
5728       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
5729       coords.resize(coords.size() + 1);
5730       coords[1] = bld.copy(bld.def(v1), Operand(0u));
5731       if (is_array)
5732          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
5733    } else {
5734       for (int i = 0; i < count; i++)
5735          coords[i] = emit_extract_vector(ctx, src0, i, v1);
5736    }
5737
5738    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
5739        instr->intrinsic == nir_intrinsic_image_deref_store) {
5740       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
5741       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
5742
5743       if (!level_zero)
5744          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
5745    }
5746
5747    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
5748    for (unsigned i = 0; i < coords.size(); i++)
5749       vec->operands[i] = Operand(coords[i]);
5750    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
5751    vec->definitions[0] = Definition(res);
5752    ctx->block->instructions.emplace_back(std::move(vec));
5753    return res;
5754 }
5755
5756
5757 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
5758 {
5759    Builder bld(ctx->program, ctx->block);
5760    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5761    const struct glsl_type *type = glsl_without_array(var->type);
5762    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5763    bool is_array = glsl_sampler_type_is_array(type);
5764    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5765
5766    if (dim == GLSL_SAMPLER_DIM_BUF) {
5767       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
5768       unsigned num_channels = util_last_bit(mask);
5769       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5770       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5771
5772       aco_opcode opcode;
5773       switch (num_channels) {
5774       case 1:
5775          opcode = aco_opcode::buffer_load_format_x;
5776          break;
5777       case 2:
5778          opcode = aco_opcode::buffer_load_format_xy;
5779          break;
5780       case 3:
5781          opcode = aco_opcode::buffer_load_format_xyz;
5782          break;
5783       case 4:
5784          opcode = aco_opcode::buffer_load_format_xyzw;
5785          break;
5786       default:
5787          unreachable(">4 channel buffer image load");
5788       }
5789       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
5790       load->operands[0] = Operand(rsrc);
5791       load->operands[1] = Operand(vindex);
5792       load->operands[2] = Operand((uint32_t) 0);
5793       Temp tmp;
5794       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5795          tmp = dst;
5796       else
5797          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
5798       load->definitions[0] = Definition(tmp);
5799       load->idxen = true;
5800       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
5801       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5802       load->barrier = barrier_image;
5803       ctx->block->instructions.emplace_back(std::move(load));
5804
5805       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
5806       return;
5807    }
5808
5809    Temp coords = get_image_coords(ctx, instr, type);
5810    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5811
5812    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
5813    unsigned num_components = util_bitcount(dmask);
5814    Temp tmp;
5815    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
5816       tmp = dst;
5817    else
5818       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
5819
5820    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
5821    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
5822
5823    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
5824    load->operands[0] = Operand(resource);
5825    load->operands[1] = Operand(s4); /* no sampler */
5826    load->operands[2] = Operand(coords);
5827    load->definitions[0] = Definition(tmp);
5828    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
5829    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
5830    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5831    load->dmask = dmask;
5832    load->unrm = true;
5833    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5834    load->barrier = barrier_image;
5835    ctx->block->instructions.emplace_back(std::move(load));
5836
5837    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
5838    return;
5839 }
5840
5841 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
5842 {
5843    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5844    const struct glsl_type *type = glsl_without_array(var->type);
5845    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5846    bool is_array = glsl_sampler_type_is_array(type);
5847    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5848
5849    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
5850
5851    if (dim == GLSL_SAMPLER_DIM_BUF) {
5852       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5853       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5854       aco_opcode opcode;
5855       switch (data.size()) {
5856       case 1:
5857          opcode = aco_opcode::buffer_store_format_x;
5858          break;
5859       case 2:
5860          opcode = aco_opcode::buffer_store_format_xy;
5861          break;
5862       case 3:
5863          opcode = aco_opcode::buffer_store_format_xyz;
5864          break;
5865       case 4:
5866          opcode = aco_opcode::buffer_store_format_xyzw;
5867          break;
5868       default:
5869          unreachable(">4 channel buffer image store");
5870       }
5871       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
5872       store->operands[0] = Operand(rsrc);
5873       store->operands[1] = Operand(vindex);
5874       store->operands[2] = Operand((uint32_t) 0);
5875       store->operands[3] = Operand(data);
5876       store->idxen = true;
5877       store->glc = glc;
5878       store->dlc = false;
5879       store->disable_wqm = true;
5880       store->barrier = barrier_image;
5881       ctx->program->needs_exact = true;
5882       ctx->block->instructions.emplace_back(std::move(store));
5883       return;
5884    }
5885
5886    assert(data.type() == RegType::vgpr);
5887    Temp coords = get_image_coords(ctx, instr, type);
5888    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5889
5890    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
5891    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
5892
5893    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
5894    store->operands[0] = Operand(resource);
5895    store->operands[1] = Operand(data);
5896    store->operands[2] = Operand(coords);
5897    store->glc = glc;
5898    store->dlc = false;
5899    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5900    store->dmask = (1 << data.size()) - 1;
5901    store->unrm = true;
5902    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5903    store->disable_wqm = true;
5904    store->barrier = barrier_image;
5905    ctx->program->needs_exact = true;
5906    ctx->block->instructions.emplace_back(std::move(store));
5907    return;
5908 }
5909
5910 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5911 {
5912    /* return the previous value if dest is ever used */
5913    bool return_previous = false;
5914    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5915       return_previous = true;
5916       break;
5917    }
5918    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5919       return_previous = true;
5920       break;
5921    }
5922
5923    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5924    const struct glsl_type *type = glsl_without_array(var->type);
5925    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5926    bool is_array = glsl_sampler_type_is_array(type);
5927    Builder bld(ctx->program, ctx->block);
5928
5929    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5930    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
5931
5932    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
5933       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
5934
5935    aco_opcode buf_op, image_op;
5936    switch (instr->intrinsic) {
5937       case nir_intrinsic_image_deref_atomic_add:
5938          buf_op = aco_opcode::buffer_atomic_add;
5939          image_op = aco_opcode::image_atomic_add;
5940          break;
5941       case nir_intrinsic_image_deref_atomic_umin:
5942          buf_op = aco_opcode::buffer_atomic_umin;
5943          image_op = aco_opcode::image_atomic_umin;
5944          break;
5945       case nir_intrinsic_image_deref_atomic_imin:
5946          buf_op = aco_opcode::buffer_atomic_smin;
5947          image_op = aco_opcode::image_atomic_smin;
5948          break;
5949       case nir_intrinsic_image_deref_atomic_umax:
5950          buf_op = aco_opcode::buffer_atomic_umax;
5951          image_op = aco_opcode::image_atomic_umax;
5952          break;
5953       case nir_intrinsic_image_deref_atomic_imax:
5954          buf_op = aco_opcode::buffer_atomic_smax;
5955          image_op = aco_opcode::image_atomic_smax;
5956          break;
5957       case nir_intrinsic_image_deref_atomic_and:
5958          buf_op = aco_opcode::buffer_atomic_and;
5959          image_op = aco_opcode::image_atomic_and;
5960          break;
5961       case nir_intrinsic_image_deref_atomic_or:
5962          buf_op = aco_opcode::buffer_atomic_or;
5963          image_op = aco_opcode::image_atomic_or;
5964          break;
5965       case nir_intrinsic_image_deref_atomic_xor:
5966          buf_op = aco_opcode::buffer_atomic_xor;
5967          image_op = aco_opcode::image_atomic_xor;
5968          break;
5969       case nir_intrinsic_image_deref_atomic_exchange:
5970          buf_op = aco_opcode::buffer_atomic_swap;
5971          image_op = aco_opcode::image_atomic_swap;
5972          break;
5973       case nir_intrinsic_image_deref_atomic_comp_swap:
5974          buf_op = aco_opcode::buffer_atomic_cmpswap;
5975          image_op = aco_opcode::image_atomic_cmpswap;
5976          break;
5977       default:
5978          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
5979    }
5980
5981    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5982
5983    if (dim == GLSL_SAMPLER_DIM_BUF) {
5984       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5985       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5986       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
5987       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5988       mubuf->operands[0] = Operand(resource);
5989       mubuf->operands[1] = Operand(vindex);
5990       mubuf->operands[2] = Operand((uint32_t)0);
5991       mubuf->operands[3] = Operand(data);
5992       if (return_previous)
5993          mubuf->definitions[0] = Definition(dst);
5994       mubuf->offset = 0;
5995       mubuf->idxen = true;
5996       mubuf->glc = return_previous;
5997       mubuf->dlc = false; /* Not needed for atomics */
5998       mubuf->disable_wqm = true;
5999       mubuf->barrier = barrier_image;
6000       ctx->program->needs_exact = true;
6001       ctx->block->instructions.emplace_back(std::move(mubuf));
6002       return;
6003    }
6004
6005    Temp coords = get_image_coords(ctx, instr, type);
6006    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
6007    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
6008    mimg->operands[0] = Operand(resource);
6009    mimg->operands[1] = Operand(data);
6010    mimg->operands[2] = Operand(coords);
6011    if (return_previous)
6012       mimg->definitions[0] = Definition(dst);
6013    mimg->glc = return_previous;
6014    mimg->dlc = false; /* Not needed for atomics */
6015    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
6016    mimg->dmask = (1 << data.size()) - 1;
6017    mimg->unrm = true;
6018    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
6019    mimg->disable_wqm = true;
6020    mimg->barrier = barrier_image;
6021    ctx->program->needs_exact = true;
6022    ctx->block->instructions.emplace_back(std::move(mimg));
6023    return;
6024 }
6025
6026 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
6027 {
6028    if (in_elements && ctx->options->chip_class == GFX8) {
6029       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
6030       Builder bld(ctx->program, ctx->block);
6031
6032       Temp size = emit_extract_vector(ctx, desc, 2, s1);
6033
6034       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
6035       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
6036
6037       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
6038       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
6039
6040       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
6041       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
6042
6043       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
6044       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
6045                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
6046       if (dst.type() == RegType::vgpr)
6047          bld.copy(Definition(dst), shr_dst);
6048
6049       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
6050    } else {
6051       emit_extract_vector(ctx, desc, 2, dst);
6052    }
6053 }
6054
6055 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
6056 {
6057    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
6058    const struct glsl_type *type = glsl_without_array(var->type);
6059    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
6060    bool is_array = glsl_sampler_type_is_array(type);
6061    Builder bld(ctx->program, ctx->block);
6062
6063    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
6064       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
6065       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
6066    }
6067
6068    /* LOD */
6069    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6070
6071    /* Resource */
6072    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
6073
6074    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6075
6076    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
6077    mimg->operands[0] = Operand(resource);
6078    mimg->operands[1] = Operand(s4); /* no sampler */
6079    mimg->operands[2] = Operand(lod);
6080    uint8_t& dmask = mimg->dmask;
6081    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
6082    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
6083    mimg->da = glsl_sampler_type_is_array(type);
6084    mimg->can_reorder = true;
6085    Definition& def = mimg->definitions[0];
6086    ctx->block->instructions.emplace_back(std::move(mimg));
6087
6088    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
6089        glsl_sampler_type_is_array(type)) {
6090
6091       assert(instr->dest.ssa.num_components == 3);
6092       Temp tmp = {ctx->program->allocateId(), v3};
6093       def = Definition(tmp);
6094       emit_split_vector(ctx, tmp, 3);
6095
6096       /* divide 3rd value by 6 by multiplying with magic number */
6097       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6098       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
6099
6100       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6101                  emit_extract_vector(ctx, tmp, 0, v1),
6102                  emit_extract_vector(ctx, tmp, 1, v1),
6103                  by_6);
6104
6105    } else if (ctx->options->chip_class == GFX9 &&
6106               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
6107               glsl_sampler_type_is_array(type)) {
6108       assert(instr->dest.ssa.num_components == 2);
6109       def = Definition(dst);
6110       dmask = 0x5;
6111    } else {
6112       def = Definition(dst);
6113    }
6114
6115    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
6116 }
6117
6118 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
6119 {
6120    Builder bld(ctx->program, ctx->block);
6121    unsigned num_components = instr->num_components;
6122
6123    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6124    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6125    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
6126
6127    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
6128    unsigned size = instr->dest.ssa.bit_size / 8;
6129    load_buffer(ctx, num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa),
6130                nir_intrinsic_align_mul(instr), nir_intrinsic_align_offset(instr), glc, false);
6131 }
6132
6133 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
6134 {
6135    Builder bld(ctx->program, ctx->block);
6136    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
6137    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6138    unsigned writemask = widen_mask(nir_intrinsic_write_mask(instr), elem_size_bytes);
6139    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
6140
6141    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6142    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
6143
6144    bool smem = !nir_src_is_divergent(instr->src[2]) &&
6145                ctx->options->chip_class >= GFX8 &&
6146                elem_size_bytes >= 4;
6147    if (smem)
6148       offset = bld.as_uniform(offset);
6149    bool smem_nonfs = smem && ctx->stage != fragment_fs;
6150
6151    unsigned write_count = 0;
6152    Temp write_datas[32];
6153    unsigned offsets[32];
6154    split_buffer_store(ctx, instr, smem, smem_nonfs ? RegType::sgpr : (smem ? data.type() : RegType::vgpr),
6155                       data, writemask, 16, &write_count, write_datas, offsets);
6156
6157    for (unsigned i = 0; i < write_count; i++) {
6158       aco_opcode op = get_buffer_store_op(smem, write_datas[i].bytes());
6159       if (smem && ctx->stage == fragment_fs)
6160          op = aco_opcode::p_fs_buffer_store_smem;
6161
6162       if (smem) {
6163          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(op, Format::SMEM, 3, 0)};
6164          store->operands[0] = Operand(rsrc);
6165          if (offsets[i]) {
6166             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
6167                                 offset, Operand(offsets[i]));
6168             store->operands[1] = Operand(off);
6169          } else {
6170             store->operands[1] = Operand(offset);
6171          }
6172          if (op != aco_opcode::p_fs_buffer_store_smem)
6173             store->operands[1].setFixed(m0);
6174          store->operands[2] = Operand(write_datas[i]);
6175          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
6176          store->dlc = false;
6177          store->disable_wqm = true;
6178          store->barrier = barrier_buffer;
6179          ctx->block->instructions.emplace_back(std::move(store));
6180          ctx->program->wb_smem_l1_on_end = true;
6181          if (op == aco_opcode::p_fs_buffer_store_smem) {
6182             ctx->block->kind |= block_kind_needs_lowering;
6183             ctx->program->needs_exact = true;
6184          }
6185       } else {
6186          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
6187          store->operands[0] = Operand(rsrc);
6188          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
6189          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
6190          store->operands[3] = Operand(write_datas[i]);
6191          store->offset = offsets[i];
6192          store->offen = (offset.type() == RegType::vgpr);
6193          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
6194          store->dlc = false;
6195          store->disable_wqm = true;
6196          store->barrier = barrier_buffer;
6197          ctx->program->needs_exact = true;
6198          ctx->block->instructions.emplace_back(std::move(store));
6199       }
6200    }
6201 }
6202
6203 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
6204 {
6205    /* return the previous value if dest is ever used */
6206    bool return_previous = false;
6207    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6208       return_previous = true;
6209       break;
6210    }
6211    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6212       return_previous = true;
6213       break;
6214    }
6215
6216    Builder bld(ctx->program, ctx->block);
6217    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
6218
6219    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
6220       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
6221                         get_ssa_temp(ctx, instr->src[3].ssa), data);
6222
6223    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
6224    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6225    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
6226
6227    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6228
6229    aco_opcode op32, op64;
6230    switch (instr->intrinsic) {
6231       case nir_intrinsic_ssbo_atomic_add:
6232          op32 = aco_opcode::buffer_atomic_add;
6233          op64 = aco_opcode::buffer_atomic_add_x2;
6234          break;
6235       case nir_intrinsic_ssbo_atomic_imin:
6236          op32 = aco_opcode::buffer_atomic_smin;
6237          op64 = aco_opcode::buffer_atomic_smin_x2;
6238          break;
6239       case nir_intrinsic_ssbo_atomic_umin:
6240          op32 = aco_opcode::buffer_atomic_umin;
6241          op64 = aco_opcode::buffer_atomic_umin_x2;
6242          break;
6243       case nir_intrinsic_ssbo_atomic_imax:
6244          op32 = aco_opcode::buffer_atomic_smax;
6245          op64 = aco_opcode::buffer_atomic_smax_x2;
6246          break;
6247       case nir_intrinsic_ssbo_atomic_umax:
6248          op32 = aco_opcode::buffer_atomic_umax;
6249          op64 = aco_opcode::buffer_atomic_umax_x2;
6250          break;
6251       case nir_intrinsic_ssbo_atomic_and:
6252          op32 = aco_opcode::buffer_atomic_and;
6253          op64 = aco_opcode::buffer_atomic_and_x2;
6254          break;
6255       case nir_intrinsic_ssbo_atomic_or:
6256          op32 = aco_opcode::buffer_atomic_or;
6257          op64 = aco_opcode::buffer_atomic_or_x2;
6258          break;
6259       case nir_intrinsic_ssbo_atomic_xor:
6260          op32 = aco_opcode::buffer_atomic_xor;
6261          op64 = aco_opcode::buffer_atomic_xor_x2;
6262          break;
6263       case nir_intrinsic_ssbo_atomic_exchange:
6264          op32 = aco_opcode::buffer_atomic_swap;
6265          op64 = aco_opcode::buffer_atomic_swap_x2;
6266          break;
6267       case nir_intrinsic_ssbo_atomic_comp_swap:
6268          op32 = aco_opcode::buffer_atomic_cmpswap;
6269          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
6270          break;
6271       default:
6272          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
6273    }
6274    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6275    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
6276    mubuf->operands[0] = Operand(rsrc);
6277    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
6278    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
6279    mubuf->operands[3] = Operand(data);
6280    if (return_previous)
6281       mubuf->definitions[0] = Definition(dst);
6282    mubuf->offset = 0;
6283    mubuf->offen = (offset.type() == RegType::vgpr);
6284    mubuf->glc = return_previous;
6285    mubuf->dlc = false; /* Not needed for atomics */
6286    mubuf->disable_wqm = true;
6287    mubuf->barrier = barrier_buffer;
6288    ctx->program->needs_exact = true;
6289    ctx->block->instructions.emplace_back(std::move(mubuf));
6290 }
6291
6292 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
6293
6294    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6295    Builder bld(ctx->program, ctx->block);
6296    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
6297    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
6298 }
6299
6300 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
6301 {
6302    Builder bld(ctx->program, ctx->block);
6303    unsigned num_components = instr->num_components;
6304    unsigned component_size = instr->dest.ssa.bit_size / 8;
6305
6306    LoadEmitInfo info = {Operand(get_ssa_temp(ctx, instr->src[0].ssa)),
6307                         get_ssa_temp(ctx, &instr->dest.ssa),
6308                         num_components, component_size};
6309    info.glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
6310    info.align_mul = nir_intrinsic_align_mul(instr);
6311    info.align_offset = nir_intrinsic_align_offset(instr);
6312    info.barrier = barrier_buffer;
6313    info.can_reorder = false;
6314    /* VMEM stores don't update the SMEM cache and it's difficult to prove that
6315     * it's safe to use SMEM */
6316    bool can_use_smem = nir_intrinsic_access(instr) & ACCESS_NON_WRITEABLE;
6317    if (info.dst.type() == RegType::vgpr || (info.glc && ctx->options->chip_class < GFX8) || !can_use_smem) {
6318       emit_global_load(ctx, bld, &info);
6319    } else {
6320       info.offset = Operand(bld.as_uniform(info.offset));
6321       emit_smem_load(ctx, bld, &info);
6322    }
6323 }
6324
6325 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
6326 {
6327    Builder bld(ctx->program, ctx->block);
6328    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6329    unsigned writemask = widen_mask(nir_intrinsic_write_mask(instr), elem_size_bytes);
6330
6331    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6332    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
6333    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
6334
6335    if (ctx->options->chip_class >= GFX7)
6336       addr = as_vgpr(ctx, addr);
6337
6338    unsigned write_count = 0;
6339    Temp write_datas[32];
6340    unsigned offsets[32];
6341    split_buffer_store(ctx, instr, false, RegType::vgpr, data, writemask,
6342                       16, &write_count, write_datas, offsets);
6343
6344    for (unsigned i = 0; i < write_count; i++) {
6345       if (ctx->options->chip_class >= GFX7) {
6346          unsigned offset = offsets[i];
6347          Temp store_addr = addr;
6348          if (offset > 0 && ctx->options->chip_class < GFX9) {
6349             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
6350             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
6351             Temp carry = bld.tmp(bld.lm);
6352             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
6353
6354             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
6355                      Operand(offset), addr0);
6356             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
6357                      Operand(0u), addr1,
6358                      carry).def(1).setHint(vcc);
6359
6360             store_addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
6361
6362             offset = 0;
6363          }
6364
6365          bool global = ctx->options->chip_class >= GFX9;
6366          aco_opcode op;
6367          switch (write_datas[i].bytes()) {
6368          case 1:
6369             op = global ? aco_opcode::global_store_byte : aco_opcode::flat_store_byte;
6370             break;
6371          case 2:
6372             op = global ? aco_opcode::global_store_short : aco_opcode::flat_store_short;
6373             break;
6374          case 4:
6375             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
6376             break;
6377          case 8:
6378             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
6379             break;
6380          case 12:
6381             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
6382             break;
6383          case 16:
6384             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
6385             break;
6386          default:
6387             unreachable("store_global not implemented for this size.");
6388          }
6389
6390          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
6391          flat->operands[0] = Operand(store_addr);
6392          flat->operands[1] = Operand(s1);
6393          flat->operands[2] = Operand(write_datas[i]);
6394          flat->glc = glc;
6395          flat->dlc = false;
6396          flat->offset = offset;
6397          flat->disable_wqm = true;
6398          flat->barrier = barrier_buffer;
6399          ctx->program->needs_exact = true;
6400          ctx->block->instructions.emplace_back(std::move(flat));
6401       } else {
6402          assert(ctx->options->chip_class == GFX6);
6403
6404          aco_opcode op = get_buffer_store_op(false, write_datas[i].bytes());
6405
6406          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6407
6408          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
6409          mubuf->operands[0] = Operand(rsrc);
6410          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6411          mubuf->operands[2] = Operand(0u);
6412          mubuf->operands[3] = Operand(write_datas[i]);
6413          mubuf->glc = glc;
6414          mubuf->dlc = false;
6415          mubuf->offset = offsets[i];
6416          mubuf->addr64 = addr.type() == RegType::vgpr;
6417          mubuf->disable_wqm = true;
6418          mubuf->barrier = barrier_buffer;
6419          ctx->program->needs_exact = true;
6420          ctx->block->instructions.emplace_back(std::move(mubuf));
6421       }
6422    }
6423 }
6424
6425 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6426 {
6427    /* return the previous value if dest is ever used */
6428    bool return_previous = false;
6429    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6430       return_previous = true;
6431       break;
6432    }
6433    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6434       return_previous = true;
6435       break;
6436    }
6437
6438    Builder bld(ctx->program, ctx->block);
6439    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6440    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6441
6442    if (ctx->options->chip_class >= GFX7)
6443       addr = as_vgpr(ctx, addr);
6444
6445    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
6446       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
6447                         get_ssa_temp(ctx, instr->src[2].ssa), data);
6448
6449    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6450
6451    aco_opcode op32, op64;
6452
6453    if (ctx->options->chip_class >= GFX7) {
6454       bool global = ctx->options->chip_class >= GFX9;
6455       switch (instr->intrinsic) {
6456          case nir_intrinsic_global_atomic_add:
6457             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
6458             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
6459             break;
6460          case nir_intrinsic_global_atomic_imin:
6461             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
6462             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
6463             break;
6464          case nir_intrinsic_global_atomic_umin:
6465             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
6466             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
6467             break;
6468          case nir_intrinsic_global_atomic_imax:
6469             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
6470             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
6471             break;
6472          case nir_intrinsic_global_atomic_umax:
6473             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
6474             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
6475             break;
6476          case nir_intrinsic_global_atomic_and:
6477             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
6478             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
6479             break;
6480          case nir_intrinsic_global_atomic_or:
6481             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
6482             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
6483             break;
6484          case nir_intrinsic_global_atomic_xor:
6485             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
6486             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
6487             break;
6488          case nir_intrinsic_global_atomic_exchange:
6489             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
6490             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
6491             break;
6492          case nir_intrinsic_global_atomic_comp_swap:
6493             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
6494             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
6495             break;
6496          default:
6497             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6498       }
6499
6500       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6501       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
6502       flat->operands[0] = Operand(addr);
6503       flat->operands[1] = Operand(s1);
6504       flat->operands[2] = Operand(data);
6505       if (return_previous)
6506          flat->definitions[0] = Definition(dst);
6507       flat->glc = return_previous;
6508       flat->dlc = false; /* Not needed for atomics */
6509       flat->offset = 0;
6510       flat->disable_wqm = true;
6511       flat->barrier = barrier_buffer;
6512       ctx->program->needs_exact = true;
6513       ctx->block->instructions.emplace_back(std::move(flat));
6514    } else {
6515       assert(ctx->options->chip_class == GFX6);
6516
6517       switch (instr->intrinsic) {
6518          case nir_intrinsic_global_atomic_add:
6519             op32 = aco_opcode::buffer_atomic_add;
6520             op64 = aco_opcode::buffer_atomic_add_x2;
6521             break;
6522          case nir_intrinsic_global_atomic_imin:
6523             op32 = aco_opcode::buffer_atomic_smin;
6524             op64 = aco_opcode::buffer_atomic_smin_x2;
6525             break;
6526          case nir_intrinsic_global_atomic_umin:
6527             op32 = aco_opcode::buffer_atomic_umin;
6528             op64 = aco_opcode::buffer_atomic_umin_x2;
6529             break;
6530          case nir_intrinsic_global_atomic_imax:
6531             op32 = aco_opcode::buffer_atomic_smax;
6532             op64 = aco_opcode::buffer_atomic_smax_x2;
6533             break;
6534          case nir_intrinsic_global_atomic_umax:
6535             op32 = aco_opcode::buffer_atomic_umax;
6536             op64 = aco_opcode::buffer_atomic_umax_x2;
6537             break;
6538          case nir_intrinsic_global_atomic_and:
6539             op32 = aco_opcode::buffer_atomic_and;
6540             op64 = aco_opcode::buffer_atomic_and_x2;
6541             break;
6542          case nir_intrinsic_global_atomic_or:
6543             op32 = aco_opcode::buffer_atomic_or;
6544             op64 = aco_opcode::buffer_atomic_or_x2;
6545             break;
6546          case nir_intrinsic_global_atomic_xor:
6547             op32 = aco_opcode::buffer_atomic_xor;
6548             op64 = aco_opcode::buffer_atomic_xor_x2;
6549             break;
6550          case nir_intrinsic_global_atomic_exchange:
6551             op32 = aco_opcode::buffer_atomic_swap;
6552             op64 = aco_opcode::buffer_atomic_swap_x2;
6553             break;
6554          case nir_intrinsic_global_atomic_comp_swap:
6555             op32 = aco_opcode::buffer_atomic_cmpswap;
6556             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
6557             break;
6558          default:
6559             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
6560       }
6561
6562       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
6563
6564       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
6565
6566       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
6567       mubuf->operands[0] = Operand(rsrc);
6568       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
6569       mubuf->operands[2] = Operand(0u);
6570       mubuf->operands[3] = Operand(data);
6571       if (return_previous)
6572          mubuf->definitions[0] = Definition(dst);
6573       mubuf->glc = return_previous;
6574       mubuf->dlc = false;
6575       mubuf->offset = 0;
6576       mubuf->addr64 = addr.type() == RegType::vgpr;
6577       mubuf->disable_wqm = true;
6578       mubuf->barrier = barrier_buffer;
6579       ctx->program->needs_exact = true;
6580       ctx->block->instructions.emplace_back(std::move(mubuf));
6581    }
6582 }
6583
6584 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
6585    Builder bld(ctx->program, ctx->block);
6586    switch(instr->intrinsic) {
6587       case nir_intrinsic_group_memory_barrier:
6588       case nir_intrinsic_memory_barrier:
6589          bld.barrier(aco_opcode::p_memory_barrier_common);
6590          break;
6591       case nir_intrinsic_memory_barrier_buffer:
6592          bld.barrier(aco_opcode::p_memory_barrier_buffer);
6593          break;
6594       case nir_intrinsic_memory_barrier_image:
6595          bld.barrier(aco_opcode::p_memory_barrier_image);
6596          break;
6597       case nir_intrinsic_memory_barrier_tcs_patch:
6598       case nir_intrinsic_memory_barrier_shared:
6599          bld.barrier(aco_opcode::p_memory_barrier_shared);
6600          break;
6601       default:
6602          unreachable("Unimplemented memory barrier intrinsic");
6603          break;
6604    }
6605 }
6606
6607 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6608 {
6609    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
6610    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6611    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6612    Builder bld(ctx->program, ctx->block);
6613
6614    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
6615    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6616    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
6617 }
6618
6619 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
6620 {
6621    unsigned writemask = nir_intrinsic_write_mask(instr);
6622    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
6623    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6624    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6625
6626    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
6627    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
6628 }
6629
6630 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
6631 {
6632    unsigned offset = nir_intrinsic_base(instr);
6633    Builder bld(ctx->program, ctx->block);
6634    Operand m = load_lds_size_m0(bld);
6635    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6636    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6637
6638    unsigned num_operands = 3;
6639    aco_opcode op32, op64, op32_rtn, op64_rtn;
6640    switch(instr->intrinsic) {
6641       case nir_intrinsic_shared_atomic_add:
6642          op32 = aco_opcode::ds_add_u32;
6643          op64 = aco_opcode::ds_add_u64;
6644          op32_rtn = aco_opcode::ds_add_rtn_u32;
6645          op64_rtn = aco_opcode::ds_add_rtn_u64;
6646          break;
6647       case nir_intrinsic_shared_atomic_imin:
6648          op32 = aco_opcode::ds_min_i32;
6649          op64 = aco_opcode::ds_min_i64;
6650          op32_rtn = aco_opcode::ds_min_rtn_i32;
6651          op64_rtn = aco_opcode::ds_min_rtn_i64;
6652          break;
6653       case nir_intrinsic_shared_atomic_umin:
6654          op32 = aco_opcode::ds_min_u32;
6655          op64 = aco_opcode::ds_min_u64;
6656          op32_rtn = aco_opcode::ds_min_rtn_u32;
6657          op64_rtn = aco_opcode::ds_min_rtn_u64;
6658          break;
6659       case nir_intrinsic_shared_atomic_imax:
6660          op32 = aco_opcode::ds_max_i32;
6661          op64 = aco_opcode::ds_max_i64;
6662          op32_rtn = aco_opcode::ds_max_rtn_i32;
6663          op64_rtn = aco_opcode::ds_max_rtn_i64;
6664          break;
6665       case nir_intrinsic_shared_atomic_umax:
6666          op32 = aco_opcode::ds_max_u32;
6667          op64 = aco_opcode::ds_max_u64;
6668          op32_rtn = aco_opcode::ds_max_rtn_u32;
6669          op64_rtn = aco_opcode::ds_max_rtn_u64;
6670          break;
6671       case nir_intrinsic_shared_atomic_and:
6672          op32 = aco_opcode::ds_and_b32;
6673          op64 = aco_opcode::ds_and_b64;
6674          op32_rtn = aco_opcode::ds_and_rtn_b32;
6675          op64_rtn = aco_opcode::ds_and_rtn_b64;
6676          break;
6677       case nir_intrinsic_shared_atomic_or:
6678          op32 = aco_opcode::ds_or_b32;
6679          op64 = aco_opcode::ds_or_b64;
6680          op32_rtn = aco_opcode::ds_or_rtn_b32;
6681          op64_rtn = aco_opcode::ds_or_rtn_b64;
6682          break;
6683       case nir_intrinsic_shared_atomic_xor:
6684          op32 = aco_opcode::ds_xor_b32;
6685          op64 = aco_opcode::ds_xor_b64;
6686          op32_rtn = aco_opcode::ds_xor_rtn_b32;
6687          op64_rtn = aco_opcode::ds_xor_rtn_b64;
6688          break;
6689       case nir_intrinsic_shared_atomic_exchange:
6690          op32 = aco_opcode::ds_write_b32;
6691          op64 = aco_opcode::ds_write_b64;
6692          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
6693          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
6694          break;
6695       case nir_intrinsic_shared_atomic_comp_swap:
6696          op32 = aco_opcode::ds_cmpst_b32;
6697          op64 = aco_opcode::ds_cmpst_b64;
6698          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
6699          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
6700          num_operands = 4;
6701          break;
6702       default:
6703          unreachable("Unhandled shared atomic intrinsic");
6704    }
6705
6706    /* return the previous value if dest is ever used */
6707    bool return_previous = false;
6708    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6709       return_previous = true;
6710       break;
6711    }
6712    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6713       return_previous = true;
6714       break;
6715    }
6716
6717    aco_opcode op;
6718    if (data.size() == 1) {
6719       assert(instr->dest.ssa.bit_size == 32);
6720       op = return_previous ? op32_rtn : op32;
6721    } else {
6722       assert(instr->dest.ssa.bit_size == 64);
6723       op = return_previous ? op64_rtn : op64;
6724    }
6725
6726    if (offset > 65535) {
6727       address = bld.vadd32(bld.def(v1), Operand(offset), address);
6728       offset = 0;
6729    }
6730
6731    aco_ptr<DS_instruction> ds;
6732    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
6733    ds->operands[0] = Operand(address);
6734    ds->operands[1] = Operand(data);
6735    if (num_operands == 4)
6736       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
6737    ds->operands[num_operands - 1] = m;
6738    ds->offset0 = offset;
6739    if (return_previous)
6740       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
6741    ctx->block->instructions.emplace_back(std::move(ds));
6742 }
6743
6744 Temp get_scratch_resource(isel_context *ctx)
6745 {
6746    Builder bld(ctx->program, ctx->block);
6747    Temp scratch_addr = ctx->program->private_segment_buffer;
6748    if (ctx->stage != compute_cs)
6749       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
6750
6751    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
6752                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
6753
6754    if (ctx->program->chip_class >= GFX10) {
6755       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
6756                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
6757                    S_008F0C_RESOURCE_LEVEL(1);
6758    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
6759       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6760                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6761    }
6762
6763    /* older generations need element size = 16 bytes. element size removed in GFX9 */
6764    if (ctx->program->chip_class <= GFX8)
6765       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
6766
6767    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
6768 }
6769
6770 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6771    Builder bld(ctx->program, ctx->block);
6772    Temp rsrc = get_scratch_resource(ctx);
6773    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6774    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6775
6776    LoadEmitInfo info = {Operand(offset), dst, instr->dest.ssa.num_components,
6777                         instr->dest.ssa.bit_size / 8u, rsrc};
6778    info.align_mul = nir_intrinsic_align_mul(instr);
6779    info.align_offset = nir_intrinsic_align_offset(instr);
6780    info.swizzle_component_size = 16;
6781    info.can_reorder = false;
6782    info.soffset = ctx->program->scratch_offset;
6783    emit_mubuf_load(ctx, bld, &info);
6784 }
6785
6786 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6787    Builder bld(ctx->program, ctx->block);
6788    Temp rsrc = get_scratch_resource(ctx);
6789    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6790    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6791
6792    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6793    unsigned writemask = widen_mask(nir_intrinsic_write_mask(instr), elem_size_bytes);
6794
6795    unsigned write_count = 0;
6796    Temp write_datas[32];
6797    unsigned offsets[32];
6798    split_buffer_store(ctx, instr, false, RegType::vgpr, data, writemask,
6799                       16, &write_count, write_datas, offsets);
6800
6801    for (unsigned i = 0; i < write_count; i++) {
6802       aco_opcode op = get_buffer_store_op(false, write_datas[i].bytes());
6803       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_datas[i], offsets[i], true);
6804    }
6805 }
6806
6807 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
6808    uint8_t log2_ps_iter_samples;
6809    if (ctx->program->info->ps.force_persample) {
6810       log2_ps_iter_samples =
6811          util_logbase2(ctx->options->key.fs.num_samples);
6812    } else {
6813       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
6814    }
6815
6816    /* The bit pattern matches that used by fixed function fragment
6817     * processing. */
6818    static const unsigned ps_iter_masks[] = {
6819       0xffff, /* not used */
6820       0x5555,
6821       0x1111,
6822       0x0101,
6823       0x0001,
6824    };
6825    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
6826
6827    Builder bld(ctx->program, ctx->block);
6828
6829    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
6830                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6831    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
6832    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
6833    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6834    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
6835 }
6836
6837 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
6838    Builder bld(ctx->program, ctx->block);
6839
6840    unsigned stream = nir_intrinsic_stream_id(instr);
6841    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6842    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
6843    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
6844
6845    /* get GSVS ring */
6846    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
6847
6848    unsigned num_components =
6849       ctx->program->info->gs.num_stream_output_components[stream];
6850    assert(num_components);
6851
6852    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
6853    unsigned stream_offset = 0;
6854    for (unsigned i = 0; i < stream; i++) {
6855       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
6856       stream_offset += prev_stride * ctx->program->wave_size;
6857    }
6858
6859    /* Limit on the stride field for <= GFX7. */
6860    assert(stride < (1 << 14));
6861
6862    Temp gsvs_dwords[4];
6863    for (unsigned i = 0; i < 4; i++)
6864       gsvs_dwords[i] = bld.tmp(s1);
6865    bld.pseudo(aco_opcode::p_split_vector,
6866               Definition(gsvs_dwords[0]),
6867               Definition(gsvs_dwords[1]),
6868               Definition(gsvs_dwords[2]),
6869               Definition(gsvs_dwords[3]),
6870               gsvs_ring);
6871
6872    if (stream_offset) {
6873       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
6874
6875       Temp carry = bld.tmp(s1);
6876       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
6877       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
6878    }
6879
6880    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
6881    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
6882
6883    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6884                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
6885
6886    unsigned offset = 0;
6887    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
6888       if (ctx->program->info->gs.output_streams[i] != stream)
6889          continue;
6890
6891       for (unsigned j = 0; j < 4; j++) {
6892          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
6893             continue;
6894
6895          if (ctx->outputs.mask[i] & (1 << j)) {
6896             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
6897             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
6898             if (const_offset >= 4096u) {
6899                if (vaddr_offset.isUndefined())
6900                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
6901                else
6902                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
6903                const_offset %= 4096u;
6904             }
6905
6906             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
6907             mtbuf->operands[0] = Operand(gsvs_ring);
6908             mtbuf->operands[1] = vaddr_offset;
6909             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
6910             mtbuf->operands[3] = Operand(ctx->outputs.temps[i * 4u + j]);
6911             mtbuf->offen = !vaddr_offset.isUndefined();
6912             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
6913             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
6914             mtbuf->offset = const_offset;
6915             mtbuf->glc = true;
6916             mtbuf->slc = true;
6917             mtbuf->barrier = barrier_gs_data;
6918             mtbuf->can_reorder = true;
6919             bld.insert(std::move(mtbuf));
6920          }
6921
6922          offset += ctx->shader->info.gs.vertices_out;
6923       }
6924
6925       /* outputs for the next vertex are undefined and keeping them around can
6926        * create invalid IR with control flow */
6927       ctx->outputs.mask[i] = 0;
6928    }
6929
6930    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
6931 }
6932
6933 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
6934 {
6935    Builder bld(ctx->program, ctx->block);
6936
6937    if (cluster_size == 1) {
6938       return src;
6939    } if (op == nir_op_iand && cluster_size == 4) {
6940       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
6941       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6942       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
6943                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
6944    } else if (op == nir_op_ior && cluster_size == 4) {
6945       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
6946       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
6947                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
6948    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
6949       //subgroupAnd(val) -> (exec & ~val) == 0
6950       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6951       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6952       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
6953    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
6954       //subgroupOr(val) -> (val & exec) != 0
6955       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
6956       return bool_to_vector_condition(ctx, tmp);
6957    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
6958       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
6959       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6960       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
6961       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
6962       return bool_to_vector_condition(ctx, tmp);
6963    } else {
6964       //subgroupClustered{And,Or,Xor}(val, n) ->
6965       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
6966       //cluster_offset = ~(n - 1) & lane_id
6967       //cluster_mask = ((1 << n) - 1)
6968       //subgroupClusteredAnd():
6969       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
6970       //subgroupClusteredOr():
6971       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
6972       //subgroupClusteredXor():
6973       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
6974       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
6975       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
6976
6977       Temp tmp;
6978       if (op == nir_op_iand)
6979          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6980       else
6981          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6982
6983       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
6984
6985       if (ctx->program->chip_class <= GFX7)
6986          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
6987       else if (ctx->program->wave_size == 64)
6988          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
6989       else
6990          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
6991       tmp = emit_extract_vector(ctx, tmp, 0, v1);
6992       if (cluster_mask != 0xffffffff)
6993          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
6994
6995       Definition cmp_def = Definition();
6996       if (op == nir_op_iand) {
6997          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
6998       } else if (op == nir_op_ior) {
6999          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
7000       } else if (op == nir_op_ixor) {
7001          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
7002                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
7003          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
7004       }
7005       cmp_def.setHint(vcc);
7006       return cmp_def.getTemp();
7007    }
7008 }
7009
7010 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
7011 {
7012    Builder bld(ctx->program, ctx->block);
7013
7014    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
7015    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
7016    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
7017    Temp tmp;
7018    if (op == nir_op_iand)
7019       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
7020    else
7021       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
7022
7023    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
7024    Temp lo = lohi.def(0).getTemp();
7025    Temp hi = lohi.def(1).getTemp();
7026    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
7027
7028    Definition cmp_def = Definition();
7029    if (op == nir_op_iand)
7030       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
7031    else if (op == nir_op_ior)
7032       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
7033    else if (op == nir_op_ixor)
7034       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
7035                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
7036    cmp_def.setHint(vcc);
7037    return cmp_def.getTemp();
7038 }
7039
7040 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
7041 {
7042    Builder bld(ctx->program, ctx->block);
7043
7044    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
7045    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
7046    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
7047    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
7048    if (op == nir_op_iand)
7049       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7050    else if (op == nir_op_ior)
7051       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7052    else if (op == nir_op_ixor)
7053       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
7054
7055    assert(false);
7056    return Temp();
7057 }
7058
7059 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
7060 {
7061    Builder bld(ctx->program, ctx->block);
7062    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
7063    if (src.regClass().type() == RegType::vgpr) {
7064       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
7065    } else if (src.regClass() == s1) {
7066       bld.sop1(aco_opcode::s_mov_b32, dst, src);
7067    } else if (src.regClass() == s2) {
7068       bld.sop1(aco_opcode::s_mov_b64, dst, src);
7069    } else {
7070       fprintf(stderr, "Unimplemented NIR instr bit size: ");
7071       nir_print_instr(&instr->instr, stderr);
7072       fprintf(stderr, "\n");
7073    }
7074 }
7075
7076 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
7077 {
7078    Builder bld(ctx->program, ctx->block);
7079    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
7080    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
7081    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
7082
7083    Temp ddx_1, ddx_2, ddy_1, ddy_2;
7084    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
7085    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
7086    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
7087
7088    /* Build DD X/Y */
7089    if (ctx->program->chip_class >= GFX8) {
7090       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
7091       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
7092       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
7093       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
7094       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
7095       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
7096    } else {
7097       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
7098       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
7099       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
7100       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
7101       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
7102       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
7103       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
7104       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
7105       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
7106       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
7107    }
7108
7109    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
7110    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
7111    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
7112    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
7113    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
7114    Temp wqm1 = bld.tmp(v1);
7115    emit_wqm(ctx, tmp1, wqm1, true);
7116    Temp wqm2 = bld.tmp(v1);
7117    emit_wqm(ctx, tmp2, wqm2, true);
7118    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
7119    return;
7120 }
7121
7122 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
7123 {
7124    Builder bld(ctx->program, ctx->block);
7125    switch(instr->intrinsic) {
7126    case nir_intrinsic_load_barycentric_sample:
7127    case nir_intrinsic_load_barycentric_pixel:
7128    case nir_intrinsic_load_barycentric_centroid: {
7129       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
7130       Temp bary = Temp(0, s2);
7131       switch (mode) {
7132       case INTERP_MODE_SMOOTH:
7133       case INTERP_MODE_NONE:
7134          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7135             bary = get_arg(ctx, ctx->args->ac.persp_center);
7136          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7137             bary = ctx->persp_centroid;
7138          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7139             bary = get_arg(ctx, ctx->args->ac.persp_sample);
7140          break;
7141       case INTERP_MODE_NOPERSPECTIVE:
7142          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
7143             bary = get_arg(ctx, ctx->args->ac.linear_center);
7144          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
7145             bary = ctx->linear_centroid;
7146          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
7147             bary = get_arg(ctx, ctx->args->ac.linear_sample);
7148          break;
7149       default:
7150          break;
7151       }
7152       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7153       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
7154       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
7155       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7156                  Operand(p1), Operand(p2));
7157       emit_split_vector(ctx, dst, 2);
7158       break;
7159    }
7160    case nir_intrinsic_load_barycentric_model: {
7161       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
7162
7163       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7164       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
7165       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
7166       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
7167       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7168                  Operand(p1), Operand(p2), Operand(p3));
7169       emit_split_vector(ctx, dst, 3);
7170       break;
7171    }
7172    case nir_intrinsic_load_barycentric_at_sample: {
7173       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
7174       switch (ctx->options->key.fs.num_samples) {
7175          case 2: sample_pos_offset += 1 << 3; break;
7176          case 4: sample_pos_offset += 3 << 3; break;
7177          case 8: sample_pos_offset += 7 << 3; break;
7178          default: break;
7179       }
7180       Temp sample_pos;
7181       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
7182       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
7183       Temp private_segment_buffer = ctx->program->private_segment_buffer;
7184       if (addr.type() == RegType::sgpr) {
7185          Operand offset;
7186          if (const_addr) {
7187             sample_pos_offset += const_addr->u32 << 3;
7188             offset = Operand(sample_pos_offset);
7189          } else if (ctx->options->chip_class >= GFX9) {
7190             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7191          } else {
7192             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
7193             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
7194          }
7195
7196          Operand off = bld.copy(bld.def(s1), Operand(offset));
7197          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
7198
7199       } else if (ctx->options->chip_class >= GFX9) {
7200          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7201          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
7202       } else if (ctx->options->chip_class >= GFX7) {
7203          /* addr += private_segment_buffer + sample_pos_offset */
7204          Temp tmp0 = bld.tmp(s1);
7205          Temp tmp1 = bld.tmp(s1);
7206          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
7207          Definition scc_tmp = bld.def(s1, scc);
7208          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
7209          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
7210          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7211          Temp pck0 = bld.tmp(v1);
7212          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
7213          tmp1 = as_vgpr(ctx, tmp1);
7214          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
7215          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
7216
7217          /* sample_pos = flat_load_dwordx2 addr */
7218          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
7219       } else {
7220          assert(ctx->options->chip_class == GFX6);
7221
7222          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
7223                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
7224          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
7225
7226          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
7227          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
7228
7229          sample_pos = bld.tmp(v2);
7230
7231          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
7232          load->definitions[0] = Definition(sample_pos);
7233          load->operands[0] = Operand(rsrc);
7234          load->operands[1] = Operand(addr);
7235          load->operands[2] = Operand(0u);
7236          load->offset = sample_pos_offset;
7237          load->offen = 0;
7238          load->addr64 = true;
7239          load->glc = false;
7240          load->dlc = false;
7241          load->disable_wqm = false;
7242          load->barrier = barrier_none;
7243          load->can_reorder = true;
7244          ctx->block->instructions.emplace_back(std::move(load));
7245       }
7246
7247       /* sample_pos -= 0.5 */
7248       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
7249       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
7250       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
7251       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
7252       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
7253
7254       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7255       break;
7256    }
7257    case nir_intrinsic_load_barycentric_at_offset: {
7258       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
7259       RegClass rc = RegClass(offset.type(), 1);
7260       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
7261       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
7262       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
7263       break;
7264    }
7265    case nir_intrinsic_load_front_face: {
7266       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7267                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
7268       break;
7269    }
7270    case nir_intrinsic_load_view_index: {
7271       if (ctx->stage & (sw_vs | sw_gs | sw_tcs | sw_tes)) {
7272          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7273          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
7274          break;
7275       }
7276
7277       /* fallthrough */
7278    }
7279    case nir_intrinsic_load_layer_id: {
7280       unsigned idx = nir_intrinsic_base(instr);
7281       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7282                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
7283       break;
7284    }
7285    case nir_intrinsic_load_frag_coord: {
7286       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
7287       break;
7288    }
7289    case nir_intrinsic_load_sample_pos: {
7290       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
7291       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
7292       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7293                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
7294                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
7295       break;
7296    }
7297    case nir_intrinsic_load_tess_coord:
7298       visit_load_tess_coord(ctx, instr);
7299       break;
7300    case nir_intrinsic_load_interpolated_input:
7301       visit_load_interpolated_input(ctx, instr);
7302       break;
7303    case nir_intrinsic_store_output:
7304       visit_store_output(ctx, instr);
7305       break;
7306    case nir_intrinsic_load_input:
7307    case nir_intrinsic_load_input_vertex:
7308       visit_load_input(ctx, instr);
7309       break;
7310    case nir_intrinsic_load_output:
7311       visit_load_output(ctx, instr);
7312       break;
7313    case nir_intrinsic_load_per_vertex_input:
7314       visit_load_per_vertex_input(ctx, instr);
7315       break;
7316    case nir_intrinsic_load_per_vertex_output:
7317       visit_load_per_vertex_output(ctx, instr);
7318       break;
7319    case nir_intrinsic_store_per_vertex_output:
7320       visit_store_per_vertex_output(ctx, instr);
7321       break;
7322    case nir_intrinsic_load_ubo:
7323       visit_load_ubo(ctx, instr);
7324       break;
7325    case nir_intrinsic_load_push_constant:
7326       visit_load_push_constant(ctx, instr);
7327       break;
7328    case nir_intrinsic_load_constant:
7329       visit_load_constant(ctx, instr);
7330       break;
7331    case nir_intrinsic_vulkan_resource_index:
7332       visit_load_resource(ctx, instr);
7333       break;
7334    case nir_intrinsic_discard:
7335       visit_discard(ctx, instr);
7336       break;
7337    case nir_intrinsic_discard_if:
7338       visit_discard_if(ctx, instr);
7339       break;
7340    case nir_intrinsic_load_shared:
7341       visit_load_shared(ctx, instr);
7342       break;
7343    case nir_intrinsic_store_shared:
7344       visit_store_shared(ctx, instr);
7345       break;
7346    case nir_intrinsic_shared_atomic_add:
7347    case nir_intrinsic_shared_atomic_imin:
7348    case nir_intrinsic_shared_atomic_umin:
7349    case nir_intrinsic_shared_atomic_imax:
7350    case nir_intrinsic_shared_atomic_umax:
7351    case nir_intrinsic_shared_atomic_and:
7352    case nir_intrinsic_shared_atomic_or:
7353    case nir_intrinsic_shared_atomic_xor:
7354    case nir_intrinsic_shared_atomic_exchange:
7355    case nir_intrinsic_shared_atomic_comp_swap:
7356       visit_shared_atomic(ctx, instr);
7357       break;
7358    case nir_intrinsic_image_deref_load:
7359       visit_image_load(ctx, instr);
7360       break;
7361    case nir_intrinsic_image_deref_store:
7362       visit_image_store(ctx, instr);
7363       break;
7364    case nir_intrinsic_image_deref_atomic_add:
7365    case nir_intrinsic_image_deref_atomic_umin:
7366    case nir_intrinsic_image_deref_atomic_imin:
7367    case nir_intrinsic_image_deref_atomic_umax:
7368    case nir_intrinsic_image_deref_atomic_imax:
7369    case nir_intrinsic_image_deref_atomic_and:
7370    case nir_intrinsic_image_deref_atomic_or:
7371    case nir_intrinsic_image_deref_atomic_xor:
7372    case nir_intrinsic_image_deref_atomic_exchange:
7373    case nir_intrinsic_image_deref_atomic_comp_swap:
7374       visit_image_atomic(ctx, instr);
7375       break;
7376    case nir_intrinsic_image_deref_size:
7377       visit_image_size(ctx, instr);
7378       break;
7379    case nir_intrinsic_load_ssbo:
7380       visit_load_ssbo(ctx, instr);
7381       break;
7382    case nir_intrinsic_store_ssbo:
7383       visit_store_ssbo(ctx, instr);
7384       break;
7385    case nir_intrinsic_load_global:
7386       visit_load_global(ctx, instr);
7387       break;
7388    case nir_intrinsic_store_global:
7389       visit_store_global(ctx, instr);
7390       break;
7391    case nir_intrinsic_global_atomic_add:
7392    case nir_intrinsic_global_atomic_imin:
7393    case nir_intrinsic_global_atomic_umin:
7394    case nir_intrinsic_global_atomic_imax:
7395    case nir_intrinsic_global_atomic_umax:
7396    case nir_intrinsic_global_atomic_and:
7397    case nir_intrinsic_global_atomic_or:
7398    case nir_intrinsic_global_atomic_xor:
7399    case nir_intrinsic_global_atomic_exchange:
7400    case nir_intrinsic_global_atomic_comp_swap:
7401       visit_global_atomic(ctx, instr);
7402       break;
7403    case nir_intrinsic_ssbo_atomic_add:
7404    case nir_intrinsic_ssbo_atomic_imin:
7405    case nir_intrinsic_ssbo_atomic_umin:
7406    case nir_intrinsic_ssbo_atomic_imax:
7407    case nir_intrinsic_ssbo_atomic_umax:
7408    case nir_intrinsic_ssbo_atomic_and:
7409    case nir_intrinsic_ssbo_atomic_or:
7410    case nir_intrinsic_ssbo_atomic_xor:
7411    case nir_intrinsic_ssbo_atomic_exchange:
7412    case nir_intrinsic_ssbo_atomic_comp_swap:
7413       visit_atomic_ssbo(ctx, instr);
7414       break;
7415    case nir_intrinsic_load_scratch:
7416       visit_load_scratch(ctx, instr);
7417       break;
7418    case nir_intrinsic_store_scratch:
7419       visit_store_scratch(ctx, instr);
7420       break;
7421    case nir_intrinsic_get_buffer_size:
7422       visit_get_buffer_size(ctx, instr);
7423       break;
7424    case nir_intrinsic_control_barrier: {
7425       if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7426          /* GFX6 only (thanks to a hw bug workaround):
7427           * The real barrier instruction isn’t needed, because an entire patch
7428           * always fits into a single wave.
7429           */
7430          break;
7431       }
7432
7433       if (ctx->program->workgroup_size > ctx->program->wave_size)
7434          bld.sopp(aco_opcode::s_barrier);
7435
7436       break;
7437    }
7438    case nir_intrinsic_memory_barrier_tcs_patch:
7439    case nir_intrinsic_group_memory_barrier:
7440    case nir_intrinsic_memory_barrier:
7441    case nir_intrinsic_memory_barrier_buffer:
7442    case nir_intrinsic_memory_barrier_image:
7443    case nir_intrinsic_memory_barrier_shared:
7444       emit_memory_barrier(ctx, instr);
7445       break;
7446    case nir_intrinsic_load_num_work_groups: {
7447       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7448       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
7449       emit_split_vector(ctx, dst, 3);
7450       break;
7451    }
7452    case nir_intrinsic_load_local_invocation_id: {
7453       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7454       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
7455       emit_split_vector(ctx, dst, 3);
7456       break;
7457    }
7458    case nir_intrinsic_load_work_group_id: {
7459       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7460       struct ac_arg *args = ctx->args->ac.workgroup_ids;
7461       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
7462                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
7463                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
7464                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
7465       emit_split_vector(ctx, dst, 3);
7466       break;
7467    }
7468    case nir_intrinsic_load_local_invocation_index: {
7469       Temp id = emit_mbcnt(ctx, bld.def(v1));
7470
7471       /* The tg_size bits [6:11] contain the subgroup id,
7472        * we need this multiplied by the wave size, and then OR the thread id to it.
7473        */
7474       if (ctx->program->wave_size == 64) {
7475          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
7476          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
7477                                 get_arg(ctx, ctx->args->ac.tg_size));
7478          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
7479       } else {
7480          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
7481          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7482                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7483          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
7484       }
7485       break;
7486    }
7487    case nir_intrinsic_load_subgroup_id: {
7488       if (ctx->stage == compute_cs) {
7489          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
7490                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
7491       } else {
7492          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
7493       }
7494       break;
7495    }
7496    case nir_intrinsic_load_subgroup_invocation: {
7497       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
7498       break;
7499    }
7500    case nir_intrinsic_load_num_subgroups: {
7501       if (ctx->stage == compute_cs)
7502          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
7503                   get_arg(ctx, ctx->args->ac.tg_size));
7504       else
7505          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
7506       break;
7507    }
7508    case nir_intrinsic_ballot: {
7509       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7510       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7511       Definition tmp = bld.def(dst.regClass());
7512       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
7513       if (instr->src[0].ssa->bit_size == 1) {
7514          assert(src.regClass() == bld.lm);
7515          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
7516       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
7517          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
7518       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
7519          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
7520       } else {
7521          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7522          nir_print_instr(&instr->instr, stderr);
7523          fprintf(stderr, "\n");
7524       }
7525       if (dst.size() != bld.lm.size()) {
7526          /* Wave32 with ballot size set to 64 */
7527          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
7528       }
7529       emit_wqm(ctx, tmp.getTemp(), dst);
7530       break;
7531    }
7532    case nir_intrinsic_shuffle:
7533    case nir_intrinsic_read_invocation: {
7534       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7535       if (!nir_src_is_divergent(instr->src[0])) {
7536          emit_uniform_subgroup(ctx, instr, src);
7537       } else {
7538          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
7539          if (instr->intrinsic == nir_intrinsic_read_invocation || !nir_src_is_divergent(instr->src[1]))
7540             tid = bld.as_uniform(tid);
7541          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7542          if (src.regClass() == v1) {
7543             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
7544          } else if (src.regClass() == v2) {
7545             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7546             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7547             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
7548             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
7549             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7550             emit_split_vector(ctx, dst, 2);
7551          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
7552             assert(src.regClass() == bld.lm);
7553             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
7554             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7555          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
7556             assert(src.regClass() == bld.lm);
7557             Temp tmp;
7558             if (ctx->program->chip_class <= GFX7)
7559                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
7560             else if (ctx->program->wave_size == 64)
7561                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
7562             else
7563                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
7564             tmp = emit_extract_vector(ctx, tmp, 0, v1);
7565             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
7566             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
7567          } else {
7568             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7569             nir_print_instr(&instr->instr, stderr);
7570             fprintf(stderr, "\n");
7571          }
7572       }
7573       break;
7574    }
7575    case nir_intrinsic_load_sample_id: {
7576       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7577                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
7578       break;
7579    }
7580    case nir_intrinsic_load_sample_mask_in: {
7581       visit_load_sample_mask_in(ctx, instr);
7582       break;
7583    }
7584    case nir_intrinsic_read_first_invocation: {
7585       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7586       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7587       if (src.regClass() == v1b || src.regClass() == v2b || src.regClass() == v1) {
7588          emit_wqm(ctx,
7589                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
7590                   dst);
7591       } else if (src.regClass() == v2) {
7592          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7593          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7594          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
7595          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
7596          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7597          emit_split_vector(ctx, dst, 2);
7598       } else if (instr->dest.ssa.bit_size == 1) {
7599          assert(src.regClass() == bld.lm);
7600          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
7601                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
7602          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7603       } else if (src.regClass() == s1) {
7604          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
7605       } else if (src.regClass() == s2) {
7606          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
7607       } else {
7608          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7609          nir_print_instr(&instr->instr, stderr);
7610          fprintf(stderr, "\n");
7611       }
7612       break;
7613    }
7614    case nir_intrinsic_vote_all: {
7615       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7616       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7617       assert(src.regClass() == bld.lm);
7618       assert(dst.regClass() == bld.lm);
7619
7620       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
7621       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
7622       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
7623       break;
7624    }
7625    case nir_intrinsic_vote_any: {
7626       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7627       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7628       assert(src.regClass() == bld.lm);
7629       assert(dst.regClass() == bld.lm);
7630
7631       Temp tmp = bool_to_scalar_condition(ctx, src);
7632       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7633       break;
7634    }
7635    case nir_intrinsic_reduce:
7636    case nir_intrinsic_inclusive_scan:
7637    case nir_intrinsic_exclusive_scan: {
7638       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7639       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7640       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
7641       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
7642          nir_intrinsic_cluster_size(instr) : 0;
7643       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
7644
7645       if (!nir_src_is_divergent(instr->src[0]) && (op == nir_op_ior || op == nir_op_iand)) {
7646          emit_uniform_subgroup(ctx, instr, src);
7647       } else if (instr->dest.ssa.bit_size == 1) {
7648          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
7649             op = nir_op_iand;
7650          else if (op == nir_op_iadd)
7651             op = nir_op_ixor;
7652          else if (op == nir_op_umax || op == nir_op_imax)
7653             op = nir_op_ior;
7654          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
7655
7656          switch (instr->intrinsic) {
7657          case nir_intrinsic_reduce:
7658             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
7659             break;
7660          case nir_intrinsic_exclusive_scan:
7661             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
7662             break;
7663          case nir_intrinsic_inclusive_scan:
7664             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
7665             break;
7666          default:
7667             assert(false);
7668          }
7669       } else if (cluster_size == 1) {
7670          bld.copy(Definition(dst), src);
7671       } else {
7672          unsigned bit_size = instr->src[0].ssa->bit_size;
7673
7674          src = emit_extract_vector(ctx, src, 0, RegClass::get(RegType::vgpr, bit_size / 8));
7675
7676          ReduceOp reduce_op;
7677          switch (op) {
7678          #define CASEI(name) case nir_op_##name: reduce_op = (bit_size == 32) ? name##32 : (bit_size == 16) ? name##16 : (bit_size == 8) ? name##8 : name##64; break;
7679          #define CASEF(name) case nir_op_##name: reduce_op = (bit_size == 32) ? name##32 : (bit_size == 16) ? name##16 : name##64; break;
7680             CASEI(iadd)
7681             CASEI(imul)
7682             CASEI(imin)
7683             CASEI(umin)
7684             CASEI(imax)
7685             CASEI(umax)
7686             CASEI(iand)
7687             CASEI(ior)
7688             CASEI(ixor)
7689             CASEF(fadd)
7690             CASEF(fmul)
7691             CASEF(fmin)
7692             CASEF(fmax)
7693             default:
7694                unreachable("unknown reduction op");
7695          #undef CASEI
7696          #undef CASEF
7697          }
7698
7699          aco_opcode aco_op;
7700          switch (instr->intrinsic) {
7701             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
7702             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
7703             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
7704             default:
7705                unreachable("unknown reduce intrinsic");
7706          }
7707
7708          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
7709          reduce->operands[0] = Operand(src);
7710          // filled in by aco_reduce_assign.cpp, used internally as part of the
7711          // reduce sequence
7712          assert(dst.size() == 1 || dst.size() == 2);
7713          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
7714          reduce->operands[2] = Operand(v1.as_linear());
7715
7716          Temp tmp_dst = bld.tmp(dst.regClass());
7717          reduce->definitions[0] = Definition(tmp_dst);
7718          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
7719          reduce->definitions[2] = Definition();
7720          reduce->definitions[3] = Definition(scc, s1);
7721          reduce->definitions[4] = Definition();
7722          reduce->reduce_op = reduce_op;
7723          reduce->cluster_size = cluster_size;
7724          ctx->block->instructions.emplace_back(std::move(reduce));
7725
7726          emit_wqm(ctx, tmp_dst, dst);
7727       }
7728       break;
7729    }
7730    case nir_intrinsic_quad_broadcast: {
7731       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7732       if (!nir_dest_is_divergent(instr->dest)) {
7733          emit_uniform_subgroup(ctx, instr, src);
7734       } else {
7735          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7736          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
7737          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
7738
7739          if (instr->dest.ssa.bit_size == 1) {
7740             assert(src.regClass() == bld.lm);
7741             assert(dst.regClass() == bld.lm);
7742             uint32_t half_mask = 0x11111111u << lane;
7743             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
7744             Temp tmp = bld.tmp(bld.lm);
7745             bld.sop1(Builder::s_wqm, Definition(tmp),
7746                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
7747                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
7748             emit_wqm(ctx, tmp, dst);
7749          } else if (instr->dest.ssa.bit_size == 32) {
7750             if (ctx->program->chip_class >= GFX8)
7751                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
7752             else
7753                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
7754          } else if (instr->dest.ssa.bit_size == 64) {
7755             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7756             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7757             if (ctx->program->chip_class >= GFX8) {
7758                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7759                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7760             } else {
7761                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
7762                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
7763             }
7764             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7765             emit_split_vector(ctx, dst, 2);
7766          } else {
7767             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7768             nir_print_instr(&instr->instr, stderr);
7769             fprintf(stderr, "\n");
7770          }
7771       }
7772       break;
7773    }
7774    case nir_intrinsic_quad_swap_horizontal:
7775    case nir_intrinsic_quad_swap_vertical:
7776    case nir_intrinsic_quad_swap_diagonal:
7777    case nir_intrinsic_quad_swizzle_amd: {
7778       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7779       if (!nir_dest_is_divergent(instr->dest)) {
7780          emit_uniform_subgroup(ctx, instr, src);
7781          break;
7782       }
7783       uint16_t dpp_ctrl = 0;
7784       switch (instr->intrinsic) {
7785       case nir_intrinsic_quad_swap_horizontal:
7786          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
7787          break;
7788       case nir_intrinsic_quad_swap_vertical:
7789          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
7790          break;
7791       case nir_intrinsic_quad_swap_diagonal:
7792          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
7793          break;
7794       case nir_intrinsic_quad_swizzle_amd:
7795          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
7796          break;
7797       default:
7798          break;
7799       }
7800       if (ctx->program->chip_class < GFX8)
7801          dpp_ctrl |= (1 << 15);
7802
7803       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7804       if (instr->dest.ssa.bit_size == 1) {
7805          assert(src.regClass() == bld.lm);
7806          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
7807          if (ctx->program->chip_class >= GFX8)
7808             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7809          else
7810             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7811          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
7812          emit_wqm(ctx, tmp, dst);
7813       } else if (instr->dest.ssa.bit_size == 32) {
7814          Temp tmp;
7815          if (ctx->program->chip_class >= GFX8)
7816             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7817          else
7818             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7819          emit_wqm(ctx, tmp, dst);
7820       } else if (instr->dest.ssa.bit_size == 64) {
7821          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7822          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7823          if (ctx->program->chip_class >= GFX8) {
7824             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7825             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7826          } else {
7827             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
7828             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
7829          }
7830          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7831          emit_split_vector(ctx, dst, 2);
7832       } else {
7833          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7834          nir_print_instr(&instr->instr, stderr);
7835          fprintf(stderr, "\n");
7836       }
7837       break;
7838    }
7839    case nir_intrinsic_masked_swizzle_amd: {
7840       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7841       if (!nir_dest_is_divergent(instr->dest)) {
7842          emit_uniform_subgroup(ctx, instr, src);
7843          break;
7844       }
7845       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7846       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
7847       if (dst.regClass() == v1) {
7848          emit_wqm(ctx,
7849                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
7850                   dst);
7851       } else if (dst.regClass() == v2) {
7852          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7853          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7854          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
7855          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
7856          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7857          emit_split_vector(ctx, dst, 2);
7858       } else {
7859          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7860          nir_print_instr(&instr->instr, stderr);
7861          fprintf(stderr, "\n");
7862       }
7863       break;
7864    }
7865    case nir_intrinsic_write_invocation_amd: {
7866       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
7867       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
7868       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
7869       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7870       if (dst.regClass() == v1) {
7871          /* src2 is ignored for writelane. RA assigns the same reg for dst */
7872          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
7873       } else if (dst.regClass() == v2) {
7874          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
7875          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
7876          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
7877          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
7878          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
7879          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
7880          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7881          emit_split_vector(ctx, dst, 2);
7882       } else {
7883          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7884          nir_print_instr(&instr->instr, stderr);
7885          fprintf(stderr, "\n");
7886       }
7887       break;
7888    }
7889    case nir_intrinsic_mbcnt_amd: {
7890       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7891       RegClass rc = RegClass(src.type(), 1);
7892       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
7893       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
7894       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7895       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
7896       emit_wqm(ctx, wqm_tmp, dst);
7897       break;
7898    }
7899    case nir_intrinsic_load_helper_invocation: {
7900       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7901       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
7902       ctx->block->kind |= block_kind_needs_lowering;
7903       ctx->program->needs_exact = true;
7904       break;
7905    }
7906    case nir_intrinsic_is_helper_invocation: {
7907       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7908       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
7909       ctx->block->kind |= block_kind_needs_lowering;
7910       ctx->program->needs_exact = true;
7911       break;
7912    }
7913    case nir_intrinsic_demote:
7914       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
7915
7916       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7917          ctx->cf_info.exec_potentially_empty_discard = true;
7918       ctx->block->kind |= block_kind_uses_demote;
7919       ctx->program->needs_exact = true;
7920       break;
7921    case nir_intrinsic_demote_if: {
7922       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7923       assert(src.regClass() == bld.lm);
7924       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
7925       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
7926
7927       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7928          ctx->cf_info.exec_potentially_empty_discard = true;
7929       ctx->block->kind |= block_kind_uses_demote;
7930       ctx->program->needs_exact = true;
7931       break;
7932    }
7933    case nir_intrinsic_first_invocation: {
7934       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
7935                get_ssa_temp(ctx, &instr->dest.ssa));
7936       break;
7937    }
7938    case nir_intrinsic_shader_clock:
7939       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
7940       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
7941       break;
7942    case nir_intrinsic_load_vertex_id_zero_base: {
7943       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7944       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
7945       break;
7946    }
7947    case nir_intrinsic_load_first_vertex: {
7948       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7949       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
7950       break;
7951    }
7952    case nir_intrinsic_load_base_instance: {
7953       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7954       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
7955       break;
7956    }
7957    case nir_intrinsic_load_instance_id: {
7958       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7959       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
7960       break;
7961    }
7962    case nir_intrinsic_load_draw_id: {
7963       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7964       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
7965       break;
7966    }
7967    case nir_intrinsic_load_invocation_id: {
7968       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7969
7970       if (ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
7971          if (ctx->options->chip_class >= GFX10)
7972             bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7973          else
7974             bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7975       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7976          bld.vop3(aco_opcode::v_bfe_u32, Definition(dst),
7977                   get_arg(ctx, ctx->args->ac.tcs_rel_ids), Operand(8u), Operand(5u));
7978       } else {
7979          unreachable("Unsupported stage for load_invocation_id");
7980       }
7981
7982       break;
7983    }
7984    case nir_intrinsic_load_primitive_id: {
7985       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7986
7987       switch (ctx->shader->info.stage) {
7988       case MESA_SHADER_GEOMETRY:
7989          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
7990          break;
7991       case MESA_SHADER_TESS_CTRL:
7992          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tcs_patch_id));
7993          break;
7994       case MESA_SHADER_TESS_EVAL:
7995          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tes_patch_id));
7996          break;
7997       default:
7998          unreachable("Unimplemented shader stage for nir_intrinsic_load_primitive_id");
7999       }
8000
8001       break;
8002    }
8003    case nir_intrinsic_load_patch_vertices_in: {
8004       assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL ||
8005              ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
8006
8007       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8008       bld.copy(Definition(dst), Operand(ctx->args->options->key.tcs.input_vertices));
8009       break;
8010    }
8011    case nir_intrinsic_emit_vertex_with_counter: {
8012       visit_emit_vertex_with_counter(ctx, instr);
8013       break;
8014    }
8015    case nir_intrinsic_end_primitive_with_counter: {
8016       unsigned stream = nir_intrinsic_stream_id(instr);
8017       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
8018       break;
8019    }
8020    case nir_intrinsic_set_vertex_count: {
8021       /* unused, the HW keeps track of this for us */
8022       break;
8023    }
8024    default:
8025       fprintf(stderr, "Unimplemented intrinsic instr: ");
8026       nir_print_instr(&instr->instr, stderr);
8027       fprintf(stderr, "\n");
8028       abort();
8029
8030       break;
8031    }
8032 }
8033
8034
8035 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
8036                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
8037                     enum glsl_base_type *stype)
8038 {
8039    nir_deref_instr *texture_deref_instr = NULL;
8040    nir_deref_instr *sampler_deref_instr = NULL;
8041    int plane = -1;
8042
8043    for (unsigned i = 0; i < instr->num_srcs; i++) {
8044       switch (instr->src[i].src_type) {
8045       case nir_tex_src_texture_deref:
8046          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
8047          break;
8048       case nir_tex_src_sampler_deref:
8049          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
8050          break;
8051       case nir_tex_src_plane:
8052          plane = nir_src_as_int(instr->src[i].src);
8053          break;
8054       default:
8055          break;
8056       }
8057    }
8058
8059    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
8060
8061    if (!sampler_deref_instr)
8062       sampler_deref_instr = texture_deref_instr;
8063
8064    if (plane >= 0) {
8065       assert(instr->op != nir_texop_txf_ms &&
8066              instr->op != nir_texop_samples_identical);
8067       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
8068       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
8069    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
8070       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
8071    } else if (instr->op == nir_texop_fragment_mask_fetch) {
8072       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8073    } else {
8074       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
8075    }
8076    if (samp_ptr) {
8077       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
8078
8079       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
8080          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
8081          Builder bld(ctx->program, ctx->block);
8082
8083          /* to avoid unnecessary moves, we split and recombine sampler and image */
8084          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
8085                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8086          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
8087          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
8088                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
8089                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
8090          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
8091                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
8092
8093          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
8094          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
8095                                img[0], img[1], img[2], img[3],
8096                                img[4], img[5], img[6], img[7]);
8097          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
8098                                 samp[0], samp[1], samp[2], samp[3]);
8099       }
8100    }
8101    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
8102                      instr->op == nir_texop_samples_identical))
8103       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
8104 }
8105
8106 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
8107                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
8108 {
8109    Builder bld(ctx->program, ctx->block);
8110
8111    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
8112    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
8113    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
8114
8115    Operand neg_one(0xbf800000u);
8116    Operand one(0x3f800000u);
8117    Operand two(0x40000000u);
8118    Operand four(0x40800000u);
8119
8120    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
8121    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
8122    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
8123
8124    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
8125    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
8126    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
8127    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
8128
8129    // select sc
8130    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
8131    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
8132                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
8133                        one, is_ma_y);
8134    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8135
8136    // select tc
8137    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
8138    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
8139    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
8140
8141    // select ma
8142    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8143                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
8144                   deriv_z, is_ma_z);
8145    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
8146    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
8147 }
8148
8149 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
8150 {
8151    Builder bld(ctx->program, ctx->block);
8152    Temp ma, tc, sc, id;
8153
8154    if (is_array) {
8155       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
8156
8157       // see comment in ac_prepare_cube_coords()
8158       if (ctx->options->chip_class <= GFX8)
8159          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
8160    }
8161
8162    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8163
8164    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
8165    vop3a->operands[0] = Operand(ma);
8166    vop3a->abs[0] = true;
8167    Temp invma = bld.tmp(v1);
8168    vop3a->definitions[0] = Definition(invma);
8169    ctx->block->instructions.emplace_back(std::move(vop3a));
8170
8171    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8172    if (!is_deriv)
8173       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
8174
8175    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8176    if (!is_deriv)
8177       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
8178
8179    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
8180
8181    if (is_deriv) {
8182       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
8183       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
8184
8185       for (unsigned i = 0; i < 2; i++) {
8186          // see comment in ac_prepare_cube_coords()
8187          Temp deriv_ma;
8188          Temp deriv_sc, deriv_tc;
8189          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
8190                            &deriv_ma, &deriv_sc, &deriv_tc);
8191
8192          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
8193
8194          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8195                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
8196                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
8197          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
8198                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
8199                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
8200          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
8201       }
8202
8203       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
8204       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
8205    }
8206
8207    if (is_array)
8208       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
8209    coords.resize(3);
8210    coords[0] = sc;
8211    coords[1] = tc;
8212    coords[2] = id;
8213 }
8214
8215 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
8216 {
8217    if (vec->parent_instr->type != nir_instr_type_alu)
8218       return;
8219    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
8220    if (vec_instr->op != nir_op_vec(vec->num_components))
8221       return;
8222
8223    for (unsigned i = 0; i < vec->num_components; i++) {
8224       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
8225               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
8226    }
8227 }
8228
8229 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
8230 {
8231    Builder bld(ctx->program, ctx->block);
8232    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
8233         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false,
8234         has_clamped_lod = false;
8235    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
8236         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(),
8237         clamped_lod = Temp();
8238    std::vector<Temp> coords;
8239    std::vector<Temp> derivs;
8240    nir_const_value *sample_index_cv = NULL;
8241    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
8242    enum glsl_base_type stype;
8243    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
8244
8245    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
8246                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
8247    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
8248                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
8249
8250    for (unsigned i = 0; i < instr->num_srcs; i++) {
8251       switch (instr->src[i].src_type) {
8252       case nir_tex_src_coord: {
8253          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
8254          for (unsigned i = 0; i < coord.size(); i++)
8255             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
8256          break;
8257       }
8258       case nir_tex_src_bias:
8259          bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
8260          has_bias = true;
8261          break;
8262       case nir_tex_src_lod: {
8263          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
8264
8265          if (val && val->f32 <= 0.0) {
8266             level_zero = true;
8267          } else {
8268             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
8269             has_lod = true;
8270          }
8271          break;
8272       }
8273       case nir_tex_src_min_lod:
8274          clamped_lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
8275          has_clamped_lod = true;
8276          break;
8277       case nir_tex_src_comparator:
8278          if (instr->is_shadow) {
8279             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
8280             has_compare = true;
8281          }
8282          break;
8283       case nir_tex_src_offset:
8284          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
8285          get_const_vec(instr->src[i].src.ssa, const_offset);
8286          has_offset = true;
8287          break;
8288       case nir_tex_src_ddx:
8289          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
8290          has_ddx = true;
8291          break;
8292       case nir_tex_src_ddy:
8293          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
8294          has_ddy = true;
8295          break;
8296       case nir_tex_src_ms_index:
8297          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
8298          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
8299          has_sample_index = true;
8300          break;
8301       case nir_tex_src_texture_offset:
8302       case nir_tex_src_sampler_offset:
8303       default:
8304          break;
8305       }
8306    }
8307
8308    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
8309       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
8310
8311    if (instr->op == nir_texop_texture_samples) {
8312       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
8313
8314       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
8315       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
8316       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
8317
8318       Operand default_sample = Operand(1u);
8319       if (ctx->options->robust_buffer_access) {
8320          /* Extract the second dword of the descriptor, if it's
8321           * all zero, then it's a null descriptor.
8322           */
8323          Temp dword1 = emit_extract_vector(ctx, resource, 1, s1);
8324          Temp is_non_null_descriptor = bld.sopc(aco_opcode::s_cmp_gt_u32, bld.def(s1, scc), dword1, Operand(0u));
8325          default_sample = Operand(is_non_null_descriptor);
8326       }
8327
8328       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
8329       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
8330                samples, default_sample, bld.scc(is_msaa));
8331       return;
8332    }
8333
8334    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
8335       aco_ptr<Instruction> tmp_instr;
8336       Temp acc, pack = Temp();
8337
8338       uint32_t pack_const = 0;
8339       for (unsigned i = 0; i < offset.size(); i++) {
8340          if (!const_offset[i])
8341             continue;
8342          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
8343       }
8344
8345       if (offset.type() == RegType::sgpr) {
8346          for (unsigned i = 0; i < offset.size(); i++) {
8347             if (const_offset[i])
8348                continue;
8349
8350             acc = emit_extract_vector(ctx, offset, i, s1);
8351             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
8352
8353             if (i) {
8354                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
8355             }
8356
8357             if (pack == Temp()) {
8358                pack = acc;
8359             } else {
8360                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
8361             }
8362          }
8363
8364          if (pack_const && pack != Temp())
8365             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
8366       } else {
8367          for (unsigned i = 0; i < offset.size(); i++) {
8368             if (const_offset[i])
8369                continue;
8370
8371             acc = emit_extract_vector(ctx, offset, i, v1);
8372             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
8373
8374             if (i) {
8375                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
8376             }
8377
8378             if (pack == Temp()) {
8379                pack = acc;
8380             } else {
8381                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
8382             }
8383          }
8384
8385          if (pack_const && pack != Temp())
8386             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
8387       }
8388       if (pack_const && pack == Temp())
8389          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
8390       else if (pack == Temp())
8391          has_offset = false;
8392       else
8393          offset = pack;
8394    }
8395
8396    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
8397       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
8398
8399    /* pack derivatives */
8400    if (has_ddx || has_ddy) {
8401       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
8402          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
8403          Temp zero = bld.copy(bld.def(v1), Operand(0u));
8404          derivs = {ddx, zero, ddy, zero};
8405       } else {
8406          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
8407             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
8408          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
8409             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
8410       }
8411       has_derivs = true;
8412    }
8413
8414    if (instr->coord_components > 1 &&
8415        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8416        instr->is_array &&
8417        instr->op != nir_texop_txf)
8418       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
8419
8420    if (instr->coord_components > 2 &&
8421       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
8422        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8423        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
8424        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8425        instr->is_array &&
8426        instr->op != nir_texop_txf &&
8427        instr->op != nir_texop_txf_ms &&
8428        instr->op != nir_texop_fragment_fetch &&
8429        instr->op != nir_texop_fragment_mask_fetch)
8430       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
8431
8432    if (ctx->options->chip_class == GFX9 &&
8433        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8434        instr->op != nir_texop_lod && instr->coord_components) {
8435       assert(coords.size() > 0 && coords.size() < 3);
8436
8437       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
8438                                                                      Operand((uint32_t) 0) :
8439                                                                      Operand((uint32_t) 0x3f000000)));
8440    }
8441
8442    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
8443
8444    if (instr->op == nir_texop_samples_identical)
8445       resource = fmask_ptr;
8446
8447    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
8448              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
8449             instr->op != nir_texop_txs &&
8450             instr->op != nir_texop_fragment_fetch &&
8451             instr->op != nir_texop_fragment_mask_fetch) {
8452       assert(has_sample_index);
8453       Operand op(sample_index);
8454       if (sample_index_cv)
8455          op = Operand(sample_index_cv->u32);
8456       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
8457    }
8458
8459    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
8460       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
8461          Temp off = emit_extract_vector(ctx, offset, i, v1);
8462          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
8463       }
8464       has_offset = false;
8465    }
8466
8467    /* Build tex instruction */
8468    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
8469    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
8470                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
8471                   : 0;
8472    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8473    Temp tmp_dst = dst;
8474
8475    /* gather4 selects the component by dmask and always returns vec4 */
8476    if (instr->op == nir_texop_tg4) {
8477       assert(instr->dest.ssa.num_components == 4);
8478       if (instr->is_shadow)
8479          dmask = 1;
8480       else
8481          dmask = 1 << instr->component;
8482       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
8483          tmp_dst = bld.tmp(v4);
8484    } else if (instr->op == nir_texop_samples_identical) {
8485       tmp_dst = bld.tmp(v1);
8486    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
8487       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
8488    }
8489
8490    aco_ptr<MIMG_instruction> tex;
8491    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
8492       if (!has_lod)
8493          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8494
8495       bool div_by_6 = instr->op == nir_texop_txs &&
8496                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
8497                       instr->is_array &&
8498                       (dmask & (1 << 2));
8499       if (tmp_dst.id() == dst.id() && div_by_6)
8500          tmp_dst = bld.tmp(tmp_dst.regClass());
8501
8502       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8503       tex->operands[0] = Operand(resource);
8504       tex->operands[1] = Operand(s4); /* no sampler */
8505       tex->operands[2] = Operand(as_vgpr(ctx,lod));
8506       if (ctx->options->chip_class == GFX9 &&
8507           instr->op == nir_texop_txs &&
8508           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
8509           instr->is_array) {
8510          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
8511       } else if (instr->op == nir_texop_query_levels) {
8512          tex->dmask = 1 << 3;
8513       } else {
8514          tex->dmask = dmask;
8515       }
8516       tex->da = da;
8517       tex->definitions[0] = Definition(tmp_dst);
8518       tex->dim = dim;
8519       tex->can_reorder = true;
8520       ctx->block->instructions.emplace_back(std::move(tex));
8521
8522       if (div_by_6) {
8523          /* divide 3rd value by 6 by multiplying with magic number */
8524          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8525          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
8526          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
8527          assert(instr->dest.ssa.num_components == 3);
8528          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
8529          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8530                               emit_extract_vector(ctx, tmp_dst, 0, v1),
8531                               emit_extract_vector(ctx, tmp_dst, 1, v1),
8532                               by_6);
8533
8534       }
8535
8536       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8537       return;
8538    }
8539
8540    Temp tg4_compare_cube_wa64 = Temp();
8541
8542    if (tg4_integer_workarounds) {
8543       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
8544       tex->operands[0] = Operand(resource);
8545       tex->operands[1] = Operand(s4); /* no sampler */
8546       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
8547       tex->dim = dim;
8548       tex->dmask = 0x3;
8549       tex->da = da;
8550       Temp size = bld.tmp(v2);
8551       tex->definitions[0] = Definition(size);
8552       tex->can_reorder = true;
8553       ctx->block->instructions.emplace_back(std::move(tex));
8554       emit_split_vector(ctx, size, size.size());
8555
8556       Temp half_texel[2];
8557       for (unsigned i = 0; i < 2; i++) {
8558          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
8559          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
8560          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
8561          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
8562       }
8563
8564       Temp new_coords[2] = {
8565          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
8566          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
8567       };
8568
8569       if (tg4_integer_cube_workaround) {
8570          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
8571          Temp desc[resource.size()];
8572          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
8573                                                                            Format::PSEUDO, 1, resource.size())};
8574          split->operands[0] = Operand(resource);
8575          for (unsigned i = 0; i < resource.size(); i++) {
8576             desc[i] = bld.tmp(s1);
8577             split->definitions[i] = Definition(desc[i]);
8578          }
8579          ctx->block->instructions.emplace_back(std::move(split));
8580
8581          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
8582          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
8583                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
8584
8585          Temp nfmt;
8586          if (stype == GLSL_TYPE_UINT) {
8587             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8588                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
8589                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
8590                             bld.scc(compare_cube_wa));
8591          } else {
8592             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8593                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
8594                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
8595                             bld.scc(compare_cube_wa));
8596          }
8597          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
8598          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
8599
8600          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
8601
8602          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
8603                             Operand((uint32_t)C_008F14_NUM_FORMAT));
8604          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
8605
8606          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
8607                                                                          Format::PSEUDO, resource.size(), 1)};
8608          for (unsigned i = 0; i < resource.size(); i++)
8609             vec->operands[i] = Operand(desc[i]);
8610          resource = bld.tmp(resource.regClass());
8611          vec->definitions[0] = Definition(resource);
8612          ctx->block->instructions.emplace_back(std::move(vec));
8613
8614          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8615                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
8616          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8617                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
8618       }
8619       coords[0] = new_coords[0];
8620       coords[1] = new_coords[1];
8621    }
8622
8623    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
8624       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
8625
8626       assert(coords.size() == 1);
8627       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
8628       aco_opcode op;
8629       switch (last_bit) {
8630       case 1:
8631          op = aco_opcode::buffer_load_format_x; break;
8632       case 2:
8633          op = aco_opcode::buffer_load_format_xy; break;
8634       case 3:
8635          op = aco_opcode::buffer_load_format_xyz; break;
8636       case 4:
8637          op = aco_opcode::buffer_load_format_xyzw; break;
8638       default:
8639          unreachable("Tex instruction loads more than 4 components.");
8640       }
8641
8642       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
8643       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
8644          tmp_dst = dst;
8645       else
8646          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
8647
8648       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
8649       mubuf->operands[0] = Operand(resource);
8650       mubuf->operands[1] = Operand(coords[0]);
8651       mubuf->operands[2] = Operand((uint32_t) 0);
8652       mubuf->definitions[0] = Definition(tmp_dst);
8653       mubuf->idxen = true;
8654       mubuf->can_reorder = true;
8655       ctx->block->instructions.emplace_back(std::move(mubuf));
8656
8657       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
8658       return;
8659    }
8660
8661    /* gather MIMG address components */
8662    std::vector<Temp> args;
8663    if (has_offset)
8664       args.emplace_back(offset);
8665    if (has_bias)
8666       args.emplace_back(bias);
8667    if (has_compare)
8668       args.emplace_back(compare);
8669    if (has_derivs)
8670       args.insert(args.end(), derivs.begin(), derivs.end());
8671
8672    args.insert(args.end(), coords.begin(), coords.end());
8673    if (has_sample_index)
8674       args.emplace_back(sample_index);
8675    if (has_lod)
8676       args.emplace_back(lod);
8677    if (has_clamped_lod)
8678       args.emplace_back(clamped_lod);
8679
8680    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
8681    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
8682    vec->definitions[0] = Definition(arg);
8683    for (unsigned i = 0; i < args.size(); i++)
8684       vec->operands[i] = Operand(args[i]);
8685    ctx->block->instructions.emplace_back(std::move(vec));
8686
8687
8688    if (instr->op == nir_texop_txf ||
8689        instr->op == nir_texop_txf_ms ||
8690        instr->op == nir_texop_samples_identical ||
8691        instr->op == nir_texop_fragment_fetch ||
8692        instr->op == nir_texop_fragment_mask_fetch) {
8693       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
8694       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
8695       tex->operands[0] = Operand(resource);
8696       tex->operands[1] = Operand(s4); /* no sampler */
8697       tex->operands[2] = Operand(arg);
8698       tex->dim = dim;
8699       tex->dmask = dmask;
8700       tex->unrm = true;
8701       tex->da = da;
8702       tex->definitions[0] = Definition(tmp_dst);
8703       tex->can_reorder = true;
8704       ctx->block->instructions.emplace_back(std::move(tex));
8705
8706       if (instr->op == nir_texop_samples_identical) {
8707          assert(dmask == 1 && dst.regClass() == v1);
8708          assert(dst.id() != tmp_dst.id());
8709
8710          Temp tmp = bld.tmp(bld.lm);
8711          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
8712          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
8713
8714       } else {
8715          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8716       }
8717       return;
8718    }
8719
8720    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
8721    aco_opcode opcode = aco_opcode::image_sample;
8722    if (has_offset) { /* image_sample_*_o */
8723       if (has_clamped_lod) {
8724          if (has_compare) {
8725             opcode = aco_opcode::image_sample_c_cl_o;
8726             if (has_derivs)
8727                opcode = aco_opcode::image_sample_c_d_cl_o;
8728             if (has_bias)
8729                opcode = aco_opcode::image_sample_c_b_cl_o;
8730          } else {
8731             opcode = aco_opcode::image_sample_cl_o;
8732             if (has_derivs)
8733                opcode = aco_opcode::image_sample_d_cl_o;
8734             if (has_bias)
8735                opcode = aco_opcode::image_sample_b_cl_o;
8736          }
8737       } else if (has_compare) {
8738          opcode = aco_opcode::image_sample_c_o;
8739          if (has_derivs)
8740             opcode = aco_opcode::image_sample_c_d_o;
8741          if (has_bias)
8742             opcode = aco_opcode::image_sample_c_b_o;
8743          if (level_zero)
8744             opcode = aco_opcode::image_sample_c_lz_o;
8745          if (has_lod)
8746             opcode = aco_opcode::image_sample_c_l_o;
8747       } else {
8748          opcode = aco_opcode::image_sample_o;
8749          if (has_derivs)
8750             opcode = aco_opcode::image_sample_d_o;
8751          if (has_bias)
8752             opcode = aco_opcode::image_sample_b_o;
8753          if (level_zero)
8754             opcode = aco_opcode::image_sample_lz_o;
8755          if (has_lod)
8756             opcode = aco_opcode::image_sample_l_o;
8757       }
8758    } else if (has_clamped_lod) { /* image_sample_*_cl */
8759       if (has_compare) {
8760          opcode = aco_opcode::image_sample_c_cl;
8761          if (has_derivs)
8762             opcode = aco_opcode::image_sample_c_d_cl;
8763          if (has_bias)
8764             opcode = aco_opcode::image_sample_c_b_cl;
8765       } else {
8766          opcode = aco_opcode::image_sample_cl;
8767          if (has_derivs)
8768             opcode = aco_opcode::image_sample_d_cl;
8769          if (has_bias)
8770             opcode = aco_opcode::image_sample_b_cl;
8771       }
8772    } else { /* no offset */
8773       if (has_compare) {
8774          opcode = aco_opcode::image_sample_c;
8775          if (has_derivs)
8776             opcode = aco_opcode::image_sample_c_d;
8777          if (has_bias)
8778             opcode = aco_opcode::image_sample_c_b;
8779          if (level_zero)
8780             opcode = aco_opcode::image_sample_c_lz;
8781          if (has_lod)
8782             opcode = aco_opcode::image_sample_c_l;
8783       } else {
8784          opcode = aco_opcode::image_sample;
8785          if (has_derivs)
8786             opcode = aco_opcode::image_sample_d;
8787          if (has_bias)
8788             opcode = aco_opcode::image_sample_b;
8789          if (level_zero)
8790             opcode = aco_opcode::image_sample_lz;
8791          if (has_lod)
8792             opcode = aco_opcode::image_sample_l;
8793       }
8794    }
8795
8796    if (instr->op == nir_texop_tg4) {
8797       if (has_offset) {
8798          opcode = aco_opcode::image_gather4_lz_o;
8799          if (has_compare)
8800             opcode = aco_opcode::image_gather4_c_lz_o;
8801       } else {
8802          opcode = aco_opcode::image_gather4_lz;
8803          if (has_compare)
8804             opcode = aco_opcode::image_gather4_c_lz;
8805       }
8806    } else if (instr->op == nir_texop_lod) {
8807       opcode = aco_opcode::image_get_lod;
8808    }
8809
8810    /* we don't need the bias, sample index, compare value or offset to be
8811     * computed in WQM but if the p_create_vector copies the coordinates, then it
8812     * needs to be in WQM */
8813    if (ctx->stage == fragment_fs &&
8814        !has_derivs && !has_lod && !level_zero &&
8815        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
8816        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
8817       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
8818
8819    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
8820    tex->operands[0] = Operand(resource);
8821    tex->operands[1] = Operand(sampler);
8822    tex->operands[2] = Operand(arg);
8823    tex->dim = dim;
8824    tex->dmask = dmask;
8825    tex->da = da;
8826    tex->definitions[0] = Definition(tmp_dst);
8827    tex->can_reorder = true;
8828    ctx->block->instructions.emplace_back(std::move(tex));
8829
8830    if (tg4_integer_cube_workaround) {
8831       assert(tmp_dst.id() != dst.id());
8832       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
8833
8834       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8835       Temp val[4];
8836       for (unsigned i = 0; i < dst.size(); i++) {
8837          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
8838          Temp cvt_val;
8839          if (stype == GLSL_TYPE_UINT)
8840             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
8841          else
8842             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
8843          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
8844       }
8845       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
8846       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8847                            val[0], val[1], val[2], val[3]);
8848    }
8849    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
8850    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
8851
8852 }
8853
8854
8855 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
8856 {
8857    Temp tmp = get_ssa_temp(ctx, ssa);
8858    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
8859       return Operand(tmp.regClass());
8860    else
8861       return Operand(tmp);
8862 }
8863
8864 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
8865 {
8866    aco_ptr<Pseudo_instruction> phi;
8867    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8868    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
8869
8870    bool logical = !dst.is_linear() || nir_dest_is_divergent(instr->dest);
8871    logical |= ctx->block->kind & block_kind_merge;
8872    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
8873
8874    /* we want a sorted list of sources, since the predecessor list is also sorted */
8875    std::map<unsigned, nir_ssa_def*> phi_src;
8876    nir_foreach_phi_src(src, instr)
8877       phi_src[src->pred->index] = src->src.ssa;
8878
8879    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
8880    unsigned num_operands = 0;
8881    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size()) + 1];
8882    unsigned num_defined = 0;
8883    unsigned cur_pred_idx = 0;
8884    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
8885       if (cur_pred_idx < preds.size()) {
8886          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
8887          unsigned block = ctx->cf_info.nir_to_aco[src.first];
8888          unsigned skipped = 0;
8889          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
8890             skipped++;
8891          if (cur_pred_idx + skipped < preds.size()) {
8892             for (unsigned i = 0; i < skipped; i++)
8893                operands[num_operands++] = Operand(dst.regClass());
8894             cur_pred_idx += skipped;
8895          } else {
8896             continue;
8897          }
8898       }
8899       /* Handle missing predecessors at the end. This shouldn't happen with loop
8900        * headers and we can't ignore these sources for loop header phis. */
8901       if (!(ctx->block->kind & block_kind_loop_header) && cur_pred_idx >= preds.size())
8902          continue;
8903       cur_pred_idx++;
8904       Operand op = get_phi_operand(ctx, src.second);
8905       operands[num_operands++] = op;
8906       num_defined += !op.isUndefined();
8907    }
8908    /* handle block_kind_continue_or_break at loop exit blocks */
8909    while (cur_pred_idx++ < preds.size())
8910       operands[num_operands++] = Operand(dst.regClass());
8911
8912    /* If the loop ends with a break, still add a linear continue edge in case
8913     * that break is divergent or continue_or_break is used. We'll either remove
8914     * this operand later in visit_loop() if it's not necessary or replace the
8915     * undef with something correct. */
8916    if (!logical && ctx->block->kind & block_kind_loop_header) {
8917       nir_loop *loop = nir_cf_node_as_loop(instr->instr.block->cf_node.parent);
8918       nir_block *last = nir_loop_last_block(loop);
8919       if (last->successors[0] != instr->instr.block)
8920          operands[num_operands++] = Operand(RegClass());
8921    }
8922
8923    if (num_defined == 0) {
8924       Builder bld(ctx->program, ctx->block);
8925       if (dst.regClass() == s1) {
8926          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
8927       } else if (dst.regClass() == v1) {
8928          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
8929       } else {
8930          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8931          for (unsigned i = 0; i < dst.size(); i++)
8932             vec->operands[i] = Operand(0u);
8933          vec->definitions[0] = Definition(dst);
8934          ctx->block->instructions.emplace_back(std::move(vec));
8935       }
8936       return;
8937    }
8938
8939    /* we can use a linear phi in some cases if one src is undef */
8940    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
8941       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
8942
8943       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
8944       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
8945       assert(invert->kind & block_kind_invert);
8946
8947       unsigned then_block = invert->linear_preds[0];
8948
8949       Block* insert_block = NULL;
8950       for (unsigned i = 0; i < num_operands; i++) {
8951          Operand op = operands[i];
8952          if (op.isUndefined())
8953             continue;
8954          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
8955          phi->operands[0] = op;
8956          break;
8957       }
8958       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
8959       phi->operands[1] = Operand(dst.regClass());
8960       phi->definitions[0] = Definition(dst);
8961       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
8962       return;
8963    }
8964
8965    /* try to scalarize vector phis */
8966    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
8967       // TODO: scalarize linear phis on divergent ifs
8968       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
8969       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
8970       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
8971          Operand src = operands[i];
8972          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
8973             can_scalarize = false;
8974       }
8975       if (can_scalarize) {
8976          unsigned num_components = instr->dest.ssa.num_components;
8977          assert(dst.size() % num_components == 0);
8978          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
8979
8980          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
8981          for (unsigned k = 0; k < num_components; k++) {
8982             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8983             for (unsigned i = 0; i < num_operands; i++) {
8984                Operand src = operands[i];
8985                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
8986             }
8987             Temp phi_dst = {ctx->program->allocateId(), rc};
8988             phi->definitions[0] = Definition(phi_dst);
8989             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8990             new_vec[k] = phi_dst;
8991             vec->operands[k] = Operand(phi_dst);
8992          }
8993          vec->definitions[0] = Definition(dst);
8994          ctx->block->instructions.emplace_back(std::move(vec));
8995          ctx->allocated_vec.emplace(dst.id(), new_vec);
8996          return;
8997       }
8998    }
8999
9000    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
9001    for (unsigned i = 0; i < num_operands; i++)
9002       phi->operands[i] = operands[i];
9003    phi->definitions[0] = Definition(dst);
9004    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
9005 }
9006
9007
9008 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
9009 {
9010    Temp dst = get_ssa_temp(ctx, &instr->def);
9011
9012    assert(dst.type() == RegType::sgpr);
9013
9014    if (dst.size() == 1) {
9015       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
9016    } else {
9017       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
9018       for (unsigned i = 0; i < dst.size(); i++)
9019          vec->operands[i] = Operand(0u);
9020       vec->definitions[0] = Definition(dst);
9021       ctx->block->instructions.emplace_back(std::move(vec));
9022    }
9023 }
9024
9025 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
9026 {
9027    Builder bld(ctx->program, ctx->block);
9028    Block *logical_target;
9029    append_logical_end(ctx->block);
9030    unsigned idx = ctx->block->index;
9031
9032    switch (instr->type) {
9033    case nir_jump_break:
9034       logical_target = ctx->cf_info.parent_loop.exit;
9035       add_logical_edge(idx, logical_target);
9036       ctx->block->kind |= block_kind_break;
9037
9038       if (!ctx->cf_info.parent_if.is_divergent &&
9039           !ctx->cf_info.parent_loop.has_divergent_continue) {
9040          /* uniform break - directly jump out of the loop */
9041          ctx->block->kind |= block_kind_uniform;
9042          ctx->cf_info.has_branch = true;
9043          bld.branch(aco_opcode::p_branch);
9044          add_linear_edge(idx, logical_target);
9045          return;
9046       }
9047       ctx->cf_info.parent_loop.has_divergent_branch = true;
9048       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
9049       break;
9050    case nir_jump_continue:
9051       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
9052       add_logical_edge(idx, logical_target);
9053       ctx->block->kind |= block_kind_continue;
9054
9055       if (ctx->cf_info.parent_if.is_divergent) {
9056          /* for potential uniform breaks after this continue,
9057             we must ensure that they are handled correctly */
9058          ctx->cf_info.parent_loop.has_divergent_continue = true;
9059          ctx->cf_info.parent_loop.has_divergent_branch = true;
9060          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
9061       } else {
9062          /* uniform continue - directly jump to the loop header */
9063          ctx->block->kind |= block_kind_uniform;
9064          ctx->cf_info.has_branch = true;
9065          bld.branch(aco_opcode::p_branch);
9066          add_linear_edge(idx, logical_target);
9067          return;
9068       }
9069       break;
9070    default:
9071       fprintf(stderr, "Unknown NIR jump instr: ");
9072       nir_print_instr(&instr->instr, stderr);
9073       fprintf(stderr, "\n");
9074       abort();
9075    }
9076
9077    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
9078       ctx->cf_info.exec_potentially_empty_break = true;
9079       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
9080    }
9081
9082    /* remove critical edges from linear CFG */
9083    bld.branch(aco_opcode::p_branch);
9084    Block* break_block = ctx->program->create_and_insert_block();
9085    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9086    break_block->kind |= block_kind_uniform;
9087    add_linear_edge(idx, break_block);
9088    /* the loop_header pointer might be invalidated by this point */
9089    if (instr->type == nir_jump_continue)
9090       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
9091    add_linear_edge(break_block->index, logical_target);
9092    bld.reset(break_block);
9093    bld.branch(aco_opcode::p_branch);
9094
9095    Block* continue_block = ctx->program->create_and_insert_block();
9096    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9097    add_linear_edge(idx, continue_block);
9098    append_logical_start(continue_block);
9099    ctx->block = continue_block;
9100    return;
9101 }
9102
9103 void visit_block(isel_context *ctx, nir_block *block)
9104 {
9105    nir_foreach_instr(instr, block) {
9106       switch (instr->type) {
9107       case nir_instr_type_alu:
9108          visit_alu_instr(ctx, nir_instr_as_alu(instr));
9109          break;
9110       case nir_instr_type_load_const:
9111          visit_load_const(ctx, nir_instr_as_load_const(instr));
9112          break;
9113       case nir_instr_type_intrinsic:
9114          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
9115          break;
9116       case nir_instr_type_tex:
9117          visit_tex(ctx, nir_instr_as_tex(instr));
9118          break;
9119       case nir_instr_type_phi:
9120          visit_phi(ctx, nir_instr_as_phi(instr));
9121          break;
9122       case nir_instr_type_ssa_undef:
9123          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
9124          break;
9125       case nir_instr_type_deref:
9126          break;
9127       case nir_instr_type_jump:
9128          visit_jump(ctx, nir_instr_as_jump(instr));
9129          break;
9130       default:
9131          fprintf(stderr, "Unknown NIR instr type: ");
9132          nir_print_instr(instr, stderr);
9133          fprintf(stderr, "\n");
9134          //abort();
9135       }
9136    }
9137
9138    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9139       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
9140 }
9141
9142
9143
9144 static Operand create_continue_phis(isel_context *ctx, unsigned first, unsigned last,
9145                                     aco_ptr<Instruction>& header_phi, Operand *vals)
9146 {
9147    vals[0] = Operand(header_phi->definitions[0].getTemp());
9148    RegClass rc = vals[0].regClass();
9149
9150    unsigned loop_nest_depth = ctx->program->blocks[first].loop_nest_depth;
9151
9152    unsigned next_pred = 1;
9153
9154    for (unsigned idx = first + 1; idx <= last; idx++) {
9155       Block& block = ctx->program->blocks[idx];
9156       if (block.loop_nest_depth != loop_nest_depth) {
9157          vals[idx - first] = vals[idx - 1 - first];
9158          continue;
9159       }
9160
9161       if (block.kind & block_kind_continue) {
9162          vals[idx - first] = header_phi->operands[next_pred];
9163          next_pred++;
9164          continue;
9165       }
9166
9167       bool all_same = true;
9168       for (unsigned i = 1; all_same && (i < block.linear_preds.size()); i++)
9169          all_same = vals[block.linear_preds[i] - first] == vals[block.linear_preds[0] - first];
9170
9171       Operand val;
9172       if (all_same) {
9173          val = vals[block.linear_preds[0] - first];
9174       } else {
9175          aco_ptr<Instruction> phi(create_instruction<Pseudo_instruction>(
9176             aco_opcode::p_linear_phi, Format::PSEUDO, block.linear_preds.size(), 1));
9177          for (unsigned i = 0; i < block.linear_preds.size(); i++)
9178             phi->operands[i] = vals[block.linear_preds[i] - first];
9179          val = Operand(Temp(ctx->program->allocateId(), rc));
9180          phi->definitions[0] = Definition(val.getTemp());
9181          block.instructions.emplace(block.instructions.begin(), std::move(phi));
9182       }
9183       vals[idx - first] = val;
9184    }
9185
9186    return vals[last - first];
9187 }
9188
9189 static void visit_loop(isel_context *ctx, nir_loop *loop)
9190 {
9191    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
9192    append_logical_end(ctx->block);
9193    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
9194    Builder bld(ctx->program, ctx->block);
9195    bld.branch(aco_opcode::p_branch);
9196    unsigned loop_preheader_idx = ctx->block->index;
9197
9198    Block loop_exit = Block();
9199    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9200    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
9201
9202    Block* loop_header = ctx->program->create_and_insert_block();
9203    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
9204    loop_header->kind |= block_kind_loop_header;
9205    add_edge(loop_preheader_idx, loop_header);
9206    ctx->block = loop_header;
9207
9208    /* emit loop body */
9209    unsigned loop_header_idx = loop_header->index;
9210    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
9211    append_logical_start(ctx->block);
9212    bool unreachable = visit_cf_list(ctx, &loop->body);
9213
9214    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
9215    if (!ctx->cf_info.has_branch) {
9216       append_logical_end(ctx->block);
9217       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
9218          /* Discards can result in code running with an empty exec mask.
9219           * This would result in divergent breaks not ever being taken. As a
9220           * workaround, break the loop when the loop mask is empty instead of
9221           * always continuing. */
9222          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
9223          unsigned block_idx = ctx->block->index;
9224
9225          /* create helper blocks to avoid critical edges */
9226          Block *break_block = ctx->program->create_and_insert_block();
9227          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9228          break_block->kind = block_kind_uniform;
9229          bld.reset(break_block);
9230          bld.branch(aco_opcode::p_branch);
9231          add_linear_edge(block_idx, break_block);
9232          add_linear_edge(break_block->index, &loop_exit);
9233
9234          Block *continue_block = ctx->program->create_and_insert_block();
9235          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9236          continue_block->kind = block_kind_uniform;
9237          bld.reset(continue_block);
9238          bld.branch(aco_opcode::p_branch);
9239          add_linear_edge(block_idx, continue_block);
9240          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
9241
9242          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9243             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
9244          ctx->block = &ctx->program->blocks[block_idx];
9245       } else {
9246          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
9247          if (!ctx->cf_info.parent_loop.has_divergent_branch)
9248             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9249          else
9250             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
9251       }
9252
9253       bld.reset(ctx->block);
9254       bld.branch(aco_opcode::p_branch);
9255    }
9256
9257    /* Fixup phis in loop header from unreachable blocks.
9258     * has_branch/has_divergent_branch also indicates if the loop ends with a
9259     * break/continue instruction, but we don't emit those if unreachable=true */
9260    if (unreachable) {
9261       assert(ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch);
9262       bool linear = ctx->cf_info.has_branch;
9263       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
9264       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9265          if ((logical && instr->opcode == aco_opcode::p_phi) ||
9266              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
9267             /* the last operand should be the one that needs to be removed */
9268             instr->operands.pop_back();
9269          } else if (!is_phi(instr)) {
9270             break;
9271          }
9272       }
9273    }
9274
9275    /* Fixup linear phis in loop header from expecting a continue. Both this fixup
9276     * and the previous one shouldn't both happen at once because a break in the
9277     * merge block would get CSE'd */
9278    if (nir_loop_last_block(loop)->successors[0] != nir_loop_first_block(loop)) {
9279       unsigned num_vals = ctx->cf_info.has_branch ? 1 : (ctx->block->index - loop_header_idx + 1);
9280       Operand vals[num_vals];
9281       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
9282          if (instr->opcode == aco_opcode::p_linear_phi) {
9283             if (ctx->cf_info.has_branch)
9284                instr->operands.pop_back();
9285             else
9286                instr->operands.back() = create_continue_phis(ctx, loop_header_idx, ctx->block->index, instr, vals);
9287          } else if (!is_phi(instr)) {
9288             break;
9289          }
9290       }
9291    }
9292
9293    ctx->cf_info.has_branch = false;
9294
9295    // TODO: if the loop has not a single exit, we must add one °°
9296    /* emit loop successor block */
9297    ctx->block = ctx->program->insert_block(std::move(loop_exit));
9298    append_logical_start(ctx->block);
9299
9300    #if 0
9301    // TODO: check if it is beneficial to not branch on continues
9302    /* trim linear phis in loop header */
9303    for (auto&& instr : loop_entry->instructions) {
9304       if (instr->opcode == aco_opcode::p_linear_phi) {
9305          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
9306          new_phi->definitions[0] = instr->definitions[0];
9307          for (unsigned i = 0; i < new_phi->operands.size(); i++)
9308             new_phi->operands[i] = instr->operands[i];
9309          /* check that the remaining operands are all the same */
9310          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
9311             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
9312          instr.swap(new_phi);
9313       } else if (instr->opcode == aco_opcode::p_phi) {
9314          continue;
9315       } else {
9316          break;
9317       }
9318    }
9319    #endif
9320 }
9321
9322 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
9323 {
9324    ic->cond = cond;
9325
9326    append_logical_end(ctx->block);
9327    ctx->block->kind |= block_kind_branch;
9328
9329    /* branch to linear then block */
9330    assert(cond.regClass() == ctx->program->lane_mask);
9331    aco_ptr<Pseudo_branch_instruction> branch;
9332    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
9333    branch->operands[0] = Operand(cond);
9334    ctx->block->instructions.push_back(std::move(branch));
9335
9336    ic->BB_if_idx = ctx->block->index;
9337    ic->BB_invert = Block();
9338    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9339    /* Invert blocks are intentionally not marked as top level because they
9340     * are not part of the logical cfg. */
9341    ic->BB_invert.kind |= block_kind_invert;
9342    ic->BB_endif = Block();
9343    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9344    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
9345
9346    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
9347    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
9348    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
9349    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
9350    ctx->cf_info.parent_if.is_divergent = true;
9351
9352    /* divergent branches use cbranch_execz */
9353    ctx->cf_info.exec_potentially_empty_discard = false;
9354    ctx->cf_info.exec_potentially_empty_break = false;
9355    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9356
9357    /** emit logical then block */
9358    Block* BB_then_logical = ctx->program->create_and_insert_block();
9359    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9360    add_edge(ic->BB_if_idx, BB_then_logical);
9361    ctx->block = BB_then_logical;
9362    append_logical_start(BB_then_logical);
9363 }
9364
9365 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
9366 {
9367    Block *BB_then_logical = ctx->block;
9368    append_logical_end(BB_then_logical);
9369     /* branch from logical then block to invert block */
9370    aco_ptr<Pseudo_branch_instruction> branch;
9371    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9372    BB_then_logical->instructions.emplace_back(std::move(branch));
9373    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
9374    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9375       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
9376    BB_then_logical->kind |= block_kind_uniform;
9377    assert(!ctx->cf_info.has_branch);
9378    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9379    ctx->cf_info.parent_loop.has_divergent_branch = false;
9380
9381    /** emit linear then block */
9382    Block* BB_then_linear = ctx->program->create_and_insert_block();
9383    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9384    BB_then_linear->kind |= block_kind_uniform;
9385    add_linear_edge(ic->BB_if_idx, BB_then_linear);
9386    /* branch from linear then block to invert block */
9387    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9388    BB_then_linear->instructions.emplace_back(std::move(branch));
9389    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
9390
9391    /** emit invert merge block */
9392    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
9393    ic->invert_idx = ctx->block->index;
9394
9395    /* branch to linear else block (skip else) */
9396    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
9397    branch->operands[0] = Operand(ic->cond);
9398    ctx->block->instructions.push_back(std::move(branch));
9399
9400    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
9401    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
9402    ic->exec_potentially_empty_break_depth_old =
9403       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9404    /* divergent branches use cbranch_execz */
9405    ctx->cf_info.exec_potentially_empty_discard = false;
9406    ctx->cf_info.exec_potentially_empty_break = false;
9407    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9408
9409    /** emit logical else block */
9410    Block* BB_else_logical = ctx->program->create_and_insert_block();
9411    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9412    add_logical_edge(ic->BB_if_idx, BB_else_logical);
9413    add_linear_edge(ic->invert_idx, BB_else_logical);
9414    ctx->block = BB_else_logical;
9415    append_logical_start(BB_else_logical);
9416 }
9417
9418 static void end_divergent_if(isel_context *ctx, if_context *ic)
9419 {
9420    Block *BB_else_logical = ctx->block;
9421    append_logical_end(BB_else_logical);
9422
9423    /* branch from logical else block to endif block */
9424    aco_ptr<Pseudo_branch_instruction> branch;
9425    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9426    BB_else_logical->instructions.emplace_back(std::move(branch));
9427    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
9428    if (!ctx->cf_info.parent_loop.has_divergent_branch)
9429       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
9430    BB_else_logical->kind |= block_kind_uniform;
9431
9432    assert(!ctx->cf_info.has_branch);
9433    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9434
9435
9436    /** emit linear else block */
9437    Block* BB_else_linear = ctx->program->create_and_insert_block();
9438    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9439    BB_else_linear->kind |= block_kind_uniform;
9440    add_linear_edge(ic->invert_idx, BB_else_linear);
9441
9442    /* branch from linear else block to endif block */
9443    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9444    BB_else_linear->instructions.emplace_back(std::move(branch));
9445    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
9446
9447
9448    /** emit endif merge block */
9449    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9450    append_logical_start(ctx->block);
9451
9452
9453    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
9454    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
9455    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
9456    ctx->cf_info.exec_potentially_empty_break_depth =
9457       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
9458    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
9459        !ctx->cf_info.parent_if.is_divergent) {
9460       ctx->cf_info.exec_potentially_empty_break = false;
9461       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9462    }
9463    /* uniform control flow never has an empty exec-mask */
9464    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
9465       ctx->cf_info.exec_potentially_empty_discard = false;
9466       ctx->cf_info.exec_potentially_empty_break = false;
9467       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
9468    }
9469 }
9470
9471 static void begin_uniform_if_then(isel_context *ctx, if_context *ic, Temp cond)
9472 {
9473    assert(cond.regClass() == s1);
9474
9475    append_logical_end(ctx->block);
9476    ctx->block->kind |= block_kind_uniform;
9477
9478    aco_ptr<Pseudo_branch_instruction> branch;
9479    aco_opcode branch_opcode = aco_opcode::p_cbranch_z;
9480    branch.reset(create_instruction<Pseudo_branch_instruction>(branch_opcode, Format::PSEUDO_BRANCH, 1, 0));
9481    branch->operands[0] = Operand(cond);
9482    branch->operands[0].setFixed(scc);
9483    ctx->block->instructions.emplace_back(std::move(branch));
9484
9485    ic->BB_if_idx = ctx->block->index;
9486    ic->BB_endif = Block();
9487    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
9488    ic->BB_endif.kind |= ctx->block->kind & block_kind_top_level;
9489
9490    ctx->cf_info.has_branch = false;
9491    ctx->cf_info.parent_loop.has_divergent_branch = false;
9492
9493    /** emit then block */
9494    Block* BB_then = ctx->program->create_and_insert_block();
9495    BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9496    add_edge(ic->BB_if_idx, BB_then);
9497    append_logical_start(BB_then);
9498    ctx->block = BB_then;
9499 }
9500
9501 static void begin_uniform_if_else(isel_context *ctx, if_context *ic)
9502 {
9503    Block *BB_then = ctx->block;
9504
9505    ic->uniform_has_then_branch = ctx->cf_info.has_branch;
9506    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
9507
9508    if (!ic->uniform_has_then_branch) {
9509       append_logical_end(BB_then);
9510       /* branch from then block to endif block */
9511       aco_ptr<Pseudo_branch_instruction> branch;
9512       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9513       BB_then->instructions.emplace_back(std::move(branch));
9514       add_linear_edge(BB_then->index, &ic->BB_endif);
9515       if (!ic->then_branch_divergent)
9516          add_logical_edge(BB_then->index, &ic->BB_endif);
9517       BB_then->kind |= block_kind_uniform;
9518    }
9519
9520    ctx->cf_info.has_branch = false;
9521    ctx->cf_info.parent_loop.has_divergent_branch = false;
9522
9523    /** emit else block */
9524    Block* BB_else = ctx->program->create_and_insert_block();
9525    BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
9526    add_edge(ic->BB_if_idx, BB_else);
9527    append_logical_start(BB_else);
9528    ctx->block = BB_else;
9529 }
9530
9531 static void end_uniform_if(isel_context *ctx, if_context *ic)
9532 {
9533    Block *BB_else = ctx->block;
9534
9535    if (!ctx->cf_info.has_branch) {
9536       append_logical_end(BB_else);
9537       /* branch from then block to endif block */
9538       aco_ptr<Pseudo_branch_instruction> branch;
9539       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
9540       BB_else->instructions.emplace_back(std::move(branch));
9541       add_linear_edge(BB_else->index, &ic->BB_endif);
9542       if (!ctx->cf_info.parent_loop.has_divergent_branch)
9543          add_logical_edge(BB_else->index, &ic->BB_endif);
9544       BB_else->kind |= block_kind_uniform;
9545    }
9546
9547    ctx->cf_info.has_branch &= ic->uniform_has_then_branch;
9548    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
9549
9550    /** emit endif merge block */
9551    if (!ctx->cf_info.has_branch) {
9552       ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
9553       append_logical_start(ctx->block);
9554    }
9555 }
9556
9557 static bool visit_if(isel_context *ctx, nir_if *if_stmt)
9558 {
9559    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
9560    Builder bld(ctx->program, ctx->block);
9561    aco_ptr<Pseudo_branch_instruction> branch;
9562    if_context ic;
9563
9564    if (!nir_src_is_divergent(if_stmt->condition)) { /* uniform condition */
9565       /**
9566        * Uniform conditionals are represented in the following way*) :
9567        *
9568        * The linear and logical CFG:
9569        *                        BB_IF
9570        *                        /    \
9571        *       BB_THEN (logical)      BB_ELSE (logical)
9572        *                        \    /
9573        *                        BB_ENDIF
9574        *
9575        * *) Exceptions may be due to break and continue statements within loops
9576        *    If a break/continue happens within uniform control flow, it branches
9577        *    to the loop exit/entry block. Otherwise, it branches to the next
9578        *    merge block.
9579        **/
9580
9581       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
9582       assert(cond.regClass() == ctx->program->lane_mask);
9583       cond = bool_to_scalar_condition(ctx, cond);
9584
9585       begin_uniform_if_then(ctx, &ic, cond);
9586       visit_cf_list(ctx, &if_stmt->then_list);
9587
9588       begin_uniform_if_else(ctx, &ic);
9589       visit_cf_list(ctx, &if_stmt->else_list);
9590
9591       end_uniform_if(ctx, &ic);
9592    } else { /* non-uniform condition */
9593       /**
9594        * To maintain a logical and linear CFG without critical edges,
9595        * non-uniform conditionals are represented in the following way*) :
9596        *
9597        * The linear CFG:
9598        *                        BB_IF
9599        *                        /    \
9600        *       BB_THEN (logical)      BB_THEN (linear)
9601        *                        \    /
9602        *                        BB_INVERT (linear)
9603        *                        /    \
9604        *       BB_ELSE (logical)      BB_ELSE (linear)
9605        *                        \    /
9606        *                        BB_ENDIF
9607        *
9608        * The logical CFG:
9609        *                        BB_IF
9610        *                        /    \
9611        *       BB_THEN (logical)      BB_ELSE (logical)
9612        *                        \    /
9613        *                        BB_ENDIF
9614        *
9615        * *) Exceptions may be due to break and continue statements within loops
9616        **/
9617
9618       begin_divergent_if_then(ctx, &ic, cond);
9619       visit_cf_list(ctx, &if_stmt->then_list);
9620
9621       begin_divergent_if_else(ctx, &ic);
9622       visit_cf_list(ctx, &if_stmt->else_list);
9623
9624       end_divergent_if(ctx, &ic);
9625    }
9626
9627    return !ctx->cf_info.has_branch && !ctx->block->logical_preds.empty();
9628 }
9629
9630 static bool visit_cf_list(isel_context *ctx,
9631                           struct exec_list *list)
9632 {
9633    foreach_list_typed(nir_cf_node, node, node, list) {
9634       switch (node->type) {
9635       case nir_cf_node_block:
9636          visit_block(ctx, nir_cf_node_as_block(node));
9637          break;
9638       case nir_cf_node_if:
9639          if (!visit_if(ctx, nir_cf_node_as_if(node)))
9640             return true;
9641          break;
9642       case nir_cf_node_loop:
9643          visit_loop(ctx, nir_cf_node_as_loop(node));
9644          break;
9645       default:
9646          unreachable("unimplemented cf list type");
9647       }
9648    }
9649    return false;
9650 }
9651
9652 static void create_null_export(isel_context *ctx)
9653 {
9654    /* Some shader stages always need to have exports.
9655     * So when there is none, we need to add a null export.
9656     */
9657
9658    unsigned dest = (ctx->program->stage & hw_fs) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS;
9659    bool vm = (ctx->program->stage & hw_fs) || ctx->program->chip_class >= GFX10;
9660    Builder bld(ctx->program, ctx->block);
9661    bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
9662            /* enabled_mask */ 0, dest, /* compr */ false, /* done */ true, vm);
9663 }
9664
9665 static bool export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
9666 {
9667    assert(ctx->stage == vertex_vs ||
9668           ctx->stage == tess_eval_vs ||
9669           ctx->stage == gs_copy_vs ||
9670           ctx->stage == ngg_vertex_gs ||
9671           ctx->stage == ngg_tess_eval_gs);
9672
9673    int offset = (ctx->stage & sw_tes)
9674                 ? ctx->program->info->tes.outinfo.vs_output_param_offset[slot]
9675                 : ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
9676    uint64_t mask = ctx->outputs.mask[slot];
9677    if (!is_pos && !mask)
9678       return false;
9679    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
9680       return false;
9681    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9682    exp->enabled_mask = mask;
9683    for (unsigned i = 0; i < 4; ++i) {
9684       if (mask & (1 << i))
9685          exp->operands[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9686       else
9687          exp->operands[i] = Operand(v1);
9688    }
9689    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
9690     * Setting valid_mask=1 prevents it and has no other effect.
9691     */
9692    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
9693    exp->done = false;
9694    exp->compressed = false;
9695    if (is_pos)
9696       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9697    else
9698       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
9699    ctx->block->instructions.emplace_back(std::move(exp));
9700
9701    return true;
9702 }
9703
9704 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
9705 {
9706    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9707    exp->enabled_mask = 0;
9708    for (unsigned i = 0; i < 4; ++i)
9709       exp->operands[i] = Operand(v1);
9710    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
9711       exp->operands[0] = Operand(ctx->outputs.temps[VARYING_SLOT_PSIZ * 4u]);
9712       exp->enabled_mask |= 0x1;
9713    }
9714    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
9715       exp->operands[2] = Operand(ctx->outputs.temps[VARYING_SLOT_LAYER * 4u]);
9716       exp->enabled_mask |= 0x4;
9717    }
9718    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
9719       if (ctx->options->chip_class < GFX9) {
9720          exp->operands[3] = Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]);
9721          exp->enabled_mask |= 0x8;
9722       } else {
9723          Builder bld(ctx->program, ctx->block);
9724
9725          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
9726                              Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]));
9727          if (exp->operands[2].isTemp())
9728             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
9729
9730          exp->operands[2] = Operand(out);
9731          exp->enabled_mask |= 0x4;
9732       }
9733    }
9734    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
9735    exp->done = false;
9736    exp->compressed = false;
9737    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9738    ctx->block->instructions.emplace_back(std::move(exp));
9739 }
9740
9741 static void create_export_phis(isel_context *ctx)
9742 {
9743    /* Used when exports are needed, but the output temps are defined in a preceding block.
9744     * This function will set up phis in order to access the outputs in the next block.
9745     */
9746
9747    assert(ctx->block->instructions.back()->opcode == aco_opcode::p_logical_start);
9748    aco_ptr<Instruction> logical_start = aco_ptr<Instruction>(ctx->block->instructions.back().release());
9749    ctx->block->instructions.pop_back();
9750
9751    Builder bld(ctx->program, ctx->block);
9752
9753    for (unsigned slot = 0; slot <= VARYING_SLOT_VAR31; ++slot) {
9754       uint64_t mask = ctx->outputs.mask[slot];
9755       for (unsigned i = 0; i < 4; ++i) {
9756          if (!(mask & (1 << i)))
9757             continue;
9758
9759          Temp old = ctx->outputs.temps[slot * 4 + i];
9760          Temp phi = bld.pseudo(aco_opcode::p_phi, bld.def(v1), old, Operand(v1));
9761          ctx->outputs.temps[slot * 4 + i] = phi;
9762       }
9763    }
9764
9765    bld.insert(std::move(logical_start));
9766 }
9767
9768 static void create_vs_exports(isel_context *ctx)
9769 {
9770    assert(ctx->stage == vertex_vs ||
9771           ctx->stage == tess_eval_vs ||
9772           ctx->stage == gs_copy_vs ||
9773           ctx->stage == ngg_vertex_gs ||
9774           ctx->stage == ngg_tess_eval_gs);
9775
9776    radv_vs_output_info *outinfo = (ctx->stage & sw_tes)
9777                                   ? &ctx->program->info->tes.outinfo
9778                                   : &ctx->program->info->vs.outinfo;
9779
9780    if (outinfo->export_prim_id && !(ctx->stage & hw_ngg_gs)) {
9781       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
9782       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = get_arg(ctx, ctx->args->vs_prim_id);
9783    }
9784
9785    if (ctx->options->key.has_multiview_view_index) {
9786       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
9787       ctx->outputs.temps[VARYING_SLOT_LAYER * 4u] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
9788    }
9789
9790    /* the order these position exports are created is important */
9791    int next_pos = 0;
9792    bool exported_pos = export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
9793    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
9794       export_vs_psiz_layer_viewport(ctx, &next_pos);
9795       exported_pos = true;
9796    }
9797    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9798       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
9799    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9800       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
9801
9802    if (ctx->export_clip_dists) {
9803       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9804          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
9805       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9806          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
9807    }
9808
9809    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9810       if (i < VARYING_SLOT_VAR0 &&
9811           i != VARYING_SLOT_LAYER &&
9812           i != VARYING_SLOT_PRIMITIVE_ID &&
9813           i != VARYING_SLOT_VIEWPORT)
9814          continue;
9815
9816       export_vs_varying(ctx, i, false, NULL);
9817    }
9818
9819    if (!exported_pos)
9820       create_null_export(ctx);
9821 }
9822
9823 static bool export_fs_mrt_z(isel_context *ctx)
9824 {
9825    Builder bld(ctx->program, ctx->block);
9826    unsigned enabled_channels = 0;
9827    bool compr = false;
9828    Operand values[4];
9829
9830    for (unsigned i = 0; i < 4; ++i) {
9831       values[i] = Operand(v1);
9832    }
9833
9834    /* Both stencil and sample mask only need 16-bits. */
9835    if (!ctx->program->info->ps.writes_z &&
9836        (ctx->program->info->ps.writes_stencil ||
9837         ctx->program->info->ps.writes_sample_mask)) {
9838       compr = true; /* COMPR flag */
9839
9840       if (ctx->program->info->ps.writes_stencil) {
9841          /* Stencil should be in X[23:16]. */
9842          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9843          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
9844          enabled_channels |= 0x3;
9845       }
9846
9847       if (ctx->program->info->ps.writes_sample_mask) {
9848          /* SampleMask should be in Y[15:0]. */
9849          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9850          enabled_channels |= 0xc;
9851      }
9852    } else {
9853       if (ctx->program->info->ps.writes_z) {
9854          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]);
9855          enabled_channels |= 0x1;
9856       }
9857
9858       if (ctx->program->info->ps.writes_stencil) {
9859          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9860          enabled_channels |= 0x2;
9861       }
9862
9863       if (ctx->program->info->ps.writes_sample_mask) {
9864          values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9865          enabled_channels |= 0x4;
9866       }
9867    }
9868
9869    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
9870     * writemask component.
9871     */
9872    if (ctx->options->chip_class == GFX6 &&
9873        ctx->options->family != CHIP_OLAND &&
9874        ctx->options->family != CHIP_HAINAN) {
9875             enabled_channels |= 0x1;
9876    }
9877
9878    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9879            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
9880
9881    return true;
9882 }
9883
9884 static bool export_fs_mrt_color(isel_context *ctx, int slot)
9885 {
9886    Builder bld(ctx->program, ctx->block);
9887    unsigned write_mask = ctx->outputs.mask[slot];
9888    Operand values[4];
9889
9890    for (unsigned i = 0; i < 4; ++i) {
9891       if (write_mask & (1 << i)) {
9892          values[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9893       } else {
9894          values[i] = Operand(v1);
9895       }
9896    }
9897
9898    unsigned target, col_format;
9899    unsigned enabled_channels = 0;
9900    aco_opcode compr_op = (aco_opcode)0;
9901
9902    slot -= FRAG_RESULT_DATA0;
9903    target = V_008DFC_SQ_EXP_MRT + slot;
9904    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
9905
9906    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
9907    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
9908    bool is_16bit = values[0].regClass() == v2b;
9909
9910    switch (col_format)
9911    {
9912    case V_028714_SPI_SHADER_ZERO:
9913       enabled_channels = 0; /* writemask */
9914       target = V_008DFC_SQ_EXP_NULL;
9915       break;
9916
9917    case V_028714_SPI_SHADER_32_R:
9918       enabled_channels = 1;
9919       break;
9920
9921    case V_028714_SPI_SHADER_32_GR:
9922       enabled_channels = 0x3;
9923       break;
9924
9925    case V_028714_SPI_SHADER_32_AR:
9926       if (ctx->options->chip_class >= GFX10) {
9927          /* Special case: on GFX10, the outputs are different for 32_AR */
9928          enabled_channels = 0x3;
9929          values[1] = values[3];
9930          values[3] = Operand(v1);
9931       } else {
9932          enabled_channels = 0x9;
9933       }
9934       break;
9935
9936    case V_028714_SPI_SHADER_FP16_ABGR:
9937       enabled_channels = 0x5;
9938       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
9939       if (is_16bit) {
9940          if (ctx->options->chip_class >= GFX9) {
9941             /* Pack the FP16 values together instead of converting them to
9942              * FP32 and back to FP16.
9943              * TODO: use p_create_vector and let the compiler optimizes.
9944              */
9945             compr_op = aco_opcode::v_pack_b32_f16;
9946          } else {
9947             for (unsigned i = 0; i < 4; i++) {
9948                if ((write_mask >> i) & 1)
9949                   values[i] = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), values[i]);
9950             }
9951          }
9952       }
9953       break;
9954
9955    case V_028714_SPI_SHADER_UNORM16_ABGR:
9956       enabled_channels = 0x5;
9957       if (is_16bit && ctx->options->chip_class >= GFX9) {
9958          compr_op = aco_opcode::v_cvt_pknorm_u16_f16;
9959       } else {
9960          compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
9961       }
9962       break;
9963
9964    case V_028714_SPI_SHADER_SNORM16_ABGR:
9965       enabled_channels = 0x5;
9966       if (is_16bit && ctx->options->chip_class >= GFX9) {
9967          compr_op = aco_opcode::v_cvt_pknorm_i16_f16;
9968       } else {
9969          compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
9970       }
9971       break;
9972
9973    case V_028714_SPI_SHADER_UINT16_ABGR: {
9974       enabled_channels = 0x5;
9975       compr_op = aco_opcode::v_cvt_pk_u16_u32;
9976       if (is_int8 || is_int10) {
9977          /* clamp */
9978          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
9979          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9980
9981          for (unsigned i = 0; i < 4; i++) {
9982             if ((write_mask >> i) & 1) {
9983                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
9984                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
9985                                     values[i]);
9986             }
9987          }
9988       } else if (is_16bit) {
9989          for (unsigned i = 0; i < 4; i++) {
9990             if ((write_mask >> i) & 1) {
9991                Temp tmp = convert_int(bld, values[i].getTemp(), 16, 32, false);
9992                values[i] = Operand(tmp);
9993             }
9994          }
9995       }
9996       break;
9997    }
9998
9999    case V_028714_SPI_SHADER_SINT16_ABGR:
10000       enabled_channels = 0x5;
10001       compr_op = aco_opcode::v_cvt_pk_i16_i32;
10002       if (is_int8 || is_int10) {
10003          /* clamp */
10004          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
10005          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
10006          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
10007          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
10008
10009          for (unsigned i = 0; i < 4; i++) {
10010             if ((write_mask >> i) & 1) {
10011                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
10012                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
10013                                     values[i]);
10014                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
10015                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
10016                                     values[i]);
10017             }
10018          }
10019       } else if (is_16bit) {
10020          for (unsigned i = 0; i < 4; i++) {
10021             if ((write_mask >> i) & 1) {
10022                Temp tmp = convert_int(bld, values[i].getTemp(), 16, 32, true);
10023                values[i] = Operand(tmp);
10024             }
10025          }
10026       }
10027       break;
10028
10029    case V_028714_SPI_SHADER_32_ABGR:
10030       enabled_channels = 0xF;
10031       break;
10032
10033    default:
10034       break;
10035    }
10036
10037    if (target == V_008DFC_SQ_EXP_NULL)
10038       return false;
10039
10040    if ((bool) compr_op) {
10041       for (int i = 0; i < 2; i++) {
10042          /* check if at least one of the values to be compressed is enabled */
10043          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
10044          if (enabled) {
10045             enabled_channels |= enabled << (i*2);
10046             values[i] = bld.vop3(compr_op, bld.def(v1),
10047                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
10048                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
10049          } else {
10050             values[i] = Operand(v1);
10051          }
10052       }
10053       values[2] = Operand(v1);
10054       values[3] = Operand(v1);
10055    } else {
10056       for (int i = 0; i < 4; i++)
10057          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
10058    }
10059
10060    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
10061            enabled_channels, target, (bool) compr_op);
10062    return true;
10063 }
10064
10065 static void create_fs_exports(isel_context *ctx)
10066 {
10067    bool exported = false;
10068
10069    /* Export depth, stencil and sample mask. */
10070    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
10071        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
10072        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK])
10073       exported |= export_fs_mrt_z(ctx);
10074
10075    /* Export all color render targets. */
10076    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i)
10077       if (ctx->outputs.mask[i])
10078          exported |= export_fs_mrt_color(ctx, i);
10079
10080    if (!exported)
10081       create_null_export(ctx);
10082 }
10083
10084 static void write_tcs_tess_factors(isel_context *ctx)
10085 {
10086    unsigned outer_comps;
10087    unsigned inner_comps;
10088
10089    switch (ctx->args->options->key.tcs.primitive_mode) {
10090    case GL_ISOLINES:
10091       outer_comps = 2;
10092       inner_comps = 0;
10093       break;
10094    case GL_TRIANGLES:
10095       outer_comps = 3;
10096       inner_comps = 1;
10097       break;
10098    case GL_QUADS:
10099       outer_comps = 4;
10100       inner_comps = 2;
10101       break;
10102    default:
10103       return;
10104    }
10105
10106    Builder bld(ctx->program, ctx->block);
10107
10108    bld.barrier(aco_opcode::p_memory_barrier_shared);
10109    if (unlikely(ctx->program->chip_class != GFX6 && ctx->program->workgroup_size > ctx->program->wave_size))
10110       bld.sopp(aco_opcode::s_barrier);
10111
10112    Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids);
10113    Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u));
10114
10115    Temp invocation_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), invocation_id);
10116    if_context ic_invocation_id_is_zero;
10117    begin_divergent_if_then(ctx, &ic_invocation_id_is_zero, invocation_id_is_zero);
10118    bld.reset(ctx->block);
10119
10120    Temp hs_ring_tess_factor = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_FACTOR * 16u));
10121
10122    std::pair<Temp, unsigned> lds_base = get_tcs_output_lds_offset(ctx);
10123    unsigned stride = inner_comps + outer_comps;
10124    unsigned lds_align = calculate_lds_alignment(ctx, lds_base.second);
10125    Temp tf_inner_vec;
10126    Temp tf_outer_vec;
10127    Temp out[6];
10128    assert(stride <= (sizeof(out) / sizeof(Temp)));
10129
10130    if (ctx->args->options->key.tcs.primitive_mode == GL_ISOLINES) {
10131       // LINES reversal
10132       tf_outer_vec = load_lds(ctx, 4, bld.tmp(v2), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10133       out[1] = emit_extract_vector(ctx, tf_outer_vec, 0, v1);
10134       out[0] = emit_extract_vector(ctx, tf_outer_vec, 1, v1);
10135    } else {
10136       tf_outer_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, outer_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
10137       tf_inner_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, inner_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_in_loc, lds_align);
10138
10139       for (unsigned i = 0; i < outer_comps; ++i)
10140          out[i] = emit_extract_vector(ctx, tf_outer_vec, i, v1);
10141       for (unsigned i = 0; i < inner_comps; ++i)
10142          out[outer_comps + i] = emit_extract_vector(ctx, tf_inner_vec, i, v1);
10143    }
10144
10145    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
10146    Temp tf_base = get_arg(ctx, ctx->args->tess_factor_offset);
10147    Temp byte_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, stride * 4u);
10148    unsigned tf_const_offset = 0;
10149
10150    if (ctx->program->chip_class <= GFX8) {
10151       Temp rel_patch_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), rel_patch_id);
10152       if_context ic_rel_patch_id_is_zero;
10153       begin_divergent_if_then(ctx, &ic_rel_patch_id_is_zero, rel_patch_id_is_zero);
10154       bld.reset(ctx->block);
10155
10156       /* Store the dynamic HS control word. */
10157       Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u));
10158       bld.mubuf(aco_opcode::buffer_store_dword,
10159                 /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word,
10160                 /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false,
10161                 /* disable_wqm */ false, /* glc */ true);
10162       tf_const_offset += 4;
10163
10164       begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero);
10165       end_divergent_if(ctx, &ic_rel_patch_id_is_zero);
10166       bld.reset(ctx->block);
10167    }
10168
10169    assert(stride == 2 || stride == 4 || stride == 6);
10170    Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr, 4u);
10171    store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false);
10172
10173    /* Store to offchip for TES to read - only if TES reads them */
10174    if (ctx->args->options->key.tcs.tes_reads_tess_factors) {
10175       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
10176       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
10177
10178       std::pair<Temp, unsigned> vmem_offs_outer = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_out_loc);
10179       store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, false);
10180
10181       if (likely(inner_comps)) {
10182          std::pair<Temp, unsigned> vmem_offs_inner = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_in_loc);
10183          store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, false);
10184       }
10185    }
10186
10187    begin_divergent_if_else(ctx, &ic_invocation_id_is_zero);
10188    end_divergent_if(ctx, &ic_invocation_id_is_zero);
10189 }
10190
10191 static void emit_stream_output(isel_context *ctx,
10192                                Temp const *so_buffers,
10193                                Temp const *so_write_offset,
10194                                const struct radv_stream_output *output)
10195 {
10196    unsigned num_comps = util_bitcount(output->component_mask);
10197    unsigned writemask = (1 << num_comps) - 1;
10198    unsigned loc = output->location;
10199    unsigned buf = output->buffer;
10200
10201    assert(num_comps && num_comps <= 4);
10202    if (!num_comps || num_comps > 4)
10203       return;
10204
10205    unsigned start = ffs(output->component_mask) - 1;
10206
10207    Temp out[4];
10208    bool all_undef = true;
10209    assert(ctx->stage & hw_vs);
10210    for (unsigned i = 0; i < num_comps; i++) {
10211       out[i] = ctx->outputs.temps[loc * 4 + start + i];
10212       all_undef = all_undef && !out[i].id();
10213    }
10214    if (all_undef)
10215       return;
10216
10217    while (writemask) {
10218       int start, count;
10219       u_bit_scan_consecutive_range(&writemask, &start, &count);
10220       if (count == 3 && ctx->options->chip_class == GFX6) {
10221          /* GFX6 doesn't support storing vec3, split it. */
10222          writemask |= 1u << (start + 2);
10223          count = 2;
10224       }
10225
10226       unsigned offset = output->offset + start * 4;
10227
10228       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
10229       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
10230       for (int i = 0; i < count; ++i)
10231          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
10232       vec->definitions[0] = Definition(write_data);
10233       ctx->block->instructions.emplace_back(std::move(vec));
10234
10235       aco_opcode opcode;
10236       switch (count) {
10237       case 1:
10238          opcode = aco_opcode::buffer_store_dword;
10239          break;
10240       case 2:
10241          opcode = aco_opcode::buffer_store_dwordx2;
10242          break;
10243       case 3:
10244          opcode = aco_opcode::buffer_store_dwordx3;
10245          break;
10246       case 4:
10247          opcode = aco_opcode::buffer_store_dwordx4;
10248          break;
10249       default:
10250          unreachable("Unsupported dword count.");
10251       }
10252
10253       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
10254       store->operands[0] = Operand(so_buffers[buf]);
10255       store->operands[1] = Operand(so_write_offset[buf]);
10256       store->operands[2] = Operand((uint32_t) 0);
10257       store->operands[3] = Operand(write_data);
10258       if (offset > 4095) {
10259          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
10260          Builder bld(ctx->program, ctx->block);
10261          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
10262       } else {
10263          store->offset = offset;
10264       }
10265       store->offen = true;
10266       store->glc = true;
10267       store->dlc = false;
10268       store->slc = true;
10269       store->can_reorder = true;
10270       ctx->block->instructions.emplace_back(std::move(store));
10271    }
10272 }
10273
10274 static void emit_streamout(isel_context *ctx, unsigned stream)
10275 {
10276    Builder bld(ctx->program, ctx->block);
10277
10278    Temp so_buffers[4];
10279    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
10280    for (unsigned i = 0; i < 4; i++) {
10281       unsigned stride = ctx->program->info->so.strides[i];
10282       if (!stride)
10283          continue;
10284
10285       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
10286       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
10287    }
10288
10289    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10290                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
10291
10292    Temp tid = emit_mbcnt(ctx, bld.def(v1));
10293
10294    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
10295
10296    if_context ic;
10297    begin_divergent_if_then(ctx, &ic, can_emit);
10298
10299    bld.reset(ctx->block);
10300
10301    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
10302
10303    Temp so_write_offset[4];
10304
10305    for (unsigned i = 0; i < 4; i++) {
10306       unsigned stride = ctx->program->info->so.strides[i];
10307       if (!stride)
10308          continue;
10309
10310       if (stride == 1) {
10311          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
10312                                 get_arg(ctx, ctx->args->streamout_write_idx),
10313                                 get_arg(ctx, ctx->args->streamout_offset[i]));
10314          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
10315
10316          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
10317       } else {
10318          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
10319          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
10320                                  get_arg(ctx, ctx->args->streamout_offset[i]));
10321          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
10322       }
10323    }
10324
10325    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
10326       struct radv_stream_output *output =
10327          &ctx->program->info->so.outputs[i];
10328       if (stream != output->stream)
10329          continue;
10330
10331       emit_stream_output(ctx, so_buffers, so_write_offset, output);
10332    }
10333
10334    begin_divergent_if_else(ctx, &ic);
10335    end_divergent_if(ctx, &ic);
10336 }
10337
10338 } /* end namespace */
10339
10340 void fix_ls_vgpr_init_bug(isel_context *ctx, Pseudo_instruction *startpgm)
10341 {
10342    assert(ctx->shader->info.stage == MESA_SHADER_VERTEX);
10343    Builder bld(ctx->program, ctx->block);
10344    constexpr unsigned hs_idx = 1u;
10345    Builder::Result hs_thread_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10346                                               get_arg(ctx, ctx->args->merged_wave_info),
10347                                               Operand((8u << 16) | (hs_idx * 8u)));
10348    Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp());
10349
10350    /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */
10351
10352    Temp instance_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10353                                get_arg(ctx, ctx->args->rel_auto_id),
10354                                get_arg(ctx, ctx->args->ac.instance_id),
10355                                ls_has_nonzero_hs_threads);
10356    Temp rel_auto_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10357                                get_arg(ctx, ctx->args->ac.tcs_rel_ids),
10358                                get_arg(ctx, ctx->args->rel_auto_id),
10359                                ls_has_nonzero_hs_threads);
10360    Temp vertex_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10361                              get_arg(ctx, ctx->args->ac.tcs_patch_id),
10362                              get_arg(ctx, ctx->args->ac.vertex_id),
10363                              ls_has_nonzero_hs_threads);
10364
10365    ctx->arg_temps[ctx->args->ac.instance_id.arg_index] = instance_id;
10366    ctx->arg_temps[ctx->args->rel_auto_id.arg_index] = rel_auto_id;
10367    ctx->arg_temps[ctx->args->ac.vertex_id.arg_index] = vertex_id;
10368 }
10369
10370 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
10371 {
10372    /* Split all arguments except for the first (ring_offsets) and the last
10373     * (exec) so that the dead channels don't stay live throughout the program.
10374     */
10375    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
10376       if (startpgm->definitions[i].regClass().size() > 1) {
10377          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
10378                            startpgm->definitions[i].regClass().size());
10379       }
10380    }
10381 }
10382
10383 void handle_bc_optimize(isel_context *ctx)
10384 {
10385    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
10386    Builder bld(ctx->program, ctx->block);
10387    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
10388    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
10389    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
10390    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
10391    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
10392    if (uses_center && uses_centroid) {
10393       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
10394                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
10395
10396       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
10397          Temp new_coord[2];
10398          for (unsigned i = 0; i < 2; i++) {
10399             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
10400             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
10401             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10402                                     persp_centroid, persp_center, sel);
10403          }
10404          ctx->persp_centroid = bld.tmp(v2);
10405          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
10406                     Operand(new_coord[0]), Operand(new_coord[1]));
10407          emit_split_vector(ctx, ctx->persp_centroid, 2);
10408       }
10409
10410       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
10411          Temp new_coord[2];
10412          for (unsigned i = 0; i < 2; i++) {
10413             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
10414             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
10415             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
10416                                     linear_centroid, linear_center, sel);
10417          }
10418          ctx->linear_centroid = bld.tmp(v2);
10419          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
10420                     Operand(new_coord[0]), Operand(new_coord[1]));
10421          emit_split_vector(ctx, ctx->linear_centroid, 2);
10422       }
10423    }
10424 }
10425
10426 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
10427 {
10428    Program *program = ctx->program;
10429
10430    unsigned float_controls = shader->info.float_controls_execution_mode;
10431
10432    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
10433       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
10434    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
10435       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
10436                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
10437
10438    program->next_fp_mode.must_flush_denorms32 =
10439       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
10440    program->next_fp_mode.must_flush_denorms16_64 =
10441       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
10442                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
10443
10444    program->next_fp_mode.care_about_round32 =
10445       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
10446
10447    program->next_fp_mode.care_about_round16_64 =
10448       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
10449                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
10450
10451    /* default to preserving fp16 and fp64 denorms, since it's free */
10452    if (program->next_fp_mode.must_flush_denorms16_64)
10453       program->next_fp_mode.denorm16_64 = 0;
10454    else
10455       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10456
10457    /* preserving fp32 denorms is expensive, so only do it if asked */
10458    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
10459       program->next_fp_mode.denorm32 = fp_denorm_keep;
10460    else
10461       program->next_fp_mode.denorm32 = 0;
10462
10463    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
10464       program->next_fp_mode.round32 = fp_round_tz;
10465    else
10466       program->next_fp_mode.round32 = fp_round_ne;
10467
10468    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
10469       program->next_fp_mode.round16_64 = fp_round_tz;
10470    else
10471       program->next_fp_mode.round16_64 = fp_round_ne;
10472
10473    ctx->block->fp_mode = program->next_fp_mode;
10474 }
10475
10476 void cleanup_cfg(Program *program)
10477 {
10478    /* create linear_succs/logical_succs */
10479    for (Block& BB : program->blocks) {
10480       for (unsigned idx : BB.linear_preds)
10481          program->blocks[idx].linear_succs.emplace_back(BB.index);
10482       for (unsigned idx : BB.logical_preds)
10483          program->blocks[idx].logical_succs.emplace_back(BB.index);
10484    }
10485 }
10486
10487 Temp merged_wave_info_to_mask(isel_context *ctx, unsigned i)
10488 {
10489    Builder bld(ctx->program, ctx->block);
10490
10491    /* The s_bfm only cares about s0.u[5:0] so we don't need either s_bfe nor s_and here */
10492    Temp count = i == 0
10493                 ? get_arg(ctx, ctx->args->merged_wave_info)
10494                 : bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc),
10495                            get_arg(ctx, ctx->args->merged_wave_info), Operand(i * 8u));
10496
10497    Temp mask = bld.sop2(aco_opcode::s_bfm_b64, bld.def(s2), count, Operand(0u));
10498    Temp cond;
10499
10500    if (ctx->program->wave_size == 64) {
10501       /* Special case for 64 active invocations, because 64 doesn't work with s_bfm */
10502       Temp active_64 = bld.sopc(aco_opcode::s_bitcmp1_b32, bld.def(s1, scc), count, Operand(6u /* log2(64) */));
10503       cond = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), mask, bld.scc(active_64));
10504    } else {
10505       /* We use s_bfm_b64 (not _b32) which works with 32, but we need to extract the lower half of the register */
10506       cond = emit_extract_vector(ctx, mask, 0, bld.lm);
10507    }
10508
10509    return cond;
10510 }
10511
10512 bool ngg_early_prim_export(isel_context *ctx)
10513 {
10514    /* TODO: Check edge flags, and if they are written, return false. (Needed for OpenGL, not for Vulkan.) */
10515    return true;
10516 }
10517
10518 void ngg_emit_sendmsg_gs_alloc_req(isel_context *ctx)
10519 {
10520    Builder bld(ctx->program, ctx->block);
10521
10522    /* It is recommended to do the GS_ALLOC_REQ as soon and as quickly as possible, so we set the maximum priority (3). */
10523    bld.sopp(aco_opcode::s_setprio, -1u, 0x3u);
10524
10525    /* Get the id of the current wave within the threadgroup (workgroup) */
10526    Builder::Result wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10527                                             get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10528
10529    /* Execute the following code only on the first wave (wave id 0),
10530     * use the SCC def to tell if the wave id is zero or not.
10531     */
10532    Temp cond = wave_id_in_tg.def(1).getTemp();
10533    if_context ic;
10534    begin_uniform_if_then(ctx, &ic, cond);
10535    begin_uniform_if_else(ctx, &ic);
10536    bld.reset(ctx->block);
10537
10538    /* Number of vertices output by VS/TES */
10539    Temp vtx_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10540                            get_arg(ctx, ctx->args->gs_tg_info), Operand(12u | (9u << 16u)));
10541    /* Number of primitives output by VS/TES */
10542    Temp prm_cnt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10543                            get_arg(ctx, ctx->args->gs_tg_info), Operand(22u | (9u << 16u)));
10544
10545    /* Put the number of vertices and primitives into m0 for the GS_ALLOC_REQ */
10546    Temp tmp = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), prm_cnt, Operand(12u));
10547    tmp = bld.sop2(aco_opcode::s_or_b32, bld.m0(bld.def(s1)), bld.def(s1, scc), tmp, vtx_cnt);
10548
10549    /* Request the SPI to allocate space for the primitives and vertices that will be exported by the threadgroup. */
10550    bld.sopp(aco_opcode::s_sendmsg, bld.m0(tmp), -1, sendmsg_gs_alloc_req);
10551
10552    end_uniform_if(ctx, &ic);
10553
10554    /* After the GS_ALLOC_REQ is done, reset priority to default (0). */
10555    bld.reset(ctx->block);
10556    bld.sopp(aco_opcode::s_setprio, -1u, 0x0u);
10557 }
10558
10559 Temp ngg_get_prim_exp_arg(isel_context *ctx, unsigned num_vertices, const Temp vtxindex[])
10560 {
10561    Builder bld(ctx->program, ctx->block);
10562
10563    if (ctx->args->options->key.vs_common_out.as_ngg_passthrough) {
10564       return get_arg(ctx, ctx->args->gs_vtx_offset[0]);
10565    }
10566
10567    Temp gs_invocation_id = get_arg(ctx, ctx->args->ac.gs_invocation_id);
10568    Temp tmp;
10569
10570    for (unsigned i = 0; i < num_vertices; ++i) {
10571       assert(vtxindex[i].id());
10572
10573       if (i)
10574          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), vtxindex[i], Operand(10u * i), tmp);
10575       else
10576          tmp = vtxindex[i];
10577
10578       /* The initial edge flag is always false in tess eval shaders. */
10579       if (ctx->stage == ngg_vertex_gs) {
10580          Temp edgeflag = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), gs_invocation_id, Operand(8 + i), Operand(1u));
10581          tmp = bld.vop3(aco_opcode::v_lshl_add_u32, bld.def(v1), edgeflag, Operand(10u * i + 9u), tmp);
10582       }
10583    }
10584
10585    /* TODO: Set isnull field in case of merged NGG VS+GS. */
10586
10587    return tmp;
10588 }
10589
10590 void ngg_emit_prim_export(isel_context *ctx, unsigned num_vertices_per_primitive, const Temp vtxindex[])
10591 {
10592    Builder bld(ctx->program, ctx->block);
10593    Temp prim_exp_arg = ngg_get_prim_exp_arg(ctx, num_vertices_per_primitive, vtxindex);
10594
10595    bld.exp(aco_opcode::exp, prim_exp_arg, Operand(v1), Operand(v1), Operand(v1),
10596         1 /* enabled mask */, V_008DFC_SQ_EXP_PRIM /* dest */,
10597         false /* compressed */, true/* done */, false /* valid mask */);
10598 }
10599
10600 void ngg_emit_nogs_gsthreads(isel_context *ctx)
10601 {
10602    /* Emit the things that NGG GS threads need to do, for shaders that don't have SW GS.
10603     * These must always come before VS exports.
10604     *
10605     * It is recommended to do these as early as possible. They can be at the beginning when
10606     * there is no SW GS and the shader doesn't write edge flags.
10607     */
10608
10609    if_context ic;
10610    Temp is_gs_thread = merged_wave_info_to_mask(ctx, 1);
10611    begin_divergent_if_then(ctx, &ic, is_gs_thread);
10612
10613    Builder bld(ctx->program, ctx->block);
10614    constexpr unsigned max_vertices_per_primitive = 3;
10615    unsigned num_vertices_per_primitive = max_vertices_per_primitive;
10616
10617    if (ctx->stage == ngg_vertex_gs) {
10618       /* TODO: optimize for points & lines */
10619    } else if (ctx->stage == ngg_tess_eval_gs) {
10620       if (ctx->shader->info.tess.point_mode)
10621          num_vertices_per_primitive = 1;
10622       else if (ctx->shader->info.tess.primitive_mode == GL_ISOLINES)
10623          num_vertices_per_primitive = 2;
10624    } else {
10625       unreachable("Unsupported NGG shader stage");
10626    }
10627
10628    Temp vtxindex[max_vertices_per_primitive];
10629    vtxindex[0] = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10630                           get_arg(ctx, ctx->args->gs_vtx_offset[0]));
10631    vtxindex[1] = num_vertices_per_primitive < 2 ? Temp(0, v1) :
10632                  bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
10633                           get_arg(ctx, ctx->args->gs_vtx_offset[0]), Operand(16u), Operand(16u));
10634    vtxindex[2] = num_vertices_per_primitive < 3 ? Temp(0, v1) :
10635                  bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu),
10636                           get_arg(ctx, ctx->args->gs_vtx_offset[2]));
10637
10638    /* Export primitive data to the index buffer. */
10639    ngg_emit_prim_export(ctx, num_vertices_per_primitive, vtxindex);
10640
10641    /* Export primitive ID. */
10642    if (ctx->stage == ngg_vertex_gs && ctx->args->options->key.vs_common_out.export_prim_id) {
10643       /* Copy Primitive IDs from GS threads to the LDS address corresponding to the ES thread of the provoking vertex. */
10644       Temp prim_id = get_arg(ctx, ctx->args->ac.gs_prim_id);
10645       Temp provoking_vtx_index = vtxindex[0];
10646       Temp addr = bld.v_mul_imm(bld.def(v1), provoking_vtx_index, 4u);
10647
10648       store_lds(ctx, 4, prim_id, 0x1u, addr, 0u, 4u);
10649    }
10650
10651    begin_divergent_if_else(ctx, &ic);
10652    end_divergent_if(ctx, &ic);
10653 }
10654
10655 void ngg_emit_nogs_output(isel_context *ctx)
10656 {
10657    /* Emits NGG GS output, for stages that don't have SW GS. */
10658
10659    if_context ic;
10660    Builder bld(ctx->program, ctx->block);
10661    bool late_prim_export = !ngg_early_prim_export(ctx);
10662
10663    /* NGG streamout is currently disabled by default. */
10664    assert(!ctx->args->shader_info->so.num_outputs);
10665
10666    if (late_prim_export) {
10667       /* VS exports are output to registers in a predecessor block. Emit phis to get them into this block. */
10668       create_export_phis(ctx);
10669       /* Do what we need to do in the GS threads. */
10670       ngg_emit_nogs_gsthreads(ctx);
10671
10672       /* What comes next should be executed on ES threads. */
10673       Temp is_es_thread = merged_wave_info_to_mask(ctx, 0);
10674       begin_divergent_if_then(ctx, &ic, is_es_thread);
10675       bld.reset(ctx->block);
10676    }
10677
10678    /* Export VS outputs */
10679    ctx->block->kind |= block_kind_export_end;
10680    create_vs_exports(ctx);
10681
10682    /* Export primitive ID */
10683    if (ctx->args->options->key.vs_common_out.export_prim_id) {
10684       Temp prim_id;
10685
10686       if (ctx->stage == ngg_vertex_gs) {
10687          /* Wait for GS threads to store primitive ID in LDS. */
10688          bld.barrier(aco_opcode::p_memory_barrier_shared);
10689          bld.sopp(aco_opcode::s_barrier);
10690
10691          /* Calculate LDS address where the GS threads stored the primitive ID. */
10692          Temp wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10693                                        get_arg(ctx, ctx->args->merged_wave_info), Operand(24u | (4u << 16)));
10694          Temp thread_id_in_wave = emit_mbcnt(ctx, bld.def(v1));
10695          Temp wave_id_mul = bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_id_in_tg), ctx->program->wave_size);
10696          Temp thread_id_in_tg = bld.vadd32(bld.def(v1), Operand(wave_id_mul), Operand(thread_id_in_wave));
10697          Temp addr = bld.v_mul24_imm(bld.def(v1), thread_id_in_tg, 4u);
10698
10699          /* Load primitive ID from LDS. */
10700          prim_id = load_lds(ctx, 4, bld.tmp(v1), addr, 0u, 4u);
10701       } else if (ctx->stage == ngg_tess_eval_gs) {
10702          /* TES: Just use the patch ID as the primitive ID. */
10703          prim_id = get_arg(ctx, ctx->args->ac.tes_patch_id);
10704       } else {
10705          unreachable("unsupported NGG shader stage.");
10706       }
10707
10708       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
10709       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = prim_id;
10710
10711       export_vs_varying(ctx, VARYING_SLOT_PRIMITIVE_ID, false, nullptr);
10712    }
10713
10714    if (late_prim_export) {
10715       begin_divergent_if_else(ctx, &ic);
10716       end_divergent_if(ctx, &ic);
10717       bld.reset(ctx->block);
10718    }
10719 }
10720
10721 void select_program(Program *program,
10722                     unsigned shader_count,
10723                     struct nir_shader *const *shaders,
10724                     ac_shader_config* config,
10725                     struct radv_shader_args *args)
10726 {
10727    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
10728    if_context ic_merged_wave_info;
10729    bool ngg_no_gs = ctx.stage == ngg_vertex_gs || ctx.stage == ngg_tess_eval_gs;
10730
10731    for (unsigned i = 0; i < shader_count; i++) {
10732       nir_shader *nir = shaders[i];
10733       init_context(&ctx, nir);
10734
10735       setup_fp_mode(&ctx, nir);
10736
10737       if (!i) {
10738          /* needs to be after init_context() for FS */
10739          Pseudo_instruction *startpgm = add_startpgm(&ctx);
10740          append_logical_start(ctx.block);
10741
10742          if (unlikely(args->options->has_ls_vgpr_init_bug && ctx.stage == vertex_tess_control_hs))
10743             fix_ls_vgpr_init_bug(&ctx, startpgm);
10744
10745          split_arguments(&ctx, startpgm);
10746       }
10747
10748       if (ngg_no_gs) {
10749          ngg_emit_sendmsg_gs_alloc_req(&ctx);
10750
10751          if (ngg_early_prim_export(&ctx))
10752             ngg_emit_nogs_gsthreads(&ctx);
10753       }
10754
10755       /* In a merged VS+TCS HS, the VS implementation can be completely empty. */
10756       nir_function_impl *func = nir_shader_get_entrypoint(nir);
10757       bool empty_shader = nir_cf_list_is_empty_block(&func->body) &&
10758                           ((nir->info.stage == MESA_SHADER_VERTEX &&
10759                             (ctx.stage == vertex_tess_control_hs || ctx.stage == vertex_geometry_gs)) ||
10760                            (nir->info.stage == MESA_SHADER_TESS_EVAL &&
10761                             ctx.stage == tess_eval_geometry_gs));
10762
10763       bool check_merged_wave_info = ctx.tcs_in_out_eq ? i == 0 : ((shader_count >= 2 && !empty_shader) || ngg_no_gs);
10764       bool endif_merged_wave_info = ctx.tcs_in_out_eq ? i == 1 : check_merged_wave_info;
10765       if (check_merged_wave_info) {
10766          Temp cond = merged_wave_info_to_mask(&ctx, i);
10767          begin_divergent_if_then(&ctx, &ic_merged_wave_info, cond);
10768       }
10769
10770       if (i) {
10771          Builder bld(ctx.program, ctx.block);
10772
10773          bld.barrier(aco_opcode::p_memory_barrier_shared);
10774          bld.sopp(aco_opcode::s_barrier);
10775
10776          if (ctx.stage == vertex_geometry_gs || ctx.stage == tess_eval_geometry_gs) {
10777             ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
10778          }
10779       } else if (ctx.stage == geometry_gs)
10780          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
10781
10782       if (ctx.stage == fragment_fs)
10783          handle_bc_optimize(&ctx);
10784
10785       visit_cf_list(&ctx, &func->body);
10786
10787       if (ctx.program->info->so.num_outputs && (ctx.stage & hw_vs))
10788          emit_streamout(&ctx, 0);
10789
10790       if (ctx.stage & hw_vs) {
10791          create_vs_exports(&ctx);
10792          ctx.block->kind |= block_kind_export_end;
10793       } else if (ngg_no_gs && ngg_early_prim_export(&ctx)) {
10794          ngg_emit_nogs_output(&ctx);
10795       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
10796          Builder bld(ctx.program, ctx.block);
10797          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
10798          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
10799       } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
10800          write_tcs_tess_factors(&ctx);
10801       }
10802
10803       if (ctx.stage == fragment_fs) {
10804          create_fs_exports(&ctx);
10805          ctx.block->kind |= block_kind_export_end;
10806       }
10807
10808       if (endif_merged_wave_info) {
10809          begin_divergent_if_else(&ctx, &ic_merged_wave_info);
10810          end_divergent_if(&ctx, &ic_merged_wave_info);
10811       }
10812
10813       if (ngg_no_gs && !ngg_early_prim_export(&ctx))
10814          ngg_emit_nogs_output(&ctx);
10815
10816       if (i == 0 && ctx.stage == vertex_tess_control_hs && ctx.tcs_in_out_eq) {
10817          /* Outputs of the previous stage are inputs to the next stage */
10818          ctx.inputs = ctx.outputs;
10819          ctx.outputs = shader_io_state();
10820       }
10821    }
10822
10823    program->config->float_mode = program->blocks[0].fp_mode.val;
10824
10825    append_logical_end(ctx.block);
10826    ctx.block->kind |= block_kind_uniform;
10827    Builder bld(ctx.program, ctx.block);
10828    if (ctx.program->wb_smem_l1_on_end)
10829       bld.smem(aco_opcode::s_dcache_wb, false);
10830    bld.sopp(aco_opcode::s_endpgm);
10831
10832    cleanup_cfg(program);
10833 }
10834
10835 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
10836                            ac_shader_config* config,
10837                            struct radv_shader_args *args)
10838 {
10839    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
10840
10841    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
10842    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
10843    program->next_fp_mode.must_flush_denorms32 = false;
10844    program->next_fp_mode.must_flush_denorms16_64 = false;
10845    program->next_fp_mode.care_about_round32 = false;
10846    program->next_fp_mode.care_about_round16_64 = false;
10847    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
10848    program->next_fp_mode.denorm32 = 0;
10849    program->next_fp_mode.round32 = fp_round_ne;
10850    program->next_fp_mode.round16_64 = fp_round_ne;
10851    ctx.block->fp_mode = program->next_fp_mode;
10852
10853    add_startpgm(&ctx);
10854    append_logical_start(ctx.block);
10855
10856    Builder bld(ctx.program, ctx.block);
10857
10858    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
10859
10860    Operand stream_id(0u);
10861    if (args->shader_info->so.num_outputs)
10862       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
10863                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
10864
10865    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
10866
10867    std::stack<Block> endif_blocks;
10868
10869    for (unsigned stream = 0; stream < 4; stream++) {
10870       if (stream_id.isConstant() && stream != stream_id.constantValue())
10871          continue;
10872
10873       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
10874       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
10875          continue;
10876
10877       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
10878
10879       unsigned BB_if_idx = ctx.block->index;
10880       Block BB_endif = Block();
10881       if (!stream_id.isConstant()) {
10882          /* begin IF */
10883          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
10884          append_logical_end(ctx.block);
10885          ctx.block->kind |= block_kind_uniform;
10886          bld.branch(aco_opcode::p_cbranch_z, cond);
10887
10888          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
10889
10890          ctx.block = ctx.program->create_and_insert_block();
10891          add_edge(BB_if_idx, ctx.block);
10892          bld.reset(ctx.block);
10893          append_logical_start(ctx.block);
10894       }
10895
10896       unsigned offset = 0;
10897       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
10898          if (args->shader_info->gs.output_streams[i] != stream)
10899             continue;
10900
10901          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
10902          unsigned length = util_last_bit(output_usage_mask);
10903          for (unsigned j = 0; j < length; ++j) {
10904             if (!(output_usage_mask & (1 << j)))
10905                continue;
10906
10907             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
10908             Temp voffset = vtx_offset;
10909             if (const_offset >= 4096u) {
10910                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
10911                const_offset %= 4096u;
10912             }
10913
10914             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
10915             mubuf->definitions[0] = bld.def(v1);
10916             mubuf->operands[0] = Operand(gsvs_ring);
10917             mubuf->operands[1] = Operand(voffset);
10918             mubuf->operands[2] = Operand(0u);
10919             mubuf->offen = true;
10920             mubuf->offset = const_offset;
10921             mubuf->glc = true;
10922             mubuf->slc = true;
10923             mubuf->dlc = args->options->chip_class >= GFX10;
10924             mubuf->barrier = barrier_none;
10925             mubuf->can_reorder = true;
10926
10927             ctx.outputs.mask[i] |= 1 << j;
10928             ctx.outputs.temps[i * 4u + j] = mubuf->definitions[0].getTemp();
10929
10930             bld.insert(std::move(mubuf));
10931
10932             offset++;
10933          }
10934       }
10935
10936       if (args->shader_info->so.num_outputs) {
10937          emit_streamout(&ctx, stream);
10938          bld.reset(ctx.block);
10939       }
10940
10941       if (stream == 0) {
10942          create_vs_exports(&ctx);
10943          ctx.block->kind |= block_kind_export_end;
10944       }
10945
10946       if (!stream_id.isConstant()) {
10947          append_logical_end(ctx.block);
10948
10949          /* branch from then block to endif block */
10950          bld.branch(aco_opcode::p_branch);
10951          add_edge(ctx.block->index, &BB_endif);
10952          ctx.block->kind |= block_kind_uniform;
10953
10954          /* emit else block */
10955          ctx.block = ctx.program->create_and_insert_block();
10956          add_edge(BB_if_idx, ctx.block);
10957          bld.reset(ctx.block);
10958          append_logical_start(ctx.block);
10959
10960          endif_blocks.push(std::move(BB_endif));
10961       }
10962    }
10963
10964    while (!endif_blocks.empty()) {
10965       Block BB_endif = std::move(endif_blocks.top());
10966       endif_blocks.pop();
10967
10968       Block *BB_else = ctx.block;
10969
10970       append_logical_end(BB_else);
10971       /* branch from else block to endif block */
10972       bld.branch(aco_opcode::p_branch);
10973       add_edge(BB_else->index, &BB_endif);
10974       BB_else->kind |= block_kind_uniform;
10975
10976       /** emit endif merge block */
10977       ctx.block = program->insert_block(std::move(BB_endif));
10978       bld.reset(ctx.block);
10979       append_logical_start(ctx.block);
10980    }
10981
10982    program->config->float_mode = program->blocks[0].fp_mode.val;
10983
10984    append_logical_end(ctx.block);
10985    ctx.block->kind |= block_kind_uniform;
10986    bld.sopp(aco_opcode::s_endpgm);
10987
10988    cleanup_cfg(program);
10989 }
10990 }