src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool then_branch_divergent;
  89    Block BB_invert;
  90    Block BB_endif;
  91 };
  92
  93 static void visit_cf_list(struct isel_context *ctx,
  94                           struct exec_list *list);
  95
  96 static void add_logical_edge(unsigned pred_idx, Block *succ)
  97 {
  98    succ->logical_preds.emplace_back(pred_idx);
  99 }
 100
 101
 102 static void add_linear_edge(unsigned pred_idx, Block *succ)
 103 {
 104    succ->linear_preds.emplace_back(pred_idx);
 105 }
 106
 107 static void add_edge(unsigned pred_idx, Block *succ)
 108 {
 109    add_logical_edge(pred_idx, succ);
 110    add_linear_edge(pred_idx, succ);
 111 }
 112
 113 static void append_logical_start(Block *b)
 114 {
 115    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 116 }
 117
 118 static void append_logical_end(Block *b)
 119 {
 120    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 121 }
 122
 123 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 124 {
 125    assert(ctx->allocated[def->index].id());
 126    return ctx->allocated[def->index];
 127 }
 128
 129 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 130                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 131 {
 132    Builder bld(ctx->program, ctx->block);
 133    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 134    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 135
 136    if (ctx->program->wave_size == 32) {
 137       return thread_id_lo;
 138    } else {
 139       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 140       return thread_id_hi;
 141    }
 142 }
 143
 144 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 145 {
 146    Builder bld(ctx->program, ctx->block);
 147
 148    if (!dst.id())
 149       dst = bld.tmp(src.regClass());
 150
 151    assert(src.size() == dst.size());
 152
 153    if (ctx->stage != fragment_fs) {
 154       if (!dst.id())
 155          return src;
 156
 157       bld.copy(Definition(dst), src);
 158       return dst;
 159    }
 160
 161    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 162    ctx->program->needs_wqm |= program_needs_wqm;
 163    return dst;
 164 }
 165
 166 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 167 {
 168    if (index.regClass() == s1)
 169       return bld.readlane(bld.def(s1), data, index);
 170
 171    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 172
 173    /* Currently not implemented on GFX6-7 */
 174    assert(ctx->options->chip_class >= GFX8);
 175
 176    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 177       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 178    }
 179
 180    /* GFX10, wave64 mode:
 181     * The bpermute instruction is limited to half-wave operation, which means that it can't
 182     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 183     * emulate it here.
 184     */
 185    if (!ctx->has_gfx10_wave64_bpermute) {
 186       ctx->has_gfx10_wave64_bpermute = true;
 187       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 188       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 189    }
 190
 191    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 192    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 193    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 194    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 195
 196    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 197                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 198 }
 199
 200 Temp as_vgpr(isel_context *ctx, Temp val)
 201 {
 202    if (val.type() == RegType::sgpr) {
 203       Builder bld(ctx->program, ctx->block);
 204       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 205    }
 206    assert(val.type() == RegType::vgpr);
 207    return val;
 208 }
 209
 210 //assumes a != 0xffffffff
 211 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 212 {
 213    assert(b != 0);
 214    Builder bld(ctx->program, ctx->block);
 215
 216    if (util_is_power_of_two_or_zero(b)) {
 217       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 218       return;
 219    }
 220
 221    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 222
 223    assert(info.multiplier <= 0xffffffff);
 224
 225    bool pre_shift = info.pre_shift != 0;
 226    bool increment = info.increment != 0;
 227    bool multiply = true;
 228    bool post_shift = info.post_shift != 0;
 229
 230    if (!pre_shift && !increment && !multiply && !post_shift) {
 231       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 232       return;
 233    }
 234
 235    Temp pre_shift_dst = a;
 236    if (pre_shift) {
 237       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 238       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 239    }
 240
 241    Temp increment_dst = pre_shift_dst;
 242    if (increment) {
 243       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 244       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 245    }
 246
 247    Temp multiply_dst = increment_dst;
 248    if (multiply) {
 249       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 250       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 251                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 252    }
 253
 254    if (post_shift) {
 255       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 256    }
 257 }
 258
 259 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 260 {
 261    Builder bld(ctx->program, ctx->block);
 262    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 263 }
 264
 265
 266 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 267 {
 268    /* no need to extract the whole vector */
 269    if (src.regClass() == dst_rc) {
 270       assert(idx == 0);
 271       return src;
 272    }
 273    assert(src.size() > idx);
 274    Builder bld(ctx->program, ctx->block);
 275    auto it = ctx->allocated_vec.find(src.id());
 276    /* the size check needs to be early because elements other than 0 may be garbage */
 277    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 278       if (it->second[idx].regClass() == dst_rc) {
 279          return it->second[idx];
 280       } else {
 281          assert(dst_rc.size() == it->second[idx].regClass().size());
 282          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 283          return bld.copy(bld.def(dst_rc), it->second[idx]);
 284       }
 285    }
 286
 287    if (src.size() == dst_rc.size()) {
 288       assert(idx == 0);
 289       return bld.copy(bld.def(dst_rc), src);
 290    } else {
 291       Temp dst = bld.tmp(dst_rc);
 292       emit_extract_vector(ctx, src, idx, dst);
 293       return dst;
 294    }
 295 }
 296
 297 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 298 {
 299    if (num_components == 1)
 300       return;
 301    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 302       return;
 303    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 304    split->operands[0] = Operand(vec_src);
 305    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 306    for (unsigned i = 0; i < num_components; i++) {
 307       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 308       split->definitions[i] = Definition(elems[i]);
 309    }
 310    ctx->block->instructions.emplace_back(std::move(split));
 311    ctx->allocated_vec.emplace(vec_src.id(), elems);
 312 }
 313
 314 /* This vector expansion uses a mask to determine which elements in the new vector
 315  * come from the original vector. The other elements are undefined. */
 316 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 317 {
 318    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 319
 320    if (vec_src == dst)
 321       return;
 322
 323    Builder bld(ctx->program, ctx->block);
 324    if (num_components == 1) {
 325       if (dst.type() == RegType::sgpr)
 326          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 327       else
 328          bld.copy(Definition(dst), vec_src);
 329       return;
 330    }
 331
 332    unsigned component_size = dst.size() / num_components;
 333    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 334
 335    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 336    vec->definitions[0] = Definition(dst);
 337    unsigned k = 0;
 338    for (unsigned i = 0; i < num_components; i++) {
 339       if (mask & (1 << i)) {
 340          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 341          if (dst.type() == RegType::sgpr)
 342             src = bld.as_uniform(src);
 343          vec->operands[i] = Operand(src);
 344       } else {
 345          vec->operands[i] = Operand(0u);
 346       }
 347       elems[i] = vec->operands[i].getTemp();
 348    }
 349    ctx->block->instructions.emplace_back(std::move(vec));
 350    ctx->allocated_vec.emplace(dst.id(), elems);
 351 }
 352
 353 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 354 {
 355    Builder bld(ctx->program, ctx->block);
 356    if (!dst.id())
 357       dst = bld.tmp(bld.lm);
 358
 359    assert(val.regClass() == s1);
 360    assert(dst.regClass() == bld.lm);
 361
 362    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 363 }
 364
 365 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 366 {
 367    Builder bld(ctx->program, ctx->block);
 368    if (!dst.id())
 369       dst = bld.tmp(s1);
 370
 371    assert(val.regClass() == bld.lm);
 372    assert(dst.regClass() == s1);
 373
 374    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 375    Temp tmp = bld.tmp(s1);
 376    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 377    return emit_wqm(ctx, tmp, dst);
 378 }
 379
 380 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 381 {
 382    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 383       return get_ssa_temp(ctx, src.src.ssa);
 384
 385    if (src.src.ssa->num_components == size) {
 386       bool identity_swizzle = true;
 387       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 388          if (src.swizzle[i] != i)
 389             identity_swizzle = false;
 390       }
 391       if (identity_swizzle)
 392          return get_ssa_temp(ctx, src.src.ssa);
 393    }
 394
 395    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 396    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 397    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 398    assert(vec.size() % elem_size == 0);
 399
 400    RegClass elem_rc = RegClass(vec.type(), elem_size);
 401    if (size == 1) {
 402       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 403    } else {
 404       assert(size <= 4);
 405       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 406       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 407       for (unsigned i = 0; i < size; ++i) {
 408          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 409          vec_instr->operands[i] = Operand{elems[i]};
 410       }
 411       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 412       vec_instr->definitions[0] = Definition(dst);
 413       ctx->block->instructions.emplace_back(std::move(vec_instr));
 414       ctx->allocated_vec.emplace(dst.id(), elems);
 415       return dst;
 416    }
 417 }
 418
 419 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 420 {
 421    if (ptr.size() == 2)
 422       return ptr;
 423    Builder bld(ctx->program, ctx->block);
 424    if (ptr.type() == RegType::vgpr)
 425       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 426    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 427                      ptr, Operand((unsigned)ctx->options->address32_hi));
 428 }
 429
 430 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 431 {
 432    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 433    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 434    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 435    sop2->definitions[0] = Definition(dst);
 436    if (writes_scc)
 437       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 438    ctx->block->instructions.emplace_back(std::move(sop2));
 439 }
 440
 441 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 442                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 443 {
 444    Builder bld(ctx->program, ctx->block);
 445    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 446    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 447    if (src1.type() == RegType::sgpr) {
 448       if (commutative && src0.type() == RegType::vgpr) {
 449          Temp t = src0;
 450          src0 = src1;
 451          src1 = t;
 452       } else if (src0.type() == RegType::vgpr &&
 453                  op != aco_opcode::v_madmk_f32 &&
 454                  op != aco_opcode::v_madak_f32 &&
 455                  op != aco_opcode::v_madmk_f16 &&
 456                  op != aco_opcode::v_madak_f16) {
 457          /* If the instruction is not commutative, we emit a VOP3A instruction */
 458          bld.vop2_e64(op, Definition(dst), src0, src1);
 459          return;
 460       } else {
 461          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 462       }
 463    }
 464
 465    if (flush_denorms && ctx->program->chip_class < GFX9) {
 466       assert(dst.size() == 1);
 467       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 468       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 469    } else {
 470       bld.vop2(op, Definition(dst), src0, src1);
 471    }
 472 }
 473
 474 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 475                             bool flush_denorms = false)
 476 {
 477    Temp src0 = get_alu_src(ctx, instr->src[0]);
 478    Temp src1 = get_alu_src(ctx, instr->src[1]);
 479    Temp src2 = get_alu_src(ctx, instr->src[2]);
 480
 481    /* ensure that the instruction has at most 1 sgpr operand
 482     * The optimizer will inline constants for us */
 483    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 484       src0 = as_vgpr(ctx, src0);
 485    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 486       src1 = as_vgpr(ctx, src1);
 487    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 488       src2 = as_vgpr(ctx, src2);
 489
 490    Builder bld(ctx->program, ctx->block);
 491    if (flush_denorms && ctx->program->chip_class < GFX9) {
 492       assert(dst.size() == 1);
 493       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 494       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 495    } else {
 496       bld.vop3(op, Definition(dst), src0, src1, src2);
 497    }
 498 }
 499
 500 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 501 {
 502    Builder bld(ctx->program, ctx->block);
 503    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 504 }
 505
 506 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 507 {
 508    Temp src0 = get_alu_src(ctx, instr->src[0]);
 509    Temp src1 = get_alu_src(ctx, instr->src[1]);
 510    assert(src0.size() == src1.size());
 511
 512    aco_ptr<Instruction> vopc;
 513    if (src1.type() == RegType::sgpr) {
 514       if (src0.type() == RegType::vgpr) {
 515          /* to swap the operands, we might also have to change the opcode */
 516          switch (op) {
 517             case aco_opcode::v_cmp_lt_f32:
 518                op = aco_opcode::v_cmp_gt_f32;
 519                break;
 520             case aco_opcode::v_cmp_ge_f32:
 521                op = aco_opcode::v_cmp_le_f32;
 522                break;
 523             case aco_opcode::v_cmp_lt_i32:
 524                op = aco_opcode::v_cmp_gt_i32;
 525                break;
 526             case aco_opcode::v_cmp_ge_i32:
 527                op = aco_opcode::v_cmp_le_i32;
 528                break;
 529             case aco_opcode::v_cmp_lt_u32:
 530                op = aco_opcode::v_cmp_gt_u32;
 531                break;
 532             case aco_opcode::v_cmp_ge_u32:
 533                op = aco_opcode::v_cmp_le_u32;
 534                break;
 535             case aco_opcode::v_cmp_lt_f64:
 536                op = aco_opcode::v_cmp_gt_f64;
 537                break;
 538             case aco_opcode::v_cmp_ge_f64:
 539                op = aco_opcode::v_cmp_le_f64;
 540                break;
 541             case aco_opcode::v_cmp_lt_i64:
 542                op = aco_opcode::v_cmp_gt_i64;
 543                break;
 544             case aco_opcode::v_cmp_ge_i64:
 545                op = aco_opcode::v_cmp_le_i64;
 546                break;
 547             case aco_opcode::v_cmp_lt_u64:
 548                op = aco_opcode::v_cmp_gt_u64;
 549                break;
 550             case aco_opcode::v_cmp_ge_u64:
 551                op = aco_opcode::v_cmp_le_u64;
 552                break;
 553             default: /* eq and ne are commutative */
 554                break;
 555          }
 556          Temp t = src0;
 557          src0 = src1;
 558          src1 = t;
 559       } else {
 560          src1 = as_vgpr(ctx, src1);
 561       }
 562    }
 563
 564    Builder bld(ctx->program, ctx->block);
 565    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 566 }
 567
 568 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 569 {
 570    Temp src0 = get_alu_src(ctx, instr->src[0]);
 571    Temp src1 = get_alu_src(ctx, instr->src[1]);
 572    Builder bld(ctx->program, ctx->block);
 573
 574    assert(dst.regClass() == bld.lm);
 575    assert(src0.type() == RegType::sgpr);
 576    assert(src1.type() == RegType::sgpr);
 577    assert(src0.regClass() == src1.regClass());
 578
 579    /* Emit the SALU comparison instruction */
 580    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 581    /* Turn the result into a per-lane bool */
 582    bool_to_vector_condition(ctx, cmp, dst);
 583 }
 584
 585 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 586                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 587 {
 588    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 589    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 590    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 591    bool use_valu = s_op == aco_opcode::num_opcodes ||
 592                    divergent_vals ||
 593                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 594                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 595    aco_opcode op = use_valu ? v_op : s_op;
 596    assert(op != aco_opcode::num_opcodes);
 597    assert(dst.regClass() == ctx->program->lane_mask);
 598
 599    if (use_valu)
 600       emit_vopc_instruction(ctx, instr, op, dst);
 601    else
 602       emit_sopc_instruction(ctx, instr, op, dst);
 603 }
 604
 605 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 606 {
 607    Builder bld(ctx->program, ctx->block);
 608    Temp src0 = get_alu_src(ctx, instr->src[0]);
 609    Temp src1 = get_alu_src(ctx, instr->src[1]);
 610
 611    assert(dst.regClass() == bld.lm);
 612    assert(src0.regClass() == bld.lm);
 613    assert(src1.regClass() == bld.lm);
 614
 615    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 616 }
 617
 618 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 619 {
 620    Builder bld(ctx->program, ctx->block);
 621    Temp cond = get_alu_src(ctx, instr->src[0]);
 622    Temp then = get_alu_src(ctx, instr->src[1]);
 623    Temp els = get_alu_src(ctx, instr->src[2]);
 624
 625    assert(cond.regClass() == bld.lm);
 626
 627    if (dst.type() == RegType::vgpr) {
 628       aco_ptr<Instruction> bcsel;
 629       if (dst.size() == 1) {
 630          then = as_vgpr(ctx, then);
 631          els = as_vgpr(ctx, els);
 632
 633          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 634       } else if (dst.size() == 2) {
 635          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 637          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 638          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 639
 640          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 641          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 642
 643          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 644       } else {
 645          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 646          nir_print_instr(&instr->instr, stderr);
 647          fprintf(stderr, "\n");
 648       }
 649       return;
 650    }
 651
 652    if (instr->dest.dest.ssa.bit_size == 1) {
 653       assert(dst.regClass() == bld.lm);
 654       assert(then.regClass() == bld.lm);
 655       assert(els.regClass() == bld.lm);
 656    }
 657
 658    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 659       if (dst.regClass() == s1 || dst.regClass() == s2) {
 660          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 661          assert(dst.size() == then.size());
 662          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 663          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 664       } else {
 665          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 666          nir_print_instr(&instr->instr, stderr);
 667          fprintf(stderr, "\n");
 668       }
 669       return;
 670    }
 671
 672    /* divergent boolean bcsel
 673     * this implements bcsel on bools: dst = s0 ? s1 : s2
 674     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 675    assert(instr->dest.dest.ssa.bit_size == 1);
 676
 677    if (cond.id() != then.id())
 678       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 679
 680    if (cond.id() == els.id())
 681       bld.sop1(Builder::s_mov, Definition(dst), then);
 682    else
 683       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 684                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 685 }
 686
 687 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 688                     aco_opcode op, uint32_t undo)
 689 {
 690    /* multiply by 16777216 to handle denormals */
 691    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 692                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 693    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 694    scaled = bld.vop1(op, bld.def(v1), scaled);
 695    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 696
 697    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 698
 699    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 700 }
 701
 702 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 703 {
 704    if (ctx->block->fp_mode.denorm32 == 0) {
 705       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 706       return;
 707    }
 708
 709    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 710 }
 711
 712 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 713 {
 714    if (ctx->block->fp_mode.denorm32 == 0) {
 715       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 716       return;
 717    }
 718
 719    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 720 }
 721
 722 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 723 {
 724    if (ctx->block->fp_mode.denorm32 == 0) {
 725       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 726       return;
 727    }
 728
 729    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 730 }
 731
 732 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 733 {
 734    if (ctx->block->fp_mode.denorm32 == 0) {
 735       bld.vop1(aco_opcode::v_log_f32, dst, val);
 736       return;
 737    }
 738
 739    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 740 }
 741
 742 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 743 {
 744    if (ctx->options->chip_class >= GFX7)
 745       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 746
 747    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 748    /* TODO: create more efficient code! */
 749    if (val.type() == RegType::sgpr)
 750       val = as_vgpr(ctx, val);
 751
 752    /* Split the input value. */
 753    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 754    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 755
 756    /* Extract the exponent and compute the unbiased value. */
 757    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 758
 759    /* Extract the fractional part. */
 760    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 761    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 762
 763    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 764    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 765
 766    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 767    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 768    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 769    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 770    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 771
 772    /* Get the sign bit. */
 773    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 774
 775    /* Decide the operation to apply depending on the unbiased exponent. */
 776    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 777    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 778    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 779    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 780    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 781    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 782
 783    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 784 }
 785
 786 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 787 {
 788    if (ctx->options->chip_class >= GFX7)
 789       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 790
 791    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 792    Temp src0 = as_vgpr(ctx, val);
 793
 794    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 795    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 796
 797    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 798    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 799    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 800
 801    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 803    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 804    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 805
 806    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 807    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 808
 809    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 810
 811    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 812    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 813
 814    return add->definitions[0].getTemp();
 815 }
 816
 817 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 818 {
 819    if (!instr->dest.dest.is_ssa) {
 820       fprintf(stderr, "nir alu dst not in ssa: ");
 821       nir_print_instr(&instr->instr, stderr);
 822       fprintf(stderr, "\n");
 823       abort();
 824    }
 825    Builder bld(ctx->program, ctx->block);
 826    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 827    switch(instr->op) {
 828    case nir_op_vec2:
 829    case nir_op_vec3:
 830    case nir_op_vec4: {
 831       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 832       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 833       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 834          elems[i] = get_alu_src(ctx, instr->src[i]);
 835          vec->operands[i] = Operand{elems[i]};
 836       }
 837       vec->definitions[0] = Definition(dst);
 838       ctx->block->instructions.emplace_back(std::move(vec));
 839       ctx->allocated_vec.emplace(dst.id(), elems);
 840       break;
 841    }
 842    case nir_op_mov: {
 843       Temp src = get_alu_src(ctx, instr->src[0]);
 844       aco_ptr<Instruction> mov;
 845       if (dst.type() == RegType::sgpr) {
 846          if (src.type() == RegType::vgpr)
 847             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 848          else if (src.regClass() == s1)
 849             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 850          else if (src.regClass() == s2)
 851             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 852          else
 853             unreachable("wrong src register class for nir_op_imov");
 854       } else if (dst.regClass() == v1) {
 855          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 856       } else if (dst.regClass() == v2) {
 857          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 858       } else {
 859          nir_print_instr(&instr->instr, stderr);
 860          unreachable("Should have been lowered to scalar.");
 861       }
 862       break;
 863    }
 864    case nir_op_inot: {
 865       Temp src = get_alu_src(ctx, instr->src[0]);
 866       if (instr->dest.dest.ssa.bit_size == 1) {
 867          assert(src.regClass() == bld.lm);
 868          assert(dst.regClass() == bld.lm);
 869          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 870          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 871          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 872       } else if (dst.regClass() == v1) {
 873          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 874       } else if (dst.type() == RegType::sgpr) {
 875          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 876          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 877       } else {
 878          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 879          nir_print_instr(&instr->instr, stderr);
 880          fprintf(stderr, "\n");
 881       }
 882       break;
 883    }
 884    case nir_op_ineg: {
 885       Temp src = get_alu_src(ctx, instr->src[0]);
 886       if (dst.regClass() == v1) {
 887          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 888       } else if (dst.regClass() == s1) {
 889          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 890       } else if (dst.size() == 2) {
 891          Temp src0 = bld.tmp(dst.type(), 1);
 892          Temp src1 = bld.tmp(dst.type(), 1);
 893          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 894
 895          if (dst.regClass() == s2) {
 896             Temp carry = bld.tmp(s1);
 897             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 898             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 899             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 900          } else {
 901             Temp lower = bld.tmp(v1);
 902             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 903             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 904             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 905          }
 906       } else {
 907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 908          nir_print_instr(&instr->instr, stderr);
 909          fprintf(stderr, "\n");
 910       }
 911       break;
 912    }
 913    case nir_op_iabs: {
 914       if (dst.regClass() == s1) {
 915          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 916       } else if (dst.regClass() == v1) {
 917          Temp src = get_alu_src(ctx, instr->src[0]);
 918          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 919       } else {
 920          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 921          nir_print_instr(&instr->instr, stderr);
 922          fprintf(stderr, "\n");
 923       }
 924       break;
 925    }
 926    case nir_op_isign: {
 927       Temp src = get_alu_src(ctx, instr->src[0]);
 928       if (dst.regClass() == s1) {
 929          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 930          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 931          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 932       } else if (dst.regClass() == s2) {
 933          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 934          Temp neqz;
 935          if (ctx->program->chip_class >= GFX8)
 936             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 937          else
 938             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 939          /* SCC gets zero-extended to 64 bit */
 940          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 941       } else if (dst.regClass() == v1) {
 942          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 943          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 944          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 945       } else if (dst.regClass() == v2) {
 946          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 947          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 948          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 949          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 950          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 951          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_imax: {
 960       if (dst.regClass() == v1) {
 961          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 962       } else if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 964       } else {
 965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 966          nir_print_instr(&instr->instr, stderr);
 967          fprintf(stderr, "\n");
 968       }
 969       break;
 970    }
 971    case nir_op_umax: {
 972       if (dst.regClass() == v1) {
 973          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 974       } else if (dst.regClass() == s1) {
 975          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 976       } else {
 977          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 978          nir_print_instr(&instr->instr, stderr);
 979          fprintf(stderr, "\n");
 980       }
 981       break;
 982    }
 983    case nir_op_imin: {
 984       if (dst.regClass() == v1) {
 985          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 986       } else if (dst.regClass() == s1) {
 987          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 988       } else {
 989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 990          nir_print_instr(&instr->instr, stderr);
 991          fprintf(stderr, "\n");
 992       }
 993       break;
 994    }
 995    case nir_op_umin: {
 996       if (dst.regClass() == v1) {
 997          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 998       } else if (dst.regClass() == s1) {
 999          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1000       } else {
1001          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1002          nir_print_instr(&instr->instr, stderr);
1003          fprintf(stderr, "\n");
1004       }
1005       break;
1006    }
1007    case nir_op_ior: {
1008       if (instr->dest.dest.ssa.bit_size == 1) {
1009          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1010       } else if (dst.regClass() == v1) {
1011          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1012       } else if (dst.regClass() == s1) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1014       } else if (dst.regClass() == s2) {
1015          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1016       } else {
1017          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1018          nir_print_instr(&instr->instr, stderr);
1019          fprintf(stderr, "\n");
1020       }
1021       break;
1022    }
1023    case nir_op_iand: {
1024       if (instr->dest.dest.ssa.bit_size == 1) {
1025          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1026       } else if (dst.regClass() == v1) {
1027          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1028       } else if (dst.regClass() == s1) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1030       } else if (dst.regClass() == s2) {
1031          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1032       } else {
1033          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1034          nir_print_instr(&instr->instr, stderr);
1035          fprintf(stderr, "\n");
1036       }
1037       break;
1038    }
1039    case nir_op_ixor: {
1040       if (instr->dest.dest.ssa.bit_size == 1) {
1041          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1042       } else if (dst.regClass() == v1) {
1043          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1044       } else if (dst.regClass() == s1) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1046       } else if (dst.regClass() == s2) {
1047          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1048       } else {
1049          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1050          nir_print_instr(&instr->instr, stderr);
1051          fprintf(stderr, "\n");
1052       }
1053       break;
1054    }
1055    case nir_op_ushr: {
1056       if (dst.regClass() == v1) {
1057          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1058       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1059          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1060                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1061       } else if (dst.regClass() == v2) {
1062          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1063                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1064       } else if (dst.regClass() == s2) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1066       } else if (dst.regClass() == s1) {
1067          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1068       } else {
1069          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1070          nir_print_instr(&instr->instr, stderr);
1071          fprintf(stderr, "\n");
1072       }
1073       break;
1074    }
1075    case nir_op_ishl: {
1076       if (dst.regClass() == v1) {
1077          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1078       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1079          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1080                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1081       } else if (dst.regClass() == v2) {
1082          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1083                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1084       } else if (dst.regClass() == s1) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1086       } else if (dst.regClass() == s2) {
1087          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1088       } else {
1089          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1090          nir_print_instr(&instr->instr, stderr);
1091          fprintf(stderr, "\n");
1092       }
1093       break;
1094    }
1095    case nir_op_ishr: {
1096       if (dst.regClass() == v1) {
1097          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1098       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1099          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1100                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1101       } else if (dst.regClass() == v2) {
1102          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1103                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1104       } else if (dst.regClass() == s1) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1106       } else if (dst.regClass() == s2) {
1107          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1108       } else {
1109          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1110          nir_print_instr(&instr->instr, stderr);
1111          fprintf(stderr, "\n");
1112       }
1113       break;
1114    }
1115    case nir_op_find_lsb: {
1116       Temp src = get_alu_src(ctx, instr->src[0]);
1117       if (src.regClass() == s1) {
1118          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1119       } else if (src.regClass() == v1) {
1120          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1121       } else if (src.regClass() == s2) {
1122          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_ufind_msb:
1131    case nir_op_ifind_msb: {
1132       Temp src = get_alu_src(ctx, instr->src[0]);
1133       if (src.regClass() == s1 || src.regClass() == s2) {
1134          aco_opcode op = src.regClass() == s2 ?
1135                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1136                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1137          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1138
1139          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1140                                         Operand(src.size() * 32u - 1u), msb_rev);
1141          Temp msb = sub.def(0).getTemp();
1142          Temp carry = sub.def(1).getTemp();
1143
1144          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1145       } else if (src.regClass() == v1) {
1146          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1147          Temp msb_rev = bld.tmp(v1);
1148          emit_vop1_instruction(ctx, instr, op, msb_rev);
1149          Temp msb = bld.tmp(v1);
1150          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1151          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1152       } else {
1153          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1154          nir_print_instr(&instr->instr, stderr);
1155          fprintf(stderr, "\n");
1156       }
1157       break;
1158    }
1159    case nir_op_bitfield_reverse: {
1160       if (dst.regClass() == s1) {
1161          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else if (dst.regClass() == v1) {
1163          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1164       } else {
1165          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1166          nir_print_instr(&instr->instr, stderr);
1167          fprintf(stderr, "\n");
1168       }
1169       break;
1170    }
1171    case nir_op_iadd: {
1172       if (dst.regClass() == s1) {
1173          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1174          break;
1175       }
1176
1177       Temp src0 = get_alu_src(ctx, instr->src[0]);
1178       Temp src1 = get_alu_src(ctx, instr->src[1]);
1179       if (dst.regClass() == v1) {
1180          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1181          break;
1182       }
1183
1184       assert(src0.size() == 2 && src1.size() == 2);
1185       Temp src00 = bld.tmp(src0.type(), 1);
1186       Temp src01 = bld.tmp(dst.type(), 1);
1187       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1188       Temp src10 = bld.tmp(src1.type(), 1);
1189       Temp src11 = bld.tmp(dst.type(), 1);
1190       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1191
1192       if (dst.regClass() == s2) {
1193          Temp carry = bld.tmp(s1);
1194          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1195          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1196          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1197       } else if (dst.regClass() == v2) {
1198          Temp dst0 = bld.tmp(v1);
1199          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1200          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1201          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1202       } else {
1203          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1204          nir_print_instr(&instr->instr, stderr);
1205          fprintf(stderr, "\n");
1206       }
1207       break;
1208    }
1209    case nir_op_uadd_sat: {
1210       Temp src0 = get_alu_src(ctx, instr->src[0]);
1211       Temp src1 = get_alu_src(ctx, instr->src[1]);
1212       if (dst.regClass() == s1) {
1213          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1214          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1215                   src0, src1);
1216          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1217       } else if (dst.regClass() == v1) {
1218          if (ctx->options->chip_class >= GFX9) {
1219             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1220             add->operands[0] = Operand(src0);
1221             add->operands[1] = Operand(src1);
1222             add->definitions[0] = Definition(dst);
1223             add->clamp = 1;
1224             ctx->block->instructions.emplace_back(std::move(add));
1225          } else {
1226             if (src1.regClass() != v1)
1227                std::swap(src0, src1);
1228             assert(src1.regClass() == v1);
1229             Temp tmp = bld.tmp(v1);
1230             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1231             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1232          }
1233       } else {
1234          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1235          nir_print_instr(&instr->instr, stderr);
1236          fprintf(stderr, "\n");
1237       }
1238       break;
1239    }
1240    case nir_op_uadd_carry: {
1241       Temp src0 = get_alu_src(ctx, instr->src[0]);
1242       Temp src1 = get_alu_src(ctx, instr->src[1]);
1243       if (dst.regClass() == s1) {
1244          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1245          break;
1246       }
1247       if (dst.regClass() == v1) {
1248          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1249          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1250          break;
1251       }
1252
1253       Temp src00 = bld.tmp(src0.type(), 1);
1254       Temp src01 = bld.tmp(dst.type(), 1);
1255       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1256       Temp src10 = bld.tmp(src1.type(), 1);
1257       Temp src11 = bld.tmp(dst.type(), 1);
1258       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1259       if (dst.regClass() == s2) {
1260          Temp carry = bld.tmp(s1);
1261          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1262          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1263          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1264       } else if (dst.regClass() == v2) {
1265          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1266          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1267          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1268          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1269       } else {
1270          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1271          nir_print_instr(&instr->instr, stderr);
1272          fprintf(stderr, "\n");
1273       }
1274       break;
1275    }
1276    case nir_op_isub: {
1277       if (dst.regClass() == s1) {
1278          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1279          break;
1280       }
1281
1282       Temp src0 = get_alu_src(ctx, instr->src[0]);
1283       Temp src1 = get_alu_src(ctx, instr->src[1]);
1284       if (dst.regClass() == v1) {
1285          bld.vsub32(Definition(dst), src0, src1);
1286          break;
1287       }
1288
1289       Temp src00 = bld.tmp(src0.type(), 1);
1290       Temp src01 = bld.tmp(dst.type(), 1);
1291       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1292       Temp src10 = bld.tmp(src1.type(), 1);
1293       Temp src11 = bld.tmp(dst.type(), 1);
1294       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1295       if (dst.regClass() == s2) {
1296          Temp carry = bld.tmp(s1);
1297          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1298          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1299          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1300       } else if (dst.regClass() == v2) {
1301          Temp lower = bld.tmp(v1);
1302          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1303          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1304          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1305       } else {
1306          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1307          nir_print_instr(&instr->instr, stderr);
1308          fprintf(stderr, "\n");
1309       }
1310       break;
1311    }
1312    case nir_op_usub_borrow: {
1313       Temp src0 = get_alu_src(ctx, instr->src[0]);
1314       Temp src1 = get_alu_src(ctx, instr->src[1]);
1315       if (dst.regClass() == s1) {
1316          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1317          break;
1318       } else if (dst.regClass() == v1) {
1319          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1320          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1321          break;
1322       }
1323
1324       Temp src00 = bld.tmp(src0.type(), 1);
1325       Temp src01 = bld.tmp(dst.type(), 1);
1326       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1327       Temp src10 = bld.tmp(src1.type(), 1);
1328       Temp src11 = bld.tmp(dst.type(), 1);
1329       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1330       if (dst.regClass() == s2) {
1331          Temp borrow = bld.tmp(s1);
1332          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1333          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1334          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1335       } else if (dst.regClass() == v2) {
1336          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1337          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1338          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1339          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1340       } else {
1341          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1342          nir_print_instr(&instr->instr, stderr);
1343          fprintf(stderr, "\n");
1344       }
1345       break;
1346    }
1347    case nir_op_imul: {
1348       if (dst.regClass() == v1) {
1349          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1350                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1351       } else if (dst.regClass() == s1) {
1352          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1353       } else {
1354          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1355          nir_print_instr(&instr->instr, stderr);
1356          fprintf(stderr, "\n");
1357       }
1358       break;
1359    }
1360    case nir_op_umul_high: {
1361       if (dst.regClass() == v1) {
1362          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1364          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1365       } else if (dst.regClass() == s1) {
1366          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1367                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1368          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1369       } else {
1370          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1371          nir_print_instr(&instr->instr, stderr);
1372          fprintf(stderr, "\n");
1373       }
1374       break;
1375    }
1376    case nir_op_imul_high: {
1377       if (dst.regClass() == v1) {
1378          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1380          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1381       } else if (dst.regClass() == s1) {
1382          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1383                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1384          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1385       } else {
1386          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1387          nir_print_instr(&instr->instr, stderr);
1388          fprintf(stderr, "\n");
1389       }
1390       break;
1391    }
1392    case nir_op_fmul: {
1393       if (dst.size() == 1) {
1394          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1395       } else if (dst.size() == 2) {
1396          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1397                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1398       } else {
1399          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1400          nir_print_instr(&instr->instr, stderr);
1401          fprintf(stderr, "\n");
1402       }
1403       break;
1404    }
1405    case nir_op_fadd: {
1406       if (dst.size() == 1) {
1407          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1408       } else if (dst.size() == 2) {
1409          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1410                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1411       } else {
1412          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1413          nir_print_instr(&instr->instr, stderr);
1414          fprintf(stderr, "\n");
1415       }
1416       break;
1417    }
1418    case nir_op_fsub: {
1419       Temp src0 = get_alu_src(ctx, instr->src[0]);
1420       Temp src1 = get_alu_src(ctx, instr->src[1]);
1421       if (dst.size() == 1) {
1422          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1424          else
1425             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1426       } else if (dst.size() == 2) {
1427          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1428                                      get_alu_src(ctx, instr->src[0]),
1429                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1430          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1431          sub->neg[1] = true;
1432       } else {
1433          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1434          nir_print_instr(&instr->instr, stderr);
1435          fprintf(stderr, "\n");
1436       }
1437       break;
1438    }
1439    case nir_op_fmax: {
1440       if (dst.size() == 1) {
1441          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1442       } else if (dst.size() == 2) {
1443          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1444             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1445                                 get_alu_src(ctx, instr->src[0]),
1446                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1447             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1448          } else {
1449             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1450                      get_alu_src(ctx, instr->src[0]),
1451                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1452          }
1453       } else {
1454          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1455          nir_print_instr(&instr->instr, stderr);
1456          fprintf(stderr, "\n");
1457       }
1458       break;
1459    }
1460    case nir_op_fmin: {
1461       if (dst.size() == 1) {
1462          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1463       } else if (dst.size() == 2) {
1464          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1465             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1466                                 get_alu_src(ctx, instr->src[0]),
1467                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1468             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1469          } else {
1470             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1471                      get_alu_src(ctx, instr->src[0]),
1472                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1473          }
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_fmax3: {
1482       if (dst.size() == 1) {
1483          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fmin3: {
1492       if (dst.size() == 1) {
1493          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1494       } else {
1495          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1496          nir_print_instr(&instr->instr, stderr);
1497          fprintf(stderr, "\n");
1498       }
1499       break;
1500    }
1501    case nir_op_fmed3: {
1502       if (dst.size() == 1) {
1503          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1504       } else {
1505          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1506          nir_print_instr(&instr->instr, stderr);
1507          fprintf(stderr, "\n");
1508       }
1509       break;
1510    }
1511    case nir_op_umax3: {
1512       if (dst.size() == 1) {
1513          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1514       } else {
1515          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1516          nir_print_instr(&instr->instr, stderr);
1517          fprintf(stderr, "\n");
1518       }
1519       break;
1520    }
1521    case nir_op_umin3: {
1522       if (dst.size() == 1) {
1523          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1524       } else {
1525          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1526          nir_print_instr(&instr->instr, stderr);
1527          fprintf(stderr, "\n");
1528       }
1529       break;
1530    }
1531    case nir_op_umed3: {
1532       if (dst.size() == 1) {
1533          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1534       } else {
1535          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1536          nir_print_instr(&instr->instr, stderr);
1537          fprintf(stderr, "\n");
1538       }
1539       break;
1540    }
1541    case nir_op_imax3: {
1542       if (dst.size() == 1) {
1543          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_imin3: {
1552       if (dst.size() == 1) {
1553          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1554       } else {
1555          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1556          nir_print_instr(&instr->instr, stderr);
1557          fprintf(stderr, "\n");
1558       }
1559       break;
1560    }
1561    case nir_op_imed3: {
1562       if (dst.size() == 1) {
1563          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1564       } else {
1565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1566          nir_print_instr(&instr->instr, stderr);
1567          fprintf(stderr, "\n");
1568       }
1569       break;
1570    }
1571    case nir_op_cube_face_coord: {
1572       Temp in = get_alu_src(ctx, instr->src[0], 3);
1573       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1574                       emit_extract_vector(ctx, in, 1, v1),
1575                       emit_extract_vector(ctx, in, 2, v1) };
1576       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1577       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1578       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1579       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1580       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1581       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1582       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1583       break;
1584    }
1585    case nir_op_cube_face_index: {
1586       Temp in = get_alu_src(ctx, instr->src[0], 3);
1587       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1588                       emit_extract_vector(ctx, in, 1, v1),
1589                       emit_extract_vector(ctx, in, 2, v1) };
1590       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1591       break;
1592    }
1593    case nir_op_bcsel: {
1594       emit_bcsel(ctx, instr, dst);
1595       break;
1596    }
1597    case nir_op_frsq: {
1598       if (dst.size() == 1) {
1599          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1600       } else if (dst.size() == 2) {
1601          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1602       } else {
1603          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1604          nir_print_instr(&instr->instr, stderr);
1605          fprintf(stderr, "\n");
1606       }
1607       break;
1608    }
1609    case nir_op_fneg: {
1610       Temp src = get_alu_src(ctx, instr->src[0]);
1611       if (dst.size() == 1) {
1612          if (ctx->block->fp_mode.must_flush_denorms32)
1613             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1614          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1615       } else if (dst.size() == 2) {
1616          if (ctx->block->fp_mode.must_flush_denorms16_64)
1617             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1618          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1619          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1620          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1621          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1622       } else {
1623          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1624          nir_print_instr(&instr->instr, stderr);
1625          fprintf(stderr, "\n");
1626       }
1627       break;
1628    }
1629    case nir_op_fabs: {
1630       Temp src = get_alu_src(ctx, instr->src[0]);
1631       if (dst.size() == 1) {
1632          if (ctx->block->fp_mode.must_flush_denorms32)
1633             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1634          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1635       } else if (dst.size() == 2) {
1636          if (ctx->block->fp_mode.must_flush_denorms16_64)
1637             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1638          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1639          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1640          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1642       } else {
1643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1644          nir_print_instr(&instr->instr, stderr);
1645          fprintf(stderr, "\n");
1646       }
1647       break;
1648    }
1649    case nir_op_fsat: {
1650       Temp src = get_alu_src(ctx, instr->src[0]);
1651       if (dst.size() == 1) {
1652          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1653          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1654          // TODO: confirm that this holds under any circumstances
1655       } else if (dst.size() == 2) {
1656          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1657          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1658          vop3->clamp = true;
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_flog2: {
1667       if (dst.size() == 1) {
1668          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_frcp: {
1677       if (dst.size() == 1) {
1678          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1679       } else if (dst.size() == 2) {
1680          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1681       } else {
1682          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1683          nir_print_instr(&instr->instr, stderr);
1684          fprintf(stderr, "\n");
1685       }
1686       break;
1687    }
1688    case nir_op_fexp2: {
1689       if (dst.size() == 1) {
1690          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1691       } else {
1692          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1693          nir_print_instr(&instr->instr, stderr);
1694          fprintf(stderr, "\n");
1695       }
1696       break;
1697    }
1698    case nir_op_fsqrt: {
1699       if (dst.size() == 1) {
1700          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1701       } else if (dst.size() == 2) {
1702          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1703       } else {
1704          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1705          nir_print_instr(&instr->instr, stderr);
1706          fprintf(stderr, "\n");
1707       }
1708       break;
1709    }
1710    case nir_op_ffract: {
1711       if (dst.size() == 1) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1713       } else if (dst.size() == 2) {
1714          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1715       } else {
1716          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1717          nir_print_instr(&instr->instr, stderr);
1718          fprintf(stderr, "\n");
1719       }
1720       break;
1721    }
1722    case nir_op_ffloor: {
1723       if (dst.size() == 1) {
1724          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1725       } else if (dst.size() == 2) {
1726          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1727       } else {
1728          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1729          nir_print_instr(&instr->instr, stderr);
1730          fprintf(stderr, "\n");
1731       }
1732       break;
1733    }
1734    case nir_op_fceil: {
1735       if (dst.size() == 1) {
1736          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1737       } else if (dst.size() == 2) {
1738          if (ctx->options->chip_class >= GFX7) {
1739             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1740          } else {
1741             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1742             Temp src0 = get_alu_src(ctx, instr->src[0]);
1743
1744             /* trunc = trunc(src0)
1745              * if (src0 > 0.0 && src0 != trunc)
1746              *    trunc += 1.0
1747              */
1748             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1749             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1750             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1751             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1752             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1753             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1754             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1755          }
1756       } else {
1757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1758          nir_print_instr(&instr->instr, stderr);
1759          fprintf(stderr, "\n");
1760       }
1761       break;
1762    }
1763    case nir_op_ftrunc: {
1764       if (dst.size() == 1) {
1765          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1766       } else if (dst.size() == 2) {
1767          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1768       } else {
1769          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1770          nir_print_instr(&instr->instr, stderr);
1771          fprintf(stderr, "\n");
1772       }
1773       break;
1774    }
1775    case nir_op_fround_even: {
1776       if (dst.size() == 1) {
1777          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1778       } else if (dst.size() == 2) {
1779          if (ctx->options->chip_class >= GFX7) {
1780             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1781          } else {
1782             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1783             Temp src0 = get_alu_src(ctx, instr->src[0]);
1784
1785             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1786             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1787
1788             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1789             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1790             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1791             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1792             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1793             tmp = sub->definitions[0].getTemp();
1794
1795             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1796             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1797             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1798             Temp cond = vop3->definitions[0].getTemp();
1799
1800             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1801             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1802             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1803             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1804
1805             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1806          }
1807       } else {
1808          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1809          nir_print_instr(&instr->instr, stderr);
1810          fprintf(stderr, "\n");
1811       }
1812       break;
1813    }
1814    case nir_op_fsin:
1815    case nir_op_fcos: {
1816       Temp src = get_alu_src(ctx, instr->src[0]);
1817       aco_ptr<Instruction> norm;
1818       if (dst.size() == 1) {
1819          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1820          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1821
1822          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1823          if (ctx->options->chip_class < GFX9)
1824             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1825
1826          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1827          bld.vop1(opcode, Definition(dst), tmp);
1828       } else {
1829          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1830          nir_print_instr(&instr->instr, stderr);
1831          fprintf(stderr, "\n");
1832       }
1833       break;
1834    }
1835    case nir_op_ldexp: {
1836       if (dst.size() == 1) {
1837          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1838                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1839                   get_alu_src(ctx, instr->src[1]));
1840       } else if (dst.size() == 2) {
1841          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1842                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1843                   get_alu_src(ctx, instr->src[1]));
1844       } else {
1845          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1846          nir_print_instr(&instr->instr, stderr);
1847          fprintf(stderr, "\n");
1848       }
1849       break;
1850    }
1851    case nir_op_frexp_sig: {
1852       if (dst.size() == 1) {
1853          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1854                   get_alu_src(ctx, instr->src[0]));
1855       } else if (dst.size() == 2) {
1856          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1857                   get_alu_src(ctx, instr->src[0]));
1858       } else {
1859          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1860          nir_print_instr(&instr->instr, stderr);
1861          fprintf(stderr, "\n");
1862       }
1863       break;
1864    }
1865    case nir_op_frexp_exp: {
1866       if (instr->src[0].src.ssa->bit_size == 32) {
1867          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1868                   get_alu_src(ctx, instr->src[0]));
1869       } else if (instr->src[0].src.ssa->bit_size == 64) {
1870          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1871                   get_alu_src(ctx, instr->src[0]));
1872       } else {
1873          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1874          nir_print_instr(&instr->instr, stderr);
1875          fprintf(stderr, "\n");
1876       }
1877       break;
1878    }
1879    case nir_op_fsign: {
1880       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1881       if (dst.size() == 1) {
1882          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1884          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1885          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1886       } else if (dst.size() == 2) {
1887          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1888          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1889          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1890
1891          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1892          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1893          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1894
1895          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1896       } else {
1897          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1898          nir_print_instr(&instr->instr, stderr);
1899          fprintf(stderr, "\n");
1900       }
1901       break;
1902    }
1903    case nir_op_f2f32: {
1904       if (instr->src[0].src.ssa->bit_size == 64) {
1905          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1906       } else {
1907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1908          nir_print_instr(&instr->instr, stderr);
1909          fprintf(stderr, "\n");
1910       }
1911       break;
1912    }
1913    case nir_op_f2f64: {
1914       if (instr->src[0].src.ssa->bit_size == 32) {
1915          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1916       } else {
1917          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1918          nir_print_instr(&instr->instr, stderr);
1919          fprintf(stderr, "\n");
1920       }
1921       break;
1922    }
1923    case nir_op_i2f32: {
1924       assert(dst.size() == 1);
1925       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1926       break;
1927    }
1928    case nir_op_i2f64: {
1929       if (instr->src[0].src.ssa->bit_size == 32) {
1930          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1931       } else if (instr->src[0].src.ssa->bit_size == 64) {
1932          Temp src = get_alu_src(ctx, instr->src[0]);
1933          RegClass rc = RegClass(src.type(), 1);
1934          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1935          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1936          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1937          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1938          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1939          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1940
1941       } else {
1942          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1943          nir_print_instr(&instr->instr, stderr);
1944          fprintf(stderr, "\n");
1945       }
1946       break;
1947    }
1948    case nir_op_u2f32: {
1949       assert(dst.size() == 1);
1950       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1951       break;
1952    }
1953    case nir_op_u2f64: {
1954       if (instr->src[0].src.ssa->bit_size == 32) {
1955          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1956       } else if (instr->src[0].src.ssa->bit_size == 64) {
1957          Temp src = get_alu_src(ctx, instr->src[0]);
1958          RegClass rc = RegClass(src.type(), 1);
1959          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1960          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1961          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1962          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1963          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1964          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1965       } else {
1966          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1967          nir_print_instr(&instr->instr, stderr);
1968          fprintf(stderr, "\n");
1969       }
1970       break;
1971    }
1972    case nir_op_f2i32: {
1973       Temp src = get_alu_src(ctx, instr->src[0]);
1974       if (instr->src[0].src.ssa->bit_size == 32) {
1975          if (dst.type() == RegType::vgpr)
1976             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1977          else
1978             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1979                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1980
1981       } else if (instr->src[0].src.ssa->bit_size == 64) {
1982          if (dst.type() == RegType::vgpr)
1983             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1984          else
1985             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1986                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1987
1988       } else {
1989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1990          nir_print_instr(&instr->instr, stderr);
1991          fprintf(stderr, "\n");
1992       }
1993       break;
1994    }
1995    case nir_op_f2u32: {
1996       Temp src = get_alu_src(ctx, instr->src[0]);
1997       if (instr->src[0].src.ssa->bit_size == 32) {
1998          if (dst.type() == RegType::vgpr)
1999             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2000          else
2001             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2002                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2003
2004       } else if (instr->src[0].src.ssa->bit_size == 64) {
2005          if (dst.type() == RegType::vgpr)
2006             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2007          else
2008             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2009                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2010
2011       } else {
2012          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2013          nir_print_instr(&instr->instr, stderr);
2014          fprintf(stderr, "\n");
2015       }
2016       break;
2017    }
2018    case nir_op_f2i64: {
2019       Temp src = get_alu_src(ctx, instr->src[0]);
2020       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2021          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2022          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2023          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2024          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2025          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2026          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2027          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2028          Temp new_exponent = bld.tmp(v1);
2029          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2030          if (ctx->program->chip_class >= GFX8)
2031             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2032          else
2033             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2034          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2035          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2036          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2037          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2038          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2039          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2040          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2041          Temp new_lower = bld.tmp(v1);
2042          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2043          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2044          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2045
2046       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2047          if (src.type() == RegType::vgpr)
2048             src = bld.as_uniform(src);
2049          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2050          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2051          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2052          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2053          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2054          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2055          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2056          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2057          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2058          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2059          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2060          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2061          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2062          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2063          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2064          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2065          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2066          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2067          Temp borrow = bld.tmp(s1);
2068          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2069          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2070          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2071
2072       } else if (instr->src[0].src.ssa->bit_size == 64) {
2073          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2074          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2075          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2076          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2077          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2078          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2079          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2080          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2081          if (dst.type() == RegType::sgpr) {
2082             lower = bld.as_uniform(lower);
2083             upper = bld.as_uniform(upper);
2084          }
2085          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2086
2087       } else {
2088          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2089          nir_print_instr(&instr->instr, stderr);
2090          fprintf(stderr, "\n");
2091       }
2092       break;
2093    }
2094    case nir_op_f2u64: {
2095       Temp src = get_alu_src(ctx, instr->src[0]);
2096       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2097          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2098          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2099          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2100          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2101          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2102          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2103          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2104          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2105          Temp new_exponent = bld.tmp(v1);
2106          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2107          if (ctx->program->chip_class >= GFX8)
2108             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2109          else
2110             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2111          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2112          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2114          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2115          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2116          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2117          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2118
2119       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2120          if (src.type() == RegType::vgpr)
2121             src = bld.as_uniform(src);
2122          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2123          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2124          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2125          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2126          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2127          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2128          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2129          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2130          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2131          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2132          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2133          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2134          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2135          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2136          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2137          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2138          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2139          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2140
2141       } else if (instr->src[0].src.ssa->bit_size == 64) {
2142          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2143          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2144          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2145          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2146          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2147          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2148          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2149          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2150          if (dst.type() == RegType::sgpr) {
2151             lower = bld.as_uniform(lower);
2152             upper = bld.as_uniform(upper);
2153          }
2154          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2155
2156       } else {
2157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2158          nir_print_instr(&instr->instr, stderr);
2159          fprintf(stderr, "\n");
2160       }
2161       break;
2162    }
2163    case nir_op_b2f32: {
2164       Temp src = get_alu_src(ctx, instr->src[0]);
2165       assert(src.regClass() == bld.lm);
2166
2167       if (dst.regClass() == s1) {
2168          src = bool_to_scalar_condition(ctx, src);
2169          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2170       } else if (dst.regClass() == v1) {
2171          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2172       } else {
2173          unreachable("Wrong destination register class for nir_op_b2f32.");
2174       }
2175       break;
2176    }
2177    case nir_op_b2f64: {
2178       Temp src = get_alu_src(ctx, instr->src[0]);
2179       assert(src.regClass() == bld.lm);
2180
2181       if (dst.regClass() == s2) {
2182          src = bool_to_scalar_condition(ctx, src);
2183          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2184       } else if (dst.regClass() == v2) {
2185          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2186          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2187          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2188       } else {
2189          unreachable("Wrong destination register class for nir_op_b2f64.");
2190       }
2191       break;
2192    }
2193    case nir_op_i2i32: {
2194       Temp src = get_alu_src(ctx, instr->src[0]);
2195       if (instr->src[0].src.ssa->bit_size == 64) {
2196          /* we can actually just say dst = src, as it would map the lower register */
2197          emit_extract_vector(ctx, src, 0, dst);
2198       } else {
2199          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2200          nir_print_instr(&instr->instr, stderr);
2201          fprintf(stderr, "\n");
2202       }
2203       break;
2204    }
2205    case nir_op_u2u32: {
2206       Temp src = get_alu_src(ctx, instr->src[0]);
2207       if (instr->src[0].src.ssa->bit_size == 16) {
2208          if (dst.regClass() == s1) {
2209             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2210          } else {
2211             // TODO: do better with SDWA
2212             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2213          }
2214       } else if (instr->src[0].src.ssa->bit_size == 64) {
2215          /* we can actually just say dst = src, as it would map the lower register */
2216          emit_extract_vector(ctx, src, 0, dst);
2217       } else {
2218          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2219          nir_print_instr(&instr->instr, stderr);
2220          fprintf(stderr, "\n");
2221       }
2222       break;
2223    }
2224    case nir_op_i2i64: {
2225       Temp src = get_alu_src(ctx, instr->src[0]);
2226       if (src.regClass() == s1) {
2227          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2228          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2229       } else if (src.regClass() == v1) {
2230          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2231          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2232       } else {
2233          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2234          nir_print_instr(&instr->instr, stderr);
2235          fprintf(stderr, "\n");
2236       }
2237       break;
2238    }
2239    case nir_op_u2u64: {
2240       Temp src = get_alu_src(ctx, instr->src[0]);
2241       if (instr->src[0].src.ssa->bit_size == 32) {
2242          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2243       } else {
2244          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2245          nir_print_instr(&instr->instr, stderr);
2246          fprintf(stderr, "\n");
2247       }
2248       break;
2249    }
2250    case nir_op_b2i32: {
2251       Temp src = get_alu_src(ctx, instr->src[0]);
2252       assert(src.regClass() == bld.lm);
2253
2254       if (dst.regClass() == s1) {
2255          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2256          bool_to_scalar_condition(ctx, src, dst);
2257       } else if (dst.regClass() == v1) {
2258          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2259       } else {
2260          unreachable("Invalid register class for b2i32");
2261       }
2262       break;
2263    }
2264    case nir_op_i2b1: {
2265       Temp src = get_alu_src(ctx, instr->src[0]);
2266       assert(dst.regClass() == bld.lm);
2267
2268       if (src.type() == RegType::vgpr) {
2269          assert(src.regClass() == v1 || src.regClass() == v2);
2270          assert(dst.regClass() == bld.lm);
2271          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2272                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2273       } else {
2274          assert(src.regClass() == s1 || src.regClass() == s2);
2275          Temp tmp;
2276          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2277             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2278          } else {
2279             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2280                            bld.scc(bld.def(s1)), Operand(0u), src);
2281          }
2282          bool_to_vector_condition(ctx, tmp, dst);
2283       }
2284       break;
2285    }
2286    case nir_op_pack_64_2x32_split: {
2287       Temp src0 = get_alu_src(ctx, instr->src[0]);
2288       Temp src1 = get_alu_src(ctx, instr->src[1]);
2289
2290       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2291       break;
2292    }
2293    case nir_op_unpack_64_2x32_split_x:
2294       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2295       break;
2296    case nir_op_unpack_64_2x32_split_y:
2297       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2298       break;
2299    case nir_op_pack_half_2x16: {
2300       Temp src = get_alu_src(ctx, instr->src[0], 2);
2301
2302       if (dst.regClass() == v1) {
2303          Temp src0 = bld.tmp(v1);
2304          Temp src1 = bld.tmp(v1);
2305          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2306          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2307             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2308          else
2309             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2310                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2311                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2312       } else {
2313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2314          nir_print_instr(&instr->instr, stderr);
2315          fprintf(stderr, "\n");
2316       }
2317       break;
2318    }
2319    case nir_op_unpack_half_2x16_split_x: {
2320       if (dst.regClass() == v1) {
2321          Builder bld(ctx->program, ctx->block);
2322          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2323       } else {
2324          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2325          nir_print_instr(&instr->instr, stderr);
2326          fprintf(stderr, "\n");
2327       }
2328       break;
2329    }
2330    case nir_op_unpack_half_2x16_split_y: {
2331       if (dst.regClass() == v1) {
2332          Builder bld(ctx->program, ctx->block);
2333          /* TODO: use SDWA here */
2334          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2335                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2336       } else {
2337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2338          nir_print_instr(&instr->instr, stderr);
2339          fprintf(stderr, "\n");
2340       }
2341       break;
2342    }
2343    case nir_op_fquantize2f16: {
2344       Temp src = get_alu_src(ctx, instr->src[0]);
2345       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2346       Temp f32, cmp_res;
2347
2348       if (ctx->program->chip_class >= GFX8) {
2349          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2350          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2351          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2352       } else {
2353          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2354           * so compare the result and flush to 0 if it's smaller.
2355           */
2356          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2357          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2358          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2359          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2360          cmp_res = vop3->definitions[0].getTemp();
2361       }
2362
2363       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2364          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2366       } else {
2367          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2368       }
2369       break;
2370    }
2371    case nir_op_bfm: {
2372       Temp bits = get_alu_src(ctx, instr->src[0]);
2373       Temp offset = get_alu_src(ctx, instr->src[1]);
2374
2375       if (dst.regClass() == s1) {
2376          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2377       } else if (dst.regClass() == v1) {
2378          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2379       } else {
2380          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2381          nir_print_instr(&instr->instr, stderr);
2382          fprintf(stderr, "\n");
2383       }
2384       break;
2385    }
2386    case nir_op_bitfield_select: {
2387       /* (mask & insert) | (~mask & base) */
2388       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2389       Temp insert = get_alu_src(ctx, instr->src[1]);
2390       Temp base = get_alu_src(ctx, instr->src[2]);
2391
2392       /* dst = (insert & bitmask) | (base & ~bitmask) */
2393       if (dst.regClass() == s1) {
2394          aco_ptr<Instruction> sop2;
2395          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2396          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2397          Operand lhs;
2398          if (const_insert && const_bitmask) {
2399             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2400          } else {
2401             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2402             lhs = Operand(insert);
2403          }
2404
2405          Operand rhs;
2406          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2407          if (const_base && const_bitmask) {
2408             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2409          } else {
2410             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2411             rhs = Operand(base);
2412          }
2413
2414          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2415
2416       } else if (dst.regClass() == v1) {
2417          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2418             base = as_vgpr(ctx, base);
2419          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2420             insert = as_vgpr(ctx, insert);
2421
2422          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2423
2424       } else {
2425          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2426          nir_print_instr(&instr->instr, stderr);
2427          fprintf(stderr, "\n");
2428       }
2429       break;
2430    }
2431    case nir_op_ubfe:
2432    case nir_op_ibfe: {
2433       Temp base = get_alu_src(ctx, instr->src[0]);
2434       Temp offset = get_alu_src(ctx, instr->src[1]);
2435       Temp bits = get_alu_src(ctx, instr->src[2]);
2436
2437       if (dst.type() == RegType::sgpr) {
2438          Operand extract;
2439          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2440          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2441          if (const_offset && const_bits) {
2442             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2443             extract = Operand(const_extract);
2444          } else {
2445             Operand width;
2446             if (const_bits) {
2447                width = Operand(const_bits->u32 << 16);
2448             } else {
2449                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2450             }
2451             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2452          }
2453
2454          aco_opcode opcode;
2455          if (dst.regClass() == s1) {
2456             if (instr->op == nir_op_ubfe)
2457                opcode = aco_opcode::s_bfe_u32;
2458             else
2459                opcode = aco_opcode::s_bfe_i32;
2460          } else if (dst.regClass() == s2) {
2461             if (instr->op == nir_op_ubfe)
2462                opcode = aco_opcode::s_bfe_u64;
2463             else
2464                opcode = aco_opcode::s_bfe_i64;
2465          } else {
2466             unreachable("Unsupported BFE bit size");
2467          }
2468
2469          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2470
2471       } else {
2472          aco_opcode opcode;
2473          if (dst.regClass() == v1) {
2474             if (instr->op == nir_op_ubfe)
2475                opcode = aco_opcode::v_bfe_u32;
2476             else
2477                opcode = aco_opcode::v_bfe_i32;
2478          } else {
2479             unreachable("Unsupported BFE bit size");
2480          }
2481
2482          emit_vop3a_instruction(ctx, instr, opcode, dst);
2483       }
2484       break;
2485    }
2486    case nir_op_bit_count: {
2487       Temp src = get_alu_src(ctx, instr->src[0]);
2488       if (src.regClass() == s1) {
2489          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2490       } else if (src.regClass() == v1) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2492       } else if (src.regClass() == v2) {
2493          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2494                   emit_extract_vector(ctx, src, 1, v1),
2495                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2496                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2497       } else if (src.regClass() == s2) {
2498          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2499       } else {
2500          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2501          nir_print_instr(&instr->instr, stderr);
2502          fprintf(stderr, "\n");
2503       }
2504       break;
2505    }
2506    case nir_op_flt: {
2507       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2508       break;
2509    }
2510    case nir_op_fge: {
2511       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2512       break;
2513    }
2514    case nir_op_feq: {
2515       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2516       break;
2517    }
2518    case nir_op_fne: {
2519       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2520       break;
2521    }
2522    case nir_op_ilt: {
2523       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2524       break;
2525    }
2526    case nir_op_ige: {
2527       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2528       break;
2529    }
2530    case nir_op_ieq: {
2531       if (instr->src[0].src.ssa->bit_size == 1)
2532          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2533       else
2534          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2535                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2536       break;
2537    }
2538    case nir_op_ine: {
2539       if (instr->src[0].src.ssa->bit_size == 1)
2540          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2541       else
2542          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2543                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2544       break;
2545    }
2546    case nir_op_ult: {
2547       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2548       break;
2549    }
2550    case nir_op_uge: {
2551       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2552       break;
2553    }
2554    case nir_op_fddx:
2555    case nir_op_fddy:
2556    case nir_op_fddx_fine:
2557    case nir_op_fddy_fine:
2558    case nir_op_fddx_coarse:
2559    case nir_op_fddy_coarse: {
2560       Temp src = get_alu_src(ctx, instr->src[0]);
2561       uint16_t dpp_ctrl1, dpp_ctrl2;
2562       if (instr->op == nir_op_fddx_fine) {
2563          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2564          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2565       } else if (instr->op == nir_op_fddy_fine) {
2566          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2567          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2568       } else {
2569          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2570          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2571             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2572          else
2573             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2574       }
2575
2576       Temp tmp;
2577       if (ctx->program->chip_class >= GFX8) {
2578          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2579          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2580       } else {
2581          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2582          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2583          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2584       }
2585       emit_wqm(ctx, tmp, dst, true);
2586       break;
2587    }
2588    default:
2589       fprintf(stderr, "Unknown NIR ALU instr: ");
2590       nir_print_instr(&instr->instr, stderr);
2591       fprintf(stderr, "\n");
2592    }
2593 }
2594
2595 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2596 {
2597    Temp dst = get_ssa_temp(ctx, &instr->def);
2598
2599    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2600    // which get truncated the lsb if double and msb if int
2601    // for now, we only use s_mov_b64 with 64bit inline constants
2602    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2603    assert(dst.type() == RegType::sgpr);
2604
2605    Builder bld(ctx->program, ctx->block);
2606
2607    if (instr->def.bit_size == 1) {
2608       assert(dst.regClass() == bld.lm);
2609       int val = instr->value[0].b ? -1 : 0;
2610       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2611       bld.sop1(Builder::s_mov, Definition(dst), op);
2612    } else if (dst.size() == 1) {
2613       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2614    } else {
2615       assert(dst.size() != 1);
2616       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2617       if (instr->def.bit_size == 64)
2618          for (unsigned i = 0; i < dst.size(); i++)
2619             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2620       else {
2621          for (unsigned i = 0; i < dst.size(); i++)
2622             vec->operands[i] = Operand{instr->value[i].u32};
2623       }
2624       vec->definitions[0] = Definition(dst);
2625       ctx->block->instructions.emplace_back(std::move(vec));
2626    }
2627 }
2628
2629 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2630 {
2631    uint32_t new_mask = 0;
2632    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2633       if (mask & (1u << i))
2634          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2635    return new_mask;
2636 }
2637
2638 Operand load_lds_size_m0(isel_context *ctx)
2639 {
2640    /* TODO: m0 does not need to be initialized on GFX9+ */
2641    Builder bld(ctx->program, ctx->block);
2642    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2643 }
2644
2645 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2646               Temp address, unsigned base_offset, unsigned align)
2647 {
2648    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2649
2650    Builder bld(ctx->program, ctx->block);
2651
2652    Operand m = load_lds_size_m0(ctx);
2653
2654    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2655    unsigned bytes_read = 0;
2656    unsigned result_size = 0;
2657    unsigned total_bytes = num_components * elem_size_bytes;
2658    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2659    bool large_ds_read = ctx->options->chip_class >= GFX7;
2660    bool usable_read2 = ctx->options->chip_class >= GFX7;
2661
2662    while (bytes_read < total_bytes) {
2663       unsigned todo = total_bytes - bytes_read;
2664       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2665       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2666
2667       aco_opcode op = aco_opcode::last_opcode;
2668       bool read2 = false;
2669       if (todo >= 16 && aligned16 && large_ds_read) {
2670          op = aco_opcode::ds_read_b128;
2671          todo = 16;
2672       } else if (todo >= 16 && aligned8 && usable_read2) {
2673          op = aco_opcode::ds_read2_b64;
2674          read2 = true;
2675          todo = 16;
2676       } else if (todo >= 12 && aligned16 && large_ds_read) {
2677          op = aco_opcode::ds_read_b96;
2678          todo = 12;
2679       } else if (todo >= 8 && aligned8) {
2680          op = aco_opcode::ds_read_b64;
2681          todo = 8;
2682       } else if (todo >= 8 && usable_read2) {
2683          op = aco_opcode::ds_read2_b32;
2684          read2 = true;
2685          todo = 8;
2686       } else if (todo >= 4) {
2687          op = aco_opcode::ds_read_b32;
2688          todo = 4;
2689       } else {
2690          assert(false);
2691       }
2692       assert(todo % elem_size_bytes == 0);
2693       unsigned num_elements = todo / elem_size_bytes;
2694       unsigned offset = base_offset + bytes_read;
2695       unsigned max_offset = read2 ? 1019 : 65535;
2696
2697       Temp address_offset = address;
2698       if (offset > max_offset) {
2699          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2700          offset = bytes_read;
2701       }
2702       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2703
2704       Temp res;
2705       if (num_components == 1 && dst.type() == RegType::vgpr)
2706          res = dst;
2707       else
2708          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2709
2710       if (read2)
2711          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2712       else
2713          res = bld.ds(op, Definition(res), address_offset, m, offset);
2714
2715       if (num_components == 1) {
2716          assert(todo == total_bytes);
2717          if (dst.type() == RegType::sgpr)
2718             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2719          return;
2720       }
2721
2722       if (dst.type() == RegType::sgpr) {
2723          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2724          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2725          res = new_res;
2726       }
2727
2728       if (num_elements == 1) {
2729          result[result_size++] = res;
2730       } else {
2731          assert(res != dst && res.size() % num_elements == 0);
2732          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2733          split->operands[0] = Operand(res);
2734          for (unsigned i = 0; i < num_elements; i++)
2735             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2736          ctx->block->instructions.emplace_back(std::move(split));
2737       }
2738
2739       bytes_read += todo;
2740    }
2741
2742    assert(result_size == num_components && result_size > 1);
2743    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2744    for (unsigned i = 0; i < result_size; i++)
2745       vec->operands[i] = Operand(result[i]);
2746    vec->definitions[0] = Definition(dst);
2747    ctx->block->instructions.emplace_back(std::move(vec));
2748    ctx->allocated_vec.emplace(dst.id(), result);
2749 }
2750
2751 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2752 {
2753    if (start == 0 && size == data.size())
2754       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2755
2756    unsigned size_hint = 1;
2757    auto it = ctx->allocated_vec.find(data.id());
2758    if (it != ctx->allocated_vec.end())
2759       size_hint = it->second[0].size();
2760    if (size % size_hint || start % size_hint)
2761       size_hint = 1;
2762
2763    start /= size_hint;
2764    size /= size_hint;
2765
2766    Temp elems[size];
2767    for (unsigned i = 0; i < size; i++)
2768       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2769
2770    if (size == 1)
2771       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2772
2773    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2774    for (unsigned i = 0; i < size; i++)
2775       vec->operands[i] = Operand(elems[i]);
2776    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2777    vec->definitions[0] = Definition(res);
2778    ctx->block->instructions.emplace_back(std::move(vec));
2779    return res;
2780 }
2781
2782 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2783 {
2784    Builder bld(ctx->program, ctx->block);
2785    unsigned bytes_written = 0;
2786    bool large_ds_write = ctx->options->chip_class >= GFX7;
2787    bool usable_write2 = ctx->options->chip_class >= GFX7;
2788
2789    while (bytes_written < total_size * 4) {
2790       unsigned todo = total_size * 4 - bytes_written;
2791       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2792       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2793
2794       aco_opcode op = aco_opcode::last_opcode;
2795       bool write2 = false;
2796       unsigned size = 0;
2797       if (todo >= 16 && aligned16 && large_ds_write) {
2798          op = aco_opcode::ds_write_b128;
2799          size = 4;
2800       } else if (todo >= 16 && aligned8 && usable_write2) {
2801          op = aco_opcode::ds_write2_b64;
2802          write2 = true;
2803          size = 4;
2804       } else if (todo >= 12 && aligned16 && large_ds_write) {
2805          op = aco_opcode::ds_write_b96;
2806          size = 3;
2807       } else if (todo >= 8 && aligned8) {
2808          op = aco_opcode::ds_write_b64;
2809          size = 2;
2810       } else if (todo >= 8 && usable_write2) {
2811          op = aco_opcode::ds_write2_b32;
2812          write2 = true;
2813          size = 2;
2814       } else if (todo >= 4) {
2815          op = aco_opcode::ds_write_b32;
2816          size = 1;
2817       } else {
2818          assert(false);
2819       }
2820
2821       unsigned offset = offset0 + offset1 + bytes_written;
2822       unsigned max_offset = write2 ? 1020 : 65535;
2823       Temp address_offset = address;
2824       if (offset > max_offset) {
2825          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2826          offset = offset1 + bytes_written;
2827       }
2828       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2829
2830       if (write2) {
2831          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2832          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2833          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2834       } else {
2835          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2836          bld.ds(op, address_offset, val, m, offset);
2837       }
2838
2839       bytes_written += size * 4;
2840    }
2841 }
2842
2843 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2844                Temp address, unsigned base_offset, unsigned align)
2845 {
2846    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2847
2848    Operand m = load_lds_size_m0(ctx);
2849
2850    /* we need at most two stores for 32bit variables */
2851    int start[2], count[2];
2852    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2853    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2854    assert(wrmask == 0);
2855
2856    /* one combined store is sufficient */
2857    if (count[0] == count[1]) {
2858       Builder bld(ctx->program, ctx->block);
2859
2860       Temp address_offset = address;
2861       if ((base_offset >> 2) + start[1] > 255) {
2862          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2863          base_offset = 0;
2864       }
2865
2866       assert(count[0] == 1);
2867       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2868       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2869       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2870       base_offset = base_offset / elem_size_bytes;
2871       bld.ds(op, address_offset, val0, val1, m,
2872              base_offset + start[0], base_offset + start[1]);
2873       return;
2874    }
2875
2876    for (unsigned i = 0; i < 2; i++) {
2877       if (count[i] == 0)
2878          continue;
2879
2880       unsigned elem_size_words = elem_size_bytes / 4;
2881       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2882                       base_offset, start[i] * elem_size_bytes, align);
2883    }
2884    return;
2885 }
2886
2887 void visit_store_vsgs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2888 {
2889    unsigned write_mask = nir_intrinsic_write_mask(instr);
2890    unsigned component = nir_intrinsic_component(instr);
2891    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2892    unsigned idx = (nir_intrinsic_base(instr) + component) * 4u;
2893    Operand offset(s1);
2894    Builder bld(ctx->program, ctx->block);
2895
2896    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2897    if (off_instr->type != nir_instr_type_load_const)
2898       offset = bld.v_mul24_imm(bld.def(v1), get_ssa_temp(ctx, instr->src[1].ssa), 16u);
2899    else
2900       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 16u;
2901
2902    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
2903    if (ctx->stage == vertex_es) {
2904       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
2905
2906       Temp elems[NIR_MAX_VEC_COMPONENTS * 2];
2907       if (elem_size_bytes == 8) {
2908          for (unsigned i = 0; i < src.size() / 2; i++) {
2909             Temp elem = emit_extract_vector(ctx, src, i, v2);
2910             elems[i*2] = bld.tmp(v1);
2911             elems[i*2+1] = bld.tmp(v1);
2912             bld.pseudo(aco_opcode::p_split_vector, Definition(elems[i*2]), Definition(elems[i*2+1]), elem);
2913          }
2914          write_mask = widen_mask(write_mask, 2);
2915          elem_size_bytes /= 2u;
2916       } else {
2917          for (unsigned i = 0; i < src.size(); i++)
2918             elems[i] = emit_extract_vector(ctx, src, i, v1);
2919       }
2920
2921       while (write_mask) {
2922          unsigned index = u_bit_scan(&write_mask);
2923          unsigned offset = index * elem_size_bytes;
2924          Temp elem = emit_extract_vector(ctx, src, index, RegClass(RegType::vgpr, elem_size_bytes / 4));
2925
2926          Operand vaddr_offset(v1);
2927          unsigned const_offset = idx + offset;
2928          if (const_offset >= 4096u) {
2929             vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
2930             const_offset %= 4096u;
2931          }
2932
2933          aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
2934          mtbuf->operands[0] = Operand(esgs_ring);
2935          mtbuf->operands[1] = vaddr_offset;
2936          mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->es2gs_offset));
2937          mtbuf->operands[3] = Operand(elem);
2938          mtbuf->offen = !vaddr_offset.isUndefined();
2939          mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
2940          mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
2941          mtbuf->offset = const_offset;
2942          mtbuf->glc = true;
2943          mtbuf->slc = true;
2944          mtbuf->barrier = barrier_none;
2945          mtbuf->can_reorder = true;
2946          bld.insert(std::move(mtbuf));
2947       }
2948    } else {
2949       unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
2950
2951       Temp vertex_idx = emit_mbcnt(ctx, bld.def(v1));
2952       Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
2953       vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), vertex_idx,
2954                             bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
2955
2956       Temp lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
2957       if (!offset.isUndefined())
2958          lds_base = bld.vadd32(bld.def(v1), offset, lds_base);
2959
2960       unsigned align = 1 << (ffs(itemsize) - 1);
2961       if (idx)
2962          align = std::min(align, 1u << (ffs(idx) - 1));
2963
2964       store_lds(ctx, elem_size_bytes, src, write_mask, lds_base, idx, align);
2965    }
2966 }
2967
2968 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2969 {
2970    if (ctx->stage == vertex_vs ||
2971        ctx->stage == fragment_fs ||
2972        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
2973       unsigned write_mask = nir_intrinsic_write_mask(instr);
2974       unsigned component = nir_intrinsic_component(instr);
2975       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2976       unsigned idx = nir_intrinsic_base(instr) + component;
2977
2978       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2979       if (off_instr->type != nir_instr_type_load_const) {
2980          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2981          nir_print_instr(off_instr, stderr);
2982          fprintf(stderr, "\n");
2983       }
2984       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2985
2986       if (instr->src[0].ssa->bit_size == 64)
2987          write_mask = widen_mask(write_mask, 2);
2988
2989       for (unsigned i = 0; i < 8; ++i) {
2990          if (write_mask & (1 << i)) {
2991             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
2992             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2993          }
2994          idx++;
2995       }
2996    } else if (ctx->stage == vertex_es ||
2997               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
2998       visit_store_vsgs_output(ctx, instr);
2999    } else {
3000       unreachable("Shader stage not implemented");
3001    }
3002 }
3003
3004 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3005 {
3006    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3007    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3008
3009    Builder bld(ctx->program, ctx->block);
3010    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3011    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3012 }
3013
3014 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3015 {
3016    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3017    for (unsigned i = 0; i < num_components; i++)
3018       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3019    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3020       assert(num_components == 4);
3021       Builder bld(ctx->program, ctx->block);
3022       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3023    }
3024
3025    for (Operand& op : vec->operands)
3026       op = op.isUndefined() ? Operand(0u) : op;
3027
3028    vec->definitions[0] = Definition(dst);
3029    ctx->block->instructions.emplace_back(std::move(vec));
3030    emit_split_vector(ctx, dst, num_components);
3031    return;
3032 }
3033
3034 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3035 {
3036    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3037    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3038    unsigned idx = nir_intrinsic_base(instr);
3039    unsigned component = nir_intrinsic_component(instr);
3040    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3041
3042    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3043    if (offset) {
3044       assert(offset->u32 == 0);
3045    } else {
3046       /* the lower 15bit of the prim_mask contain the offset into LDS
3047        * while the upper bits contain the number of prims */
3048       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3049       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3050       Builder bld(ctx->program, ctx->block);
3051       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3052       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3053       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3054       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3055       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3056    }
3057
3058    if (instr->dest.ssa.num_components == 1) {
3059       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3060    } else {
3061       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3062       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3063       {
3064          Temp tmp = {ctx->program->allocateId(), v1};
3065          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3066          vec->operands[i] = Operand(tmp);
3067       }
3068       vec->definitions[0] = Definition(dst);
3069       ctx->block->instructions.emplace_back(std::move(vec));
3070    }
3071 }
3072
3073 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3074                              unsigned offset, unsigned stride, unsigned channels)
3075 {
3076    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3077    if (vtx_info->chan_byte_size != 4 && channels == 3)
3078       return false;
3079    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3080           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3081 }
3082
3083 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3084                               unsigned offset, unsigned stride, unsigned *channels)
3085 {
3086    if (!vtx_info->chan_byte_size) {
3087       *channels = vtx_info->num_channels;
3088       return vtx_info->chan_format;
3089    }
3090
3091    unsigned num_channels = *channels;
3092    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3093       unsigned new_channels = num_channels + 1;
3094       /* first, assume more loads is worse and try using a larger data format */
3095       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3096          new_channels++;
3097          /* don't make the attribute potentially out-of-bounds */
3098          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3099             new_channels = 5;
3100       }
3101
3102       if (new_channels == 5) {
3103          /* then try decreasing load size (at the cost of more loads) */
3104          new_channels = *channels;
3105          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3106             new_channels--;
3107       }
3108
3109       if (new_channels < *channels)
3110          *channels = new_channels;
3111       num_channels = new_channels;
3112    }
3113
3114    switch (vtx_info->chan_format) {
3115    case V_008F0C_BUF_DATA_FORMAT_8:
3116       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3117                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3118    case V_008F0C_BUF_DATA_FORMAT_16:
3119       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3120                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3121    case V_008F0C_BUF_DATA_FORMAT_32:
3122       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3123                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3124    }
3125    unreachable("shouldn't reach here");
3126    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3127 }
3128
3129 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3130  * so we may need to fix it up. */
3131 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3132 {
3133    Builder bld(ctx->program, ctx->block);
3134
3135    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3136       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3137
3138    /* For the integer-like cases, do a natural sign extension.
3139     *
3140     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3141     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3142     * exponent.
3143     */
3144    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3145    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3146
3147    /* Convert back to the right type. */
3148    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3149       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3150       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3151       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3152    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3153       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3154    }
3155
3156    return alpha;
3157 }
3158
3159 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3160 {
3161    Builder bld(ctx->program, ctx->block);
3162    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3163    if (ctx->stage & sw_vs) {
3164
3165       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3166       if (off_instr->type != nir_instr_type_load_const) {
3167          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3168          nir_print_instr(off_instr, stderr);
3169          fprintf(stderr, "\n");
3170       }
3171       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3172
3173       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3174
3175       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3176       unsigned component = nir_intrinsic_component(instr);
3177       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3178       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3179       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3180       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3181
3182       unsigned dfmt = attrib_format & 0xf;
3183       unsigned nfmt = (attrib_format >> 4) & 0x7;
3184       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3185
3186       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3187       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3188       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3189       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3190       if (post_shuffle)
3191          num_channels = MAX2(num_channels, 3);
3192
3193       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3194       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3195
3196       Temp index;
3197       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3198          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3199          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3200          if (divisor) {
3201             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3202             if (divisor != 1) {
3203                Temp divided = bld.tmp(v1);
3204                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3205                index = bld.vadd32(bld.def(v1), start_instance, divided);
3206             } else {
3207                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3208             }
3209          } else {
3210             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3211          }
3212       } else {
3213          index = bld.vadd32(bld.def(v1),
3214                             get_arg(ctx, ctx->args->ac.base_vertex),
3215                             get_arg(ctx, ctx->args->ac.vertex_id));
3216       }
3217
3218       Temp channels[num_channels];
3219       unsigned channel_start = 0;
3220       bool direct_fetch = false;
3221
3222       /* skip unused channels at the start */
3223       if (vtx_info->chan_byte_size && !post_shuffle) {
3224          channel_start = ffs(mask) - 1;
3225          for (unsigned i = 0; i < channel_start; i++)
3226             channels[i] = Temp(0, s1);
3227       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3228          num_channels = 3 - (ffs(mask) - 1);
3229       }
3230
3231       /* load channels */
3232       while (channel_start < num_channels) {
3233          unsigned fetch_size = num_channels - channel_start;
3234          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3235          bool expanded = false;
3236
3237          /* use MUBUF when possible to avoid possible alignment issues */
3238          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3239          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3240                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3241                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3242                           vtx_info->chan_byte_size == 4;
3243          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3244          if (!use_mubuf) {
3245             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3246          } else {
3247             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
3248                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
3249                fetch_size = 4;
3250                expanded = true;
3251             }
3252          }
3253
3254          Temp fetch_index = index;
3255          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3256             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3257             fetch_offset = fetch_offset % attrib_stride;
3258          }
3259
3260          Operand soffset(0u);
3261          if (fetch_offset >= 4096) {
3262             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3263             fetch_offset %= 4096;
3264          }
3265
3266          aco_opcode opcode;
3267          switch (fetch_size) {
3268          case 1:
3269             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3270             break;
3271          case 2:
3272             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3273             break;
3274          case 3:
3275             assert(ctx->options->chip_class >= GFX7 ||
3276                    (!use_mubuf && ctx->options->chip_class == GFX6));
3277             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3278             break;
3279          case 4:
3280             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3281             break;
3282          default:
3283             unreachable("Unimplemented load_input vector size");
3284          }
3285
3286          Temp fetch_dst;
3287          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3288              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
3289                            num_channels <= 3)) {
3290             direct_fetch = true;
3291             fetch_dst = dst;
3292          } else {
3293             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3294          }
3295
3296          if (use_mubuf) {
3297             Instruction *mubuf = bld.mubuf(opcode,
3298                                            Definition(fetch_dst), list, fetch_index, soffset,
3299                                            fetch_offset, false, true).instr;
3300             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3301          } else {
3302             Instruction *mtbuf = bld.mtbuf(opcode,
3303                                            Definition(fetch_dst), list, fetch_index, soffset,
3304                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3305             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3306          }
3307
3308          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3309
3310          if (fetch_size == 1) {
3311             channels[channel_start] = fetch_dst;
3312          } else {
3313             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3314                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3315          }
3316
3317          channel_start += fetch_size;
3318       }
3319
3320       if (!direct_fetch) {
3321          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3322                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3323
3324          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3325          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3326          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3327
3328          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3329          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3330          unsigned num_temp = 0;
3331          for (unsigned i = 0; i < dst.size(); i++) {
3332             unsigned idx = i + component;
3333             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3334                Temp channel = channels[swizzle[idx]];
3335                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3336                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3337                vec->operands[i] = Operand(channel);
3338
3339                num_temp++;
3340                elems[i] = channel;
3341             } else if (is_float && idx == 3) {
3342                vec->operands[i] = Operand(0x3f800000u);
3343             } else if (!is_float && idx == 3) {
3344                vec->operands[i] = Operand(1u);
3345             } else {
3346                vec->operands[i] = Operand(0u);
3347             }
3348          }
3349          vec->definitions[0] = Definition(dst);
3350          ctx->block->instructions.emplace_back(std::move(vec));
3351          emit_split_vector(ctx, dst, dst.size());
3352
3353          if (num_temp == dst.size())
3354             ctx->allocated_vec.emplace(dst.id(), elems);
3355       }
3356    } else if (ctx->stage == fragment_fs) {
3357       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
3358       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
3359       if (off_instr->type != nir_instr_type_load_const ||
3360           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3361          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3362          nir_print_instr(off_instr, stderr);
3363          fprintf(stderr, "\n");
3364       }
3365
3366       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3367       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
3368       if (offset) {
3369          assert(offset->u32 == 0);
3370       } else {
3371          /* the lower 15bit of the prim_mask contain the offset into LDS
3372           * while the upper bits contain the number of prims */
3373          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
3374          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3375          Builder bld(ctx->program, ctx->block);
3376          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3377          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3378          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3379          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3380          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3381       }
3382
3383       unsigned idx = nir_intrinsic_base(instr);
3384       unsigned component = nir_intrinsic_component(instr);
3385       unsigned vertex_id = 2; /* P0 */
3386
3387       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
3388          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
3389          switch (src0->u32) {
3390          case 0:
3391             vertex_id = 2; /* P0 */
3392             break;
3393          case 1:
3394             vertex_id = 0; /* P10 */
3395             break;
3396          case 2:
3397             vertex_id = 1; /* P20 */
3398             break;
3399          default:
3400             unreachable("invalid vertex index");
3401          }
3402       }
3403
3404       if (dst.size() == 1) {
3405          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
3406       } else {
3407          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3408          for (unsigned i = 0; i < dst.size(); i++)
3409             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
3410          vec->definitions[0] = Definition(dst);
3411          bld.insert(std::move(vec));
3412       }
3413
3414    } else {
3415       unreachable("Shader stage not implemented");
3416    }
3417 }
3418
3419 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3420 {
3421    assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
3422    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3423
3424    Builder bld(ctx->program, ctx->block);
3425    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3426
3427    Temp offset = Temp();
3428    if (instr->src[0].ssa->parent_instr->type != nir_instr_type_load_const) {
3429       /* better code could be created, but this case probably doesn't happen
3430        * much in practice */
3431       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
3432       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3433          Temp elem;
3434          if (ctx->stage == vertex_geometry_gs) {
3435             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3436             if (i % 2u)
3437                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3438          } else {
3439             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3440          }
3441          if (offset.id()) {
3442             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(s2)),
3443                                  Operand(i), indirect_vertex);
3444             offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), offset, elem, cond);
3445          } else {
3446             offset = elem;
3447          }
3448       }
3449       if (ctx->stage == vertex_geometry_gs)
3450          offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), offset);
3451    } else {
3452       unsigned vertex = nir_src_as_uint(instr->src[0]);
3453       if (ctx->stage == vertex_geometry_gs)
3454          offset = bld.vop3(
3455          aco_opcode::v_bfe_u32, bld.def(v1), get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3456             Operand((vertex % 2u) * 16u), Operand(16u));
3457       else
3458          offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3459    }
3460
3461    unsigned const_offset = nir_intrinsic_base(instr);
3462    const_offset += nir_intrinsic_component(instr);
3463
3464    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3465    if (off_instr->type != nir_instr_type_load_const) {
3466       Temp indirect_offset = get_ssa_temp(ctx, instr->src[1].ssa);
3467       offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u),
3468                         bld.vadd32(bld.def(v1), indirect_offset, offset));
3469    } else {
3470       const_offset += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3471    }
3472    const_offset *= 4u;
3473
3474    offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), offset);
3475
3476    unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3477
3478    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3479    if (ctx->stage == geometry_gs) {
3480       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3481
3482       const_offset *= ctx->program->wave_size;
3483
3484       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3485       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3486          aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1)};
3487       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++) {
3488          Temp subelems[2];
3489          for (unsigned j = 0; j < elem_size_bytes / 4; j++) {
3490             Operand soffset(0u);
3491             if (const_offset >= 4096u)
3492                soffset = bld.copy(bld.def(s1), Operand(const_offset / 4096u * 4096u));
3493
3494             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
3495             mubuf->definitions[0] = bld.def(v1);
3496             subelems[j] = mubuf->definitions[0].getTemp();
3497             mubuf->operands[0] = Operand(esgs_ring);
3498             mubuf->operands[1] = Operand(offset);
3499             mubuf->operands[2] = Operand(soffset);
3500             mubuf->offen = true;
3501             mubuf->offset = const_offset % 4096u;
3502             mubuf->glc = true;
3503             mubuf->dlc = ctx->options->chip_class >= GFX10;
3504             mubuf->barrier = barrier_none;
3505             mubuf->can_reorder = true;
3506             bld.insert(std::move(mubuf));
3507
3508             const_offset += ctx->program->wave_size * 4u;
3509          }
3510
3511          if (elem_size_bytes == 4)
3512             elems[i] = subelems[0];
3513          else
3514             elems[i] = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), subelems[0], subelems[1]);
3515          vec->operands[i] = Operand(elems[i]);
3516       }
3517       vec->definitions[0] = Definition(dst);
3518       ctx->block->instructions.emplace_back(std::move(vec));
3519       ctx->allocated_vec.emplace(dst.id(), elems);
3520    } else {
3521       unsigned align = 16; /* alignment of indirect offset */
3522       align = std::min(align, 1u << (ffs(itemsize) - 1));
3523       if (const_offset)
3524          align = std::min(align, 1u << (ffs(const_offset) - 1));
3525
3526       load_lds(ctx, elem_size_bytes, dst, offset, const_offset, align);
3527    }
3528 }
3529
3530 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3531 {
3532    if (ctx->program->info->need_indirect_descriptor_sets) {
3533       Builder bld(ctx->program, ctx->block);
3534       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3535       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3536       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3537    }
3538
3539    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3540 }
3541
3542
3543 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3544 {
3545    Builder bld(ctx->program, ctx->block);
3546    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3547    if (!ctx->divergent_vals[instr->dest.ssa.index])
3548       index = bld.as_uniform(index);
3549    unsigned desc_set = nir_intrinsic_desc_set(instr);
3550    unsigned binding = nir_intrinsic_binding(instr);
3551
3552    Temp desc_ptr;
3553    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3554    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3555    unsigned offset = layout->binding[binding].offset;
3556    unsigned stride;
3557    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3558        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3559       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3560       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3561       offset = pipeline_layout->push_constant_size + 16 * idx;
3562       stride = 16;
3563    } else {
3564       desc_ptr = load_desc_ptr(ctx, desc_set);
3565       stride = layout->binding[binding].size;
3566    }
3567
3568    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3569    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3570    if (stride != 1) {
3571       if (nir_const_index) {
3572          const_index = const_index * stride;
3573       } else if (index.type() == RegType::vgpr) {
3574          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3575          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3576       } else {
3577          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3578       }
3579    }
3580    if (offset) {
3581       if (nir_const_index) {
3582          const_index = const_index + offset;
3583       } else if (index.type() == RegType::vgpr) {
3584          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3585       } else {
3586          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3587       }
3588    }
3589
3590    if (nir_const_index && const_index == 0) {
3591       index = desc_ptr;
3592    } else if (index.type() == RegType::vgpr) {
3593       index = bld.vadd32(bld.def(v1),
3594                          nir_const_index ? Operand(const_index) : Operand(index),
3595                          Operand(desc_ptr));
3596    } else {
3597       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3598                        nir_const_index ? Operand(const_index) : Operand(index),
3599                        Operand(desc_ptr));
3600    }
3601
3602    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3603 }
3604
3605 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3606                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3607 {
3608    Builder bld(ctx->program, ctx->block);
3609
3610    unsigned num_bytes = dst.size() * 4;
3611    bool dlc = glc && ctx->options->chip_class >= GFX10;
3612
3613    aco_opcode op;
3614    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3615       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3616       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3617       unsigned const_offset = 0;
3618
3619       Temp lower = Temp();
3620       if (num_bytes > 16) {
3621          assert(num_components == 3 || num_components == 4);
3622          op = aco_opcode::buffer_load_dwordx4;
3623          lower = bld.tmp(v4);
3624          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3625          mubuf->definitions[0] = Definition(lower);
3626          mubuf->operands[0] = Operand(rsrc);
3627          mubuf->operands[1] = vaddr;
3628          mubuf->operands[2] = soffset;
3629          mubuf->offen = (offset.type() == RegType::vgpr);
3630          mubuf->glc = glc;
3631          mubuf->dlc = dlc;
3632          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3633          mubuf->can_reorder = readonly;
3634          bld.insert(std::move(mubuf));
3635          emit_split_vector(ctx, lower, 2);
3636          num_bytes -= 16;
3637          const_offset = 16;
3638       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3639          /* GFX6 doesn't support loading vec3, expand to vec4. */
3640          num_bytes = 16;
3641       }
3642
3643       switch (num_bytes) {
3644          case 4:
3645             op = aco_opcode::buffer_load_dword;
3646             break;
3647          case 8:
3648             op = aco_opcode::buffer_load_dwordx2;
3649             break;
3650          case 12:
3651             assert(ctx->options->chip_class > GFX6);
3652             op = aco_opcode::buffer_load_dwordx3;
3653             break;
3654          case 16:
3655             op = aco_opcode::buffer_load_dwordx4;
3656             break;
3657          default:
3658             unreachable("Load SSBO not implemented for this size.");
3659       }
3660       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3661       mubuf->operands[0] = Operand(rsrc);
3662       mubuf->operands[1] = vaddr;
3663       mubuf->operands[2] = soffset;
3664       mubuf->offen = (offset.type() == RegType::vgpr);
3665       mubuf->glc = glc;
3666       mubuf->dlc = dlc;
3667       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3668       mubuf->can_reorder = readonly;
3669       mubuf->offset = const_offset;
3670       aco_ptr<Instruction> instr = std::move(mubuf);
3671
3672       if (dst.size() > 4) {
3673          assert(lower != Temp());
3674          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3675          instr->definitions[0] = Definition(upper);
3676          bld.insert(std::move(instr));
3677          if (dst.size() == 8)
3678             emit_split_vector(ctx, upper, 2);
3679          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3680          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3681          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3682          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3683          if (dst.size() == 8)
3684             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3685       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3686          Temp vec = bld.tmp(v4);
3687          instr->definitions[0] = Definition(vec);
3688          bld.insert(std::move(instr));
3689          emit_split_vector(ctx, vec, 4);
3690
3691          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3692          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3693          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3694          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3695       }
3696
3697       if (dst.type() == RegType::sgpr) {
3698          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3699          instr->definitions[0] = Definition(vec);
3700          bld.insert(std::move(instr));
3701          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3702       } else {
3703          instr->definitions[0] = Definition(dst);
3704          bld.insert(std::move(instr));
3705          emit_split_vector(ctx, dst, num_components);
3706       }
3707    } else {
3708       switch (num_bytes) {
3709          case 4:
3710             op = aco_opcode::s_buffer_load_dword;
3711             break;
3712          case 8:
3713             op = aco_opcode::s_buffer_load_dwordx2;
3714             break;
3715          case 12:
3716          case 16:
3717             op = aco_opcode::s_buffer_load_dwordx4;
3718             break;
3719          case 24:
3720          case 32:
3721             op = aco_opcode::s_buffer_load_dwordx8;
3722             break;
3723          default:
3724             unreachable("Load SSBO not implemented for this size.");
3725       }
3726       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3727       load->operands[0] = Operand(rsrc);
3728       load->operands[1] = Operand(bld.as_uniform(offset));
3729       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3730       load->definitions[0] = Definition(dst);
3731       load->glc = glc;
3732       load->dlc = dlc;
3733       load->barrier = readonly ? barrier_none : barrier_buffer;
3734       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3735       assert(ctx->options->chip_class >= GFX8 || !glc);
3736
3737       /* trim vector */
3738       if (dst.size() == 3) {
3739          Temp vec = bld.tmp(s4);
3740          load->definitions[0] = Definition(vec);
3741          bld.insert(std::move(load));
3742          emit_split_vector(ctx, vec, 4);
3743
3744          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3745                     emit_extract_vector(ctx, vec, 0, s1),
3746                     emit_extract_vector(ctx, vec, 1, s1),
3747                     emit_extract_vector(ctx, vec, 2, s1));
3748       } else if (dst.size() == 6) {
3749          Temp vec = bld.tmp(s8);
3750          load->definitions[0] = Definition(vec);
3751          bld.insert(std::move(load));
3752          emit_split_vector(ctx, vec, 4);
3753
3754          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3755                     emit_extract_vector(ctx, vec, 0, s2),
3756                     emit_extract_vector(ctx, vec, 1, s2),
3757                     emit_extract_vector(ctx, vec, 2, s2));
3758       } else {
3759          bld.insert(std::move(load));
3760       }
3761       emit_split_vector(ctx, dst, num_components);
3762    }
3763 }
3764
3765 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3766 {
3767    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3768    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3769
3770    Builder bld(ctx->program, ctx->block);
3771
3772    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3773    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3774    unsigned binding = nir_intrinsic_binding(idx_instr);
3775    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3776
3777    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3778       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3779                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3780                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3781                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3782       if (ctx->options->chip_class >= GFX10) {
3783          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3784                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3785                       S_008F0C_RESOURCE_LEVEL(1);
3786       } else {
3787          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3788                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3789       }
3790       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3791                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3792                                      Operand(0xFFFFFFFFu),
3793                                      Operand(desc_type));
3794       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3795                         rsrc, upper_dwords);
3796    } else {
3797       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3798       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3799    }
3800
3801    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3802 }
3803
3804 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3805 {
3806    Builder bld(ctx->program, ctx->block);
3807    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3808
3809    unsigned offset = nir_intrinsic_base(instr);
3810    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3811    if (index_cv && instr->dest.ssa.bit_size == 32) {
3812
3813       unsigned count = instr->dest.ssa.num_components;
3814       unsigned start = (offset + index_cv->u32) / 4u;
3815       start -= ctx->args->ac.base_inline_push_consts;
3816       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3817          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3818          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3819          for (unsigned i = 0; i < count; ++i) {
3820             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3821             vec->operands[i] = Operand{elems[i]};
3822          }
3823          vec->definitions[0] = Definition(dst);
3824          ctx->block->instructions.emplace_back(std::move(vec));
3825          ctx->allocated_vec.emplace(dst.id(), elems);
3826          return;
3827       }
3828    }
3829
3830    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3831    if (offset != 0) // TODO check if index != 0 as well
3832       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3833    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3834    Temp vec = dst;
3835    bool trim = false;
3836    aco_opcode op;
3837
3838    switch (dst.size()) {
3839    case 1:
3840       op = aco_opcode::s_load_dword;
3841       break;
3842    case 2:
3843       op = aco_opcode::s_load_dwordx2;
3844       break;
3845    case 3:
3846       vec = bld.tmp(s4);
3847       trim = true;
3848    case 4:
3849       op = aco_opcode::s_load_dwordx4;
3850       break;
3851    case 6:
3852       vec = bld.tmp(s8);
3853       trim = true;
3854    case 8:
3855       op = aco_opcode::s_load_dwordx8;
3856       break;
3857    default:
3858       unreachable("unimplemented or forbidden load_push_constant.");
3859    }
3860
3861    bld.smem(op, Definition(vec), ptr, index);
3862
3863    if (trim) {
3864       emit_split_vector(ctx, vec, 4);
3865       RegClass rc = dst.size() == 3 ? s1 : s2;
3866       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3867                  emit_extract_vector(ctx, vec, 0, rc),
3868                  emit_extract_vector(ctx, vec, 1, rc),
3869                  emit_extract_vector(ctx, vec, 2, rc));
3870
3871    }
3872    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3873 }
3874
3875 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3876 {
3877    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3878
3879    Builder bld(ctx->program, ctx->block);
3880
3881    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3882                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3883                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3884                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3885    if (ctx->options->chip_class >= GFX10) {
3886       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3887                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3888                    S_008F0C_RESOURCE_LEVEL(1);
3889    } else {
3890       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3891                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3892    }
3893
3894    unsigned base = nir_intrinsic_base(instr);
3895    unsigned range = nir_intrinsic_range(instr);
3896
3897    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3898    if (base && offset.type() == RegType::sgpr)
3899       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3900    else if (base && offset.type() == RegType::vgpr)
3901       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3902
3903    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3904                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3905                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3906                           Operand(desc_type));
3907
3908    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3909 }
3910
3911 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3912 {
3913    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3914       ctx->cf_info.exec_potentially_empty_discard = true;
3915
3916    ctx->program->needs_exact = true;
3917
3918    // TODO: optimize uniform conditions
3919    Builder bld(ctx->program, ctx->block);
3920    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3921    assert(src.regClass() == bld.lm);
3922    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3923    bld.pseudo(aco_opcode::p_discard_if, src);
3924    ctx->block->kind |= block_kind_uses_discard_if;
3925    return;
3926 }
3927
3928 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3929 {
3930    Builder bld(ctx->program, ctx->block);
3931
3932    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3933       ctx->cf_info.exec_potentially_empty_discard = true;
3934
3935    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3936                     ctx->cf_info.parent_loop.has_divergent_continue;
3937
3938    if (ctx->block->loop_nest_depth &&
3939        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3940       /* we handle discards the same way as jump instructions */
3941       append_logical_end(ctx->block);
3942
3943       /* in loops, discard behaves like break */
3944       Block *linear_target = ctx->cf_info.parent_loop.exit;
3945       ctx->block->kind |= block_kind_discard;
3946
3947       if (!divergent) {
3948          /* uniform discard - loop ends here */
3949          assert(nir_instr_is_last(&instr->instr));
3950          ctx->block->kind |= block_kind_uniform;
3951          ctx->cf_info.has_branch = true;
3952          bld.branch(aco_opcode::p_branch);
3953          add_linear_edge(ctx->block->index, linear_target);
3954          return;
3955       }
3956
3957       /* we add a break right behind the discard() instructions */
3958       ctx->block->kind |= block_kind_break;
3959       unsigned idx = ctx->block->index;
3960
3961       /* remove critical edges from linear CFG */
3962       bld.branch(aco_opcode::p_branch);
3963       Block* break_block = ctx->program->create_and_insert_block();
3964       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3965       break_block->kind |= block_kind_uniform;
3966       add_linear_edge(idx, break_block);
3967       add_linear_edge(break_block->index, linear_target);
3968       bld.reset(break_block);
3969       bld.branch(aco_opcode::p_branch);
3970
3971       Block* continue_block = ctx->program->create_and_insert_block();
3972       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3973       add_linear_edge(idx, continue_block);
3974       append_logical_start(continue_block);
3975       ctx->block = continue_block;
3976
3977       return;
3978    }
3979
3980    /* it can currently happen that NIR doesn't remove the unreachable code */
3981    if (!nir_instr_is_last(&instr->instr)) {
3982       ctx->program->needs_exact = true;
3983       /* save exec somewhere temporarily so that it doesn't get
3984        * overwritten before the discard from outer exec masks */
3985       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3986       bld.pseudo(aco_opcode::p_discard_if, cond);
3987       ctx->block->kind |= block_kind_uses_discard_if;
3988       return;
3989    }
3990
3991    /* This condition is incorrect for uniformly branched discards in a loop
3992     * predicated by a divergent condition, but the above code catches that case
3993     * and the discard would end up turning into a discard_if.
3994     * For example:
3995     * if (divergent) {
3996     *    while (...) {
3997     *       if (uniform) {
3998     *          discard;
3999     *       }
4000     *    }
4001     * }
4002     */
4003    if (!ctx->cf_info.parent_if.is_divergent) {
4004       /* program just ends here */
4005       ctx->block->kind |= block_kind_uniform;
4006       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
4007               0 /* enabled mask */, 9 /* dest */,
4008               false /* compressed */, true/* done */, true /* valid mask */);
4009       bld.sopp(aco_opcode::s_endpgm);
4010       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
4011    } else {
4012       ctx->block->kind |= block_kind_discard;
4013       /* branch and linear edge is added by visit_if() */
4014    }
4015 }
4016
4017 enum aco_descriptor_type {
4018    ACO_DESC_IMAGE,
4019    ACO_DESC_FMASK,
4020    ACO_DESC_SAMPLER,
4021    ACO_DESC_BUFFER,
4022    ACO_DESC_PLANE_0,
4023    ACO_DESC_PLANE_1,
4024    ACO_DESC_PLANE_2,
4025 };
4026
4027 static bool
4028 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
4029    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
4030       return false;
4031    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
4032    return dim == ac_image_cube ||
4033           dim == ac_image_1darray ||
4034           dim == ac_image_2darray ||
4035           dim == ac_image_2darraymsaa;
4036 }
4037
4038 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4039                       enum aco_descriptor_type desc_type,
4040                       const nir_tex_instr *tex_instr, bool image, bool write)
4041 {
4042 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4043    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4044    if (it != ctx->tex_desc.end())
4045       return it->second;
4046 */
4047    Temp index = Temp();
4048    bool index_set = false;
4049    unsigned constant_index = 0;
4050    unsigned descriptor_set;
4051    unsigned base_index;
4052    Builder bld(ctx->program, ctx->block);
4053
4054    if (!deref_instr) {
4055       assert(tex_instr && !image);
4056       descriptor_set = 0;
4057       base_index = tex_instr->sampler_index;
4058    } else {
4059       while(deref_instr->deref_type != nir_deref_type_var) {
4060          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4061          if (!array_size)
4062             array_size = 1;
4063
4064          assert(deref_instr->deref_type == nir_deref_type_array);
4065          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4066          if (const_value) {
4067             constant_index += array_size * const_value->u32;
4068          } else {
4069             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4070             if (indirect.type() == RegType::vgpr)
4071                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4072
4073             if (array_size != 1)
4074                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4075
4076             if (!index_set) {
4077                index = indirect;
4078                index_set = true;
4079             } else {
4080                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4081             }
4082          }
4083
4084          deref_instr = nir_src_as_deref(deref_instr->parent);
4085       }
4086       descriptor_set = deref_instr->var->data.descriptor_set;
4087       base_index = deref_instr->var->data.binding;
4088    }
4089
4090    Temp list = load_desc_ptr(ctx, descriptor_set);
4091    list = convert_pointer_to_64_bit(ctx, list);
4092
4093    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4094    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4095    unsigned offset = binding->offset;
4096    unsigned stride = binding->size;
4097    aco_opcode opcode;
4098    RegClass type;
4099
4100    assert(base_index < layout->binding_count);
4101
4102    switch (desc_type) {
4103    case ACO_DESC_IMAGE:
4104       type = s8;
4105       opcode = aco_opcode::s_load_dwordx8;
4106       break;
4107    case ACO_DESC_FMASK:
4108       type = s8;
4109       opcode = aco_opcode::s_load_dwordx8;
4110       offset += 32;
4111       break;
4112    case ACO_DESC_SAMPLER:
4113       type = s4;
4114       opcode = aco_opcode::s_load_dwordx4;
4115       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4116          offset += radv_combined_image_descriptor_sampler_offset(binding);
4117       break;
4118    case ACO_DESC_BUFFER:
4119       type = s4;
4120       opcode = aco_opcode::s_load_dwordx4;
4121       break;
4122    case ACO_DESC_PLANE_0:
4123    case ACO_DESC_PLANE_1:
4124       type = s8;
4125       opcode = aco_opcode::s_load_dwordx8;
4126       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4127       break;
4128    case ACO_DESC_PLANE_2:
4129       type = s4;
4130       opcode = aco_opcode::s_load_dwordx4;
4131       offset += 64;
4132       break;
4133    default:
4134       unreachable("invalid desc_type\n");
4135    }
4136
4137    offset += constant_index * stride;
4138
4139    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4140       (!index_set || binding->immutable_samplers_equal)) {
4141       if (binding->immutable_samplers_equal)
4142          constant_index = 0;
4143
4144       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4145       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4146                         Operand(samplers[constant_index * 4 + 0]),
4147                         Operand(samplers[constant_index * 4 + 1]),
4148                         Operand(samplers[constant_index * 4 + 2]),
4149                         Operand(samplers[constant_index * 4 + 3]));
4150    }
4151
4152    Operand off;
4153    if (!index_set) {
4154       off = bld.copy(bld.def(s1), Operand(offset));
4155    } else {
4156       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4157                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4158    }
4159
4160    Temp res = bld.smem(opcode, bld.def(type), list, off);
4161
4162    if (desc_type == ACO_DESC_PLANE_2) {
4163       Temp components[8];
4164       for (unsigned i = 0; i < 8; i++)
4165          components[i] = bld.tmp(s1);
4166       bld.pseudo(aco_opcode::p_split_vector,
4167                  Definition(components[0]),
4168                  Definition(components[1]),
4169                  Definition(components[2]),
4170                  Definition(components[3]),
4171                  res);
4172
4173       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4174       bld.pseudo(aco_opcode::p_split_vector,
4175                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4176                  Definition(components[4]),
4177                  Definition(components[5]),
4178                  Definition(components[6]),
4179                  Definition(components[7]),
4180                  desc2);
4181
4182       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4183                        components[0], components[1], components[2], components[3],
4184                        components[4], components[5], components[6], components[7]);
4185    }
4186
4187    return res;
4188 }
4189
4190 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4191 {
4192    switch (dim) {
4193    case GLSL_SAMPLER_DIM_BUF:
4194       return 1;
4195    case GLSL_SAMPLER_DIM_1D:
4196       return array ? 2 : 1;
4197    case GLSL_SAMPLER_DIM_2D:
4198       return array ? 3 : 2;
4199    case GLSL_SAMPLER_DIM_MS:
4200       return array ? 4 : 3;
4201    case GLSL_SAMPLER_DIM_3D:
4202    case GLSL_SAMPLER_DIM_CUBE:
4203       return 3;
4204    case GLSL_SAMPLER_DIM_RECT:
4205    case GLSL_SAMPLER_DIM_SUBPASS:
4206       return 2;
4207    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4208       return 3;
4209    default:
4210       break;
4211    }
4212    return 0;
4213 }
4214
4215
4216 /* Adjust the sample index according to FMASK.
4217  *
4218  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4219  * which is the identity mapping. Each nibble says which physical sample
4220  * should be fetched to get that sample.
4221  *
4222  * For example, 0x11111100 means there are only 2 samples stored and
4223  * the second sample covers 3/4 of the pixel. When reading samples 0
4224  * and 1, return physical sample 0 (determined by the first two 0s
4225  * in FMASK), otherwise return physical sample 1.
4226  *
4227  * The sample index should be adjusted as follows:
4228  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4229  */
4230 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
4231 {
4232    Builder bld(ctx->program, ctx->block);
4233    Temp fmask = bld.tmp(v1);
4234    unsigned dim = ctx->options->chip_class >= GFX10
4235                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4236                   : 0;
4237
4238    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
4239                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
4240    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
4241    load->operands[0] = Operand(fmask_desc_ptr);
4242    load->operands[1] = Operand(s4); /* no sampler */
4243    load->operands[2] = Operand(coord);
4244    load->definitions[0] = Definition(fmask);
4245    load->glc = false;
4246    load->dlc = false;
4247    load->dmask = 0x1;
4248    load->unrm = true;
4249    load->da = da;
4250    load->dim = dim;
4251    load->can_reorder = true; /* fmask images shouldn't be modified */
4252    ctx->block->instructions.emplace_back(std::move(load));
4253
4254    Operand sample_index4;
4255    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4256       sample_index4 = Operand(sample_index.constantValue() << 2);
4257    } else if (sample_index.regClass() == s1) {
4258       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4259    } else {
4260       assert(sample_index.regClass() == v1);
4261       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4262    }
4263
4264    Temp final_sample;
4265    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4266       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4267    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4268       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4269    else
4270       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4271
4272    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4273     * resource descriptor is 0 (invalid),
4274     */
4275    Temp compare = bld.tmp(bld.lm);
4276    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4277                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4278
4279    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4280
4281    /* Replace the MSAA sample index. */
4282    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4283 }
4284
4285 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4286 {
4287
4288    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4289    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4290    bool is_array = glsl_sampler_type_is_array(type);
4291    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4292    assert(!add_frag_pos && "Input attachments should be lowered.");
4293    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4294    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4295    int count = image_type_to_components_count(dim, is_array);
4296    std::vector<Temp> coords(count);
4297    Builder bld(ctx->program, ctx->block);
4298
4299    if (is_ms) {
4300       count--;
4301       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
4302       /* get sample index */
4303       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4304          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4305          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
4306          std::vector<Temp> fmask_load_address;
4307          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
4308             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
4309
4310          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4311          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
4312       } else {
4313          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
4314       }
4315    }
4316
4317    if (count == 1 && !gfx9_1d)
4318       return emit_extract_vector(ctx, src0, 0, v1);
4319
4320    if (gfx9_1d) {
4321       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
4322       coords.resize(coords.size() + 1);
4323       coords[1] = bld.copy(bld.def(v1), Operand(0u));
4324       if (is_array)
4325          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
4326    } else {
4327       for (int i = 0; i < count; i++)
4328          coords[i] = emit_extract_vector(ctx, src0, i, v1);
4329    }
4330
4331    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4332        instr->intrinsic == nir_intrinsic_image_deref_store) {
4333       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4334       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4335
4336       if (!level_zero)
4337          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
4338    }
4339
4340    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4341    for (unsigned i = 0; i < coords.size(); i++)
4342       vec->operands[i] = Operand(coords[i]);
4343    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4344    vec->definitions[0] = Definition(res);
4345    ctx->block->instructions.emplace_back(std::move(vec));
4346    return res;
4347 }
4348
4349
4350 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4351 {
4352    Builder bld(ctx->program, ctx->block);
4353    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4354    const struct glsl_type *type = glsl_without_array(var->type);
4355    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4356    bool is_array = glsl_sampler_type_is_array(type);
4357    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4358
4359    if (dim == GLSL_SAMPLER_DIM_BUF) {
4360       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4361       unsigned num_channels = util_last_bit(mask);
4362       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4363       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4364
4365       aco_opcode opcode;
4366       switch (num_channels) {
4367       case 1:
4368          opcode = aco_opcode::buffer_load_format_x;
4369          break;
4370       case 2:
4371          opcode = aco_opcode::buffer_load_format_xy;
4372          break;
4373       case 3:
4374          opcode = aco_opcode::buffer_load_format_xyz;
4375          break;
4376       case 4:
4377          opcode = aco_opcode::buffer_load_format_xyzw;
4378          break;
4379       default:
4380          unreachable(">4 channel buffer image load");
4381       }
4382       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4383       load->operands[0] = Operand(rsrc);
4384       load->operands[1] = Operand(vindex);
4385       load->operands[2] = Operand((uint32_t) 0);
4386       Temp tmp;
4387       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4388          tmp = dst;
4389       else
4390          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4391       load->definitions[0] = Definition(tmp);
4392       load->idxen = true;
4393       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4394       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4395       load->barrier = barrier_image;
4396       ctx->block->instructions.emplace_back(std::move(load));
4397
4398       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4399       return;
4400    }
4401
4402    Temp coords = get_image_coords(ctx, instr, type);
4403    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4404
4405    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4406    unsigned num_components = util_bitcount(dmask);
4407    Temp tmp;
4408    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4409       tmp = dst;
4410    else
4411       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4412
4413    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4414    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4415
4416    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
4417    load->operands[0] = Operand(resource);
4418    load->operands[1] = Operand(s4); /* no sampler */
4419    load->operands[2] = Operand(coords);
4420    load->definitions[0] = Definition(tmp);
4421    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4422    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4423    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4424    load->dmask = dmask;
4425    load->unrm = true;
4426    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4427    load->barrier = barrier_image;
4428    ctx->block->instructions.emplace_back(std::move(load));
4429
4430    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4431    return;
4432 }
4433
4434 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4435 {
4436    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4437    const struct glsl_type *type = glsl_without_array(var->type);
4438    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4439    bool is_array = glsl_sampler_type_is_array(type);
4440    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4441
4442    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4443
4444    if (dim == GLSL_SAMPLER_DIM_BUF) {
4445       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4446       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4447       aco_opcode opcode;
4448       switch (data.size()) {
4449       case 1:
4450          opcode = aco_opcode::buffer_store_format_x;
4451          break;
4452       case 2:
4453          opcode = aco_opcode::buffer_store_format_xy;
4454          break;
4455       case 3:
4456          opcode = aco_opcode::buffer_store_format_xyz;
4457          break;
4458       case 4:
4459          opcode = aco_opcode::buffer_store_format_xyzw;
4460          break;
4461       default:
4462          unreachable(">4 channel buffer image store");
4463       }
4464       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4465       store->operands[0] = Operand(rsrc);
4466       store->operands[1] = Operand(vindex);
4467       store->operands[2] = Operand((uint32_t) 0);
4468       store->operands[3] = Operand(data);
4469       store->idxen = true;
4470       store->glc = glc;
4471       store->dlc = false;
4472       store->disable_wqm = true;
4473       store->barrier = barrier_image;
4474       ctx->program->needs_exact = true;
4475       ctx->block->instructions.emplace_back(std::move(store));
4476       return;
4477    }
4478
4479    assert(data.type() == RegType::vgpr);
4480    Temp coords = get_image_coords(ctx, instr, type);
4481    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4482
4483    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4484    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4485
4486    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
4487    store->operands[0] = Operand(resource);
4488    store->operands[1] = Operand(data);
4489    store->operands[2] = Operand(coords);
4490    store->glc = glc;
4491    store->dlc = false;
4492    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4493    store->dmask = (1 << data.size()) - 1;
4494    store->unrm = true;
4495    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4496    store->disable_wqm = true;
4497    store->barrier = barrier_image;
4498    ctx->program->needs_exact = true;
4499    ctx->block->instructions.emplace_back(std::move(store));
4500    return;
4501 }
4502
4503 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4504 {
4505    /* return the previous value if dest is ever used */
4506    bool return_previous = false;
4507    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4508       return_previous = true;
4509       break;
4510    }
4511    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4512       return_previous = true;
4513       break;
4514    }
4515
4516    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4517    const struct glsl_type *type = glsl_without_array(var->type);
4518    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4519    bool is_array = glsl_sampler_type_is_array(type);
4520    Builder bld(ctx->program, ctx->block);
4521
4522    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4523    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4524
4525    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4526       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4527
4528    aco_opcode buf_op, image_op;
4529    switch (instr->intrinsic) {
4530       case nir_intrinsic_image_deref_atomic_add:
4531          buf_op = aco_opcode::buffer_atomic_add;
4532          image_op = aco_opcode::image_atomic_add;
4533          break;
4534       case nir_intrinsic_image_deref_atomic_umin:
4535          buf_op = aco_opcode::buffer_atomic_umin;
4536          image_op = aco_opcode::image_atomic_umin;
4537          break;
4538       case nir_intrinsic_image_deref_atomic_imin:
4539          buf_op = aco_opcode::buffer_atomic_smin;
4540          image_op = aco_opcode::image_atomic_smin;
4541          break;
4542       case nir_intrinsic_image_deref_atomic_umax:
4543          buf_op = aco_opcode::buffer_atomic_umax;
4544          image_op = aco_opcode::image_atomic_umax;
4545          break;
4546       case nir_intrinsic_image_deref_atomic_imax:
4547          buf_op = aco_opcode::buffer_atomic_smax;
4548          image_op = aco_opcode::image_atomic_smax;
4549          break;
4550       case nir_intrinsic_image_deref_atomic_and:
4551          buf_op = aco_opcode::buffer_atomic_and;
4552          image_op = aco_opcode::image_atomic_and;
4553          break;
4554       case nir_intrinsic_image_deref_atomic_or:
4555          buf_op = aco_opcode::buffer_atomic_or;
4556          image_op = aco_opcode::image_atomic_or;
4557          break;
4558       case nir_intrinsic_image_deref_atomic_xor:
4559          buf_op = aco_opcode::buffer_atomic_xor;
4560          image_op = aco_opcode::image_atomic_xor;
4561          break;
4562       case nir_intrinsic_image_deref_atomic_exchange:
4563          buf_op = aco_opcode::buffer_atomic_swap;
4564          image_op = aco_opcode::image_atomic_swap;
4565          break;
4566       case nir_intrinsic_image_deref_atomic_comp_swap:
4567          buf_op = aco_opcode::buffer_atomic_cmpswap;
4568          image_op = aco_opcode::image_atomic_cmpswap;
4569          break;
4570       default:
4571          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4572    }
4573
4574    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4575
4576    if (dim == GLSL_SAMPLER_DIM_BUF) {
4577       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4578       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4579       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4580       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4581       mubuf->operands[0] = Operand(resource);
4582       mubuf->operands[1] = Operand(vindex);
4583       mubuf->operands[2] = Operand((uint32_t)0);
4584       mubuf->operands[3] = Operand(data);
4585       if (return_previous)
4586          mubuf->definitions[0] = Definition(dst);
4587       mubuf->offset = 0;
4588       mubuf->idxen = true;
4589       mubuf->glc = return_previous;
4590       mubuf->dlc = false; /* Not needed for atomics */
4591       mubuf->disable_wqm = true;
4592       mubuf->barrier = barrier_image;
4593       ctx->program->needs_exact = true;
4594       ctx->block->instructions.emplace_back(std::move(mubuf));
4595       return;
4596    }
4597
4598    Temp coords = get_image_coords(ctx, instr, type);
4599    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4600    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
4601    mimg->operands[0] = Operand(resource);
4602    mimg->operands[1] = Operand(data);
4603    mimg->operands[2] = Operand(coords);
4604    if (return_previous)
4605       mimg->definitions[0] = Definition(dst);
4606    mimg->glc = return_previous;
4607    mimg->dlc = false; /* Not needed for atomics */
4608    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4609    mimg->dmask = (1 << data.size()) - 1;
4610    mimg->unrm = true;
4611    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4612    mimg->disable_wqm = true;
4613    mimg->barrier = barrier_image;
4614    ctx->program->needs_exact = true;
4615    ctx->block->instructions.emplace_back(std::move(mimg));
4616    return;
4617 }
4618
4619 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4620 {
4621    if (in_elements && ctx->options->chip_class == GFX8) {
4622       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4623       Builder bld(ctx->program, ctx->block);
4624
4625       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4626
4627       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4628       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4629
4630       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4631       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4632
4633       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4634       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4635
4636       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4637       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4638                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4639       if (dst.type() == RegType::vgpr)
4640          bld.copy(Definition(dst), shr_dst);
4641
4642       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4643    } else {
4644       emit_extract_vector(ctx, desc, 2, dst);
4645    }
4646 }
4647
4648 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4649 {
4650    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4651    const struct glsl_type *type = glsl_without_array(var->type);
4652    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4653    bool is_array = glsl_sampler_type_is_array(type);
4654    Builder bld(ctx->program, ctx->block);
4655
4656    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4657       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4658       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4659    }
4660
4661    /* LOD */
4662    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4663
4664    /* Resource */
4665    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4666
4667    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4668
4669    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
4670    mimg->operands[0] = Operand(resource);
4671    mimg->operands[1] = Operand(s4); /* no sampler */
4672    mimg->operands[2] = Operand(lod);
4673    uint8_t& dmask = mimg->dmask;
4674    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4675    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4676    mimg->da = glsl_sampler_type_is_array(type);
4677    mimg->can_reorder = true;
4678    Definition& def = mimg->definitions[0];
4679    ctx->block->instructions.emplace_back(std::move(mimg));
4680
4681    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4682        glsl_sampler_type_is_array(type)) {
4683
4684       assert(instr->dest.ssa.num_components == 3);
4685       Temp tmp = {ctx->program->allocateId(), v3};
4686       def = Definition(tmp);
4687       emit_split_vector(ctx, tmp, 3);
4688
4689       /* divide 3rd value by 6 by multiplying with magic number */
4690       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4691       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4692
4693       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4694                  emit_extract_vector(ctx, tmp, 0, v1),
4695                  emit_extract_vector(ctx, tmp, 1, v1),
4696                  by_6);
4697
4698    } else if (ctx->options->chip_class == GFX9 &&
4699               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4700               glsl_sampler_type_is_array(type)) {
4701       assert(instr->dest.ssa.num_components == 2);
4702       def = Definition(dst);
4703       dmask = 0x5;
4704    } else {
4705       def = Definition(dst);
4706    }
4707
4708    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4709 }
4710
4711 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4712 {
4713    Builder bld(ctx->program, ctx->block);
4714    unsigned num_components = instr->num_components;
4715
4716    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4717    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4718    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4719
4720    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4721    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4722 }
4723
4724 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4725 {
4726    Builder bld(ctx->program, ctx->block);
4727    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4728    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4729    unsigned writemask = nir_intrinsic_write_mask(instr);
4730    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4731
4732    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4733    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4734
4735    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4736                ctx->options->chip_class >= GFX8;
4737    if (smem)
4738       offset = bld.as_uniform(offset);
4739    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4740
4741    while (writemask) {
4742       int start, count;
4743       u_bit_scan_consecutive_range(&writemask, &start, &count);
4744       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4745          /* GFX6 doesn't support storing vec3, split it. */
4746          writemask |= 1u << (start + 2);
4747          count = 2;
4748       }
4749       int num_bytes = count * elem_size_bytes;
4750
4751       if (num_bytes > 16) {
4752          assert(elem_size_bytes == 8);
4753          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4754          count = 2;
4755          num_bytes = 16;
4756       }
4757
4758       // TODO: check alignment of sub-dword stores
4759       // TODO: split 3 bytes. there is no store instruction for that
4760
4761       Temp write_data;
4762       if (count != instr->num_components) {
4763          emit_split_vector(ctx, data, instr->num_components);
4764          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4765          for (int i = 0; i < count; i++) {
4766             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4767             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4768          }
4769          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4770          vec->definitions[0] = Definition(write_data);
4771          ctx->block->instructions.emplace_back(std::move(vec));
4772       } else if (!smem && data.type() != RegType::vgpr) {
4773          assert(num_bytes % 4 == 0);
4774          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4775       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4776          assert(num_bytes % 4 == 0);
4777          write_data = bld.as_uniform(data);
4778       } else {
4779          write_data = data;
4780       }
4781
4782       aco_opcode vmem_op, smem_op;
4783       switch (num_bytes) {
4784          case 4:
4785             vmem_op = aco_opcode::buffer_store_dword;
4786             smem_op = aco_opcode::s_buffer_store_dword;
4787             break;
4788          case 8:
4789             vmem_op = aco_opcode::buffer_store_dwordx2;
4790             smem_op = aco_opcode::s_buffer_store_dwordx2;
4791             break;
4792          case 12:
4793             vmem_op = aco_opcode::buffer_store_dwordx3;
4794             smem_op = aco_opcode::last_opcode;
4795             assert(!smem && ctx->options->chip_class > GFX6);
4796             break;
4797          case 16:
4798             vmem_op = aco_opcode::buffer_store_dwordx4;
4799             smem_op = aco_opcode::s_buffer_store_dwordx4;
4800             break;
4801          default:
4802             unreachable("Store SSBO not implemented for this size.");
4803       }
4804       if (ctx->stage == fragment_fs)
4805          smem_op = aco_opcode::p_fs_buffer_store_smem;
4806
4807       if (smem) {
4808          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4809          store->operands[0] = Operand(rsrc);
4810          if (start) {
4811             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4812                                 offset, Operand(start * elem_size_bytes));
4813             store->operands[1] = Operand(off);
4814          } else {
4815             store->operands[1] = Operand(offset);
4816          }
4817          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4818             store->operands[1].setFixed(m0);
4819          store->operands[2] = Operand(write_data);
4820          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4821          store->dlc = false;
4822          store->disable_wqm = true;
4823          store->barrier = barrier_buffer;
4824          ctx->block->instructions.emplace_back(std::move(store));
4825          ctx->program->wb_smem_l1_on_end = true;
4826          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4827             ctx->block->kind |= block_kind_needs_lowering;
4828             ctx->program->needs_exact = true;
4829          }
4830       } else {
4831          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4832          store->operands[0] = Operand(rsrc);
4833          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4834          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4835          store->operands[3] = Operand(write_data);
4836          store->offset = start * elem_size_bytes;
4837          store->offen = (offset.type() == RegType::vgpr);
4838          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4839          store->dlc = false;
4840          store->disable_wqm = true;
4841          store->barrier = barrier_buffer;
4842          ctx->program->needs_exact = true;
4843          ctx->block->instructions.emplace_back(std::move(store));
4844       }
4845    }
4846 }
4847
4848 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4849 {
4850    /* return the previous value if dest is ever used */
4851    bool return_previous = false;
4852    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4853       return_previous = true;
4854       break;
4855    }
4856    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4857       return_previous = true;
4858       break;
4859    }
4860
4861    Builder bld(ctx->program, ctx->block);
4862    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4863
4864    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4865       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4866                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4867
4868    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4869    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4870    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4871
4872    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4873
4874    aco_opcode op32, op64;
4875    switch (instr->intrinsic) {
4876       case nir_intrinsic_ssbo_atomic_add:
4877          op32 = aco_opcode::buffer_atomic_add;
4878          op64 = aco_opcode::buffer_atomic_add_x2;
4879          break;
4880       case nir_intrinsic_ssbo_atomic_imin:
4881          op32 = aco_opcode::buffer_atomic_smin;
4882          op64 = aco_opcode::buffer_atomic_smin_x2;
4883          break;
4884       case nir_intrinsic_ssbo_atomic_umin:
4885          op32 = aco_opcode::buffer_atomic_umin;
4886          op64 = aco_opcode::buffer_atomic_umin_x2;
4887          break;
4888       case nir_intrinsic_ssbo_atomic_imax:
4889          op32 = aco_opcode::buffer_atomic_smax;
4890          op64 = aco_opcode::buffer_atomic_smax_x2;
4891          break;
4892       case nir_intrinsic_ssbo_atomic_umax:
4893          op32 = aco_opcode::buffer_atomic_umax;
4894          op64 = aco_opcode::buffer_atomic_umax_x2;
4895          break;
4896       case nir_intrinsic_ssbo_atomic_and:
4897          op32 = aco_opcode::buffer_atomic_and;
4898          op64 = aco_opcode::buffer_atomic_and_x2;
4899          break;
4900       case nir_intrinsic_ssbo_atomic_or:
4901          op32 = aco_opcode::buffer_atomic_or;
4902          op64 = aco_opcode::buffer_atomic_or_x2;
4903          break;
4904       case nir_intrinsic_ssbo_atomic_xor:
4905          op32 = aco_opcode::buffer_atomic_xor;
4906          op64 = aco_opcode::buffer_atomic_xor_x2;
4907          break;
4908       case nir_intrinsic_ssbo_atomic_exchange:
4909          op32 = aco_opcode::buffer_atomic_swap;
4910          op64 = aco_opcode::buffer_atomic_swap_x2;
4911          break;
4912       case nir_intrinsic_ssbo_atomic_comp_swap:
4913          op32 = aco_opcode::buffer_atomic_cmpswap;
4914          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4915          break;
4916       default:
4917          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4918    }
4919    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4920    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4921    mubuf->operands[0] = Operand(rsrc);
4922    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4923    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4924    mubuf->operands[3] = Operand(data);
4925    if (return_previous)
4926       mubuf->definitions[0] = Definition(dst);
4927    mubuf->offset = 0;
4928    mubuf->offen = (offset.type() == RegType::vgpr);
4929    mubuf->glc = return_previous;
4930    mubuf->dlc = false; /* Not needed for atomics */
4931    mubuf->disable_wqm = true;
4932    mubuf->barrier = barrier_buffer;
4933    ctx->program->needs_exact = true;
4934    ctx->block->instructions.emplace_back(std::move(mubuf));
4935 }
4936
4937 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4938
4939    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4940    Builder bld(ctx->program, ctx->block);
4941    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4942    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4943 }
4944
4945 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4946 {
4947    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4948                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4949
4950    if (addr.type() == RegType::vgpr)
4951       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4952    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4953 }
4954
4955 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4956 {
4957    Builder bld(ctx->program, ctx->block);
4958    unsigned num_components = instr->num_components;
4959    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4960
4961    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4962    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4963
4964    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4965    bool dlc = glc && ctx->options->chip_class >= GFX10;
4966    aco_opcode op;
4967    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4968       bool global = ctx->options->chip_class >= GFX9;
4969
4970       if (ctx->options->chip_class >= GFX7) {
4971          aco_opcode op;
4972          switch (num_bytes) {
4973          case 4:
4974             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4975             break;
4976          case 8:
4977             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4978             break;
4979          case 12:
4980             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4981             break;
4982          case 16:
4983             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4984             break;
4985          default:
4986             unreachable("load_global not implemented for this size.");
4987          }
4988
4989          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4990          flat->operands[0] = Operand(addr);
4991          flat->operands[1] = Operand(s1);
4992          flat->glc = glc;
4993          flat->dlc = dlc;
4994          flat->barrier = barrier_buffer;
4995
4996          if (dst.type() == RegType::sgpr) {
4997             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4998             flat->definitions[0] = Definition(vec);
4999             ctx->block->instructions.emplace_back(std::move(flat));
5000             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5001          } else {
5002             flat->definitions[0] = Definition(dst);
5003             ctx->block->instructions.emplace_back(std::move(flat));
5004          }
5005          emit_split_vector(ctx, dst, num_components);
5006       } else {
5007          assert(ctx->options->chip_class == GFX6);
5008
5009          /* GFX6 doesn't support loading vec3, expand to vec4. */
5010          num_bytes = num_bytes == 12 ? 16 : num_bytes;
5011
5012          aco_opcode op;
5013          switch (num_bytes) {
5014          case 4:
5015             op = aco_opcode::buffer_load_dword;
5016             break;
5017          case 8:
5018             op = aco_opcode::buffer_load_dwordx2;
5019             break;
5020          case 16:
5021             op = aco_opcode::buffer_load_dwordx4;
5022             break;
5023          default:
5024             unreachable("load_global not implemented for this size.");
5025          }
5026
5027          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5028
5029          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
5030          mubuf->operands[0] = Operand(rsrc);
5031          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5032          mubuf->operands[2] = Operand(0u);
5033          mubuf->glc = glc;
5034          mubuf->dlc = false;
5035          mubuf->offset = 0;
5036          mubuf->addr64 = addr.type() == RegType::vgpr;
5037          mubuf->disable_wqm = false;
5038          mubuf->barrier = barrier_buffer;
5039          aco_ptr<Instruction> instr = std::move(mubuf);
5040
5041          /* expand vector */
5042          if (dst.size() == 3) {
5043             Temp vec = bld.tmp(v4);
5044             instr->definitions[0] = Definition(vec);
5045             bld.insert(std::move(instr));
5046             emit_split_vector(ctx, vec, 4);
5047
5048             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5049             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5050             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5051             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5052          }
5053
5054          if (dst.type() == RegType::sgpr) {
5055             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5056             instr->definitions[0] = Definition(vec);
5057             bld.insert(std::move(instr));
5058             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5059             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5060          } else {
5061             instr->definitions[0] = Definition(dst);
5062             bld.insert(std::move(instr));
5063             emit_split_vector(ctx, dst, num_components);
5064          }
5065       }
5066    } else {
5067       switch (num_bytes) {
5068          case 4:
5069             op = aco_opcode::s_load_dword;
5070             break;
5071          case 8:
5072             op = aco_opcode::s_load_dwordx2;
5073             break;
5074          case 12:
5075          case 16:
5076             op = aco_opcode::s_load_dwordx4;
5077             break;
5078          default:
5079             unreachable("load_global not implemented for this size.");
5080       }
5081       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5082       load->operands[0] = Operand(addr);
5083       load->operands[1] = Operand(0u);
5084       load->definitions[0] = Definition(dst);
5085       load->glc = glc;
5086       load->dlc = dlc;
5087       load->barrier = barrier_buffer;
5088       assert(ctx->options->chip_class >= GFX8 || !glc);
5089
5090       if (dst.size() == 3) {
5091          /* trim vector */
5092          Temp vec = bld.tmp(s4);
5093          load->definitions[0] = Definition(vec);
5094          ctx->block->instructions.emplace_back(std::move(load));
5095          emit_split_vector(ctx, vec, 4);
5096
5097          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5098                     emit_extract_vector(ctx, vec, 0, s1),
5099                     emit_extract_vector(ctx, vec, 1, s1),
5100                     emit_extract_vector(ctx, vec, 2, s1));
5101       } else {
5102          ctx->block->instructions.emplace_back(std::move(load));
5103       }
5104    }
5105 }
5106
5107 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5108 {
5109    Builder bld(ctx->program, ctx->block);
5110    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5111
5112    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5113    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5114
5115    if (ctx->options->chip_class >= GFX7)
5116       addr = as_vgpr(ctx, addr);
5117
5118    unsigned writemask = nir_intrinsic_write_mask(instr);
5119    while (writemask) {
5120       int start, count;
5121       u_bit_scan_consecutive_range(&writemask, &start, &count);
5122       if (count == 3 && ctx->options->chip_class == GFX6) {
5123          /* GFX6 doesn't support storing vec3, split it. */
5124          writemask |= 1u << (start + 2);
5125          count = 2;
5126       }
5127       unsigned num_bytes = count * elem_size_bytes;
5128
5129       Temp write_data = data;
5130       if (count != instr->num_components) {
5131          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5132          for (int i = 0; i < count; i++)
5133             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5134          write_data = bld.tmp(RegType::vgpr, count);
5135          vec->definitions[0] = Definition(write_data);
5136          ctx->block->instructions.emplace_back(std::move(vec));
5137       }
5138
5139       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5140       unsigned offset = start * elem_size_bytes;
5141
5142       if (ctx->options->chip_class >= GFX7) {
5143          if (offset > 0 && ctx->options->chip_class < GFX9) {
5144             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5145             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5146             Temp carry = bld.tmp(bld.lm);
5147             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5148
5149             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5150                      Operand(offset), addr0);
5151             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5152                      Operand(0u), addr1,
5153                      carry).def(1).setHint(vcc);
5154
5155             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5156
5157             offset = 0;
5158          }
5159
5160          bool global = ctx->options->chip_class >= GFX9;
5161          aco_opcode op;
5162          switch (num_bytes) {
5163          case 4:
5164             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5165             break;
5166          case 8:
5167             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5168             break;
5169          case 12:
5170             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5171             break;
5172          case 16:
5173             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5174             break;
5175          default:
5176             unreachable("store_global not implemented for this size.");
5177          }
5178
5179          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5180          flat->operands[0] = Operand(addr);
5181          flat->operands[1] = Operand(s1);
5182          flat->operands[2] = Operand(data);
5183          flat->glc = glc;
5184          flat->dlc = false;
5185          flat->offset = offset;
5186          flat->disable_wqm = true;
5187          flat->barrier = barrier_buffer;
5188          ctx->program->needs_exact = true;
5189          ctx->block->instructions.emplace_back(std::move(flat));
5190       } else {
5191          assert(ctx->options->chip_class == GFX6);
5192
5193          aco_opcode op;
5194          switch (num_bytes) {
5195          case 4:
5196             op = aco_opcode::buffer_store_dword;
5197             break;
5198          case 8:
5199             op = aco_opcode::buffer_store_dwordx2;
5200             break;
5201          case 16:
5202             op = aco_opcode::buffer_store_dwordx4;
5203             break;
5204          default:
5205             unreachable("store_global not implemented for this size.");
5206          }
5207
5208          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5209
5210          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5211          mubuf->operands[0] = Operand(rsrc);
5212          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5213          mubuf->operands[2] = Operand(0u);
5214          mubuf->operands[3] = Operand(write_data);
5215          mubuf->glc = glc;
5216          mubuf->dlc = false;
5217          mubuf->offset = offset;
5218          mubuf->addr64 = addr.type() == RegType::vgpr;
5219          mubuf->disable_wqm = true;
5220          mubuf->barrier = barrier_buffer;
5221          ctx->program->needs_exact = true;
5222          ctx->block->instructions.emplace_back(std::move(mubuf));
5223       }
5224    }
5225 }
5226
5227 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5228 {
5229    /* return the previous value if dest is ever used */
5230    bool return_previous = false;
5231    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5232       return_previous = true;
5233       break;
5234    }
5235    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5236       return_previous = true;
5237       break;
5238    }
5239
5240    Builder bld(ctx->program, ctx->block);
5241    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5242    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5243
5244    if (ctx->options->chip_class >= GFX7)
5245       addr = as_vgpr(ctx, addr);
5246
5247    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5248       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5249                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5250
5251    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5252
5253    aco_opcode op32, op64;
5254
5255    if (ctx->options->chip_class >= GFX7) {
5256       bool global = ctx->options->chip_class >= GFX9;
5257       switch (instr->intrinsic) {
5258          case nir_intrinsic_global_atomic_add:
5259             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5260             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5261             break;
5262          case nir_intrinsic_global_atomic_imin:
5263             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5264             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5265             break;
5266          case nir_intrinsic_global_atomic_umin:
5267             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5268             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5269             break;
5270          case nir_intrinsic_global_atomic_imax:
5271             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5272             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5273             break;
5274          case nir_intrinsic_global_atomic_umax:
5275             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5276             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5277             break;
5278          case nir_intrinsic_global_atomic_and:
5279             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5280             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5281             break;
5282          case nir_intrinsic_global_atomic_or:
5283             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5284             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5285             break;
5286          case nir_intrinsic_global_atomic_xor:
5287             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5288             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5289             break;
5290          case nir_intrinsic_global_atomic_exchange:
5291             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5292             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5293             break;
5294          case nir_intrinsic_global_atomic_comp_swap:
5295             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5296             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5297             break;
5298          default:
5299             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5300       }
5301
5302       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5303       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5304       flat->operands[0] = Operand(addr);
5305       flat->operands[1] = Operand(s1);
5306       flat->operands[2] = Operand(data);
5307       if (return_previous)
5308          flat->definitions[0] = Definition(dst);
5309       flat->glc = return_previous;
5310       flat->dlc = false; /* Not needed for atomics */
5311       flat->offset = 0;
5312       flat->disable_wqm = true;
5313       flat->barrier = barrier_buffer;
5314       ctx->program->needs_exact = true;
5315       ctx->block->instructions.emplace_back(std::move(flat));
5316    } else {
5317       assert(ctx->options->chip_class == GFX6);
5318
5319       switch (instr->intrinsic) {
5320          case nir_intrinsic_global_atomic_add:
5321             op32 = aco_opcode::buffer_atomic_add;
5322             op64 = aco_opcode::buffer_atomic_add_x2;
5323             break;
5324          case nir_intrinsic_global_atomic_imin:
5325             op32 = aco_opcode::buffer_atomic_smin;
5326             op64 = aco_opcode::buffer_atomic_smin_x2;
5327             break;
5328          case nir_intrinsic_global_atomic_umin:
5329             op32 = aco_opcode::buffer_atomic_umin;
5330             op64 = aco_opcode::buffer_atomic_umin_x2;
5331             break;
5332          case nir_intrinsic_global_atomic_imax:
5333             op32 = aco_opcode::buffer_atomic_smax;
5334             op64 = aco_opcode::buffer_atomic_smax_x2;
5335             break;
5336          case nir_intrinsic_global_atomic_umax:
5337             op32 = aco_opcode::buffer_atomic_umax;
5338             op64 = aco_opcode::buffer_atomic_umax_x2;
5339             break;
5340          case nir_intrinsic_global_atomic_and:
5341             op32 = aco_opcode::buffer_atomic_and;
5342             op64 = aco_opcode::buffer_atomic_and_x2;
5343             break;
5344          case nir_intrinsic_global_atomic_or:
5345             op32 = aco_opcode::buffer_atomic_or;
5346             op64 = aco_opcode::buffer_atomic_or_x2;
5347             break;
5348          case nir_intrinsic_global_atomic_xor:
5349             op32 = aco_opcode::buffer_atomic_xor;
5350             op64 = aco_opcode::buffer_atomic_xor_x2;
5351             break;
5352          case nir_intrinsic_global_atomic_exchange:
5353             op32 = aco_opcode::buffer_atomic_swap;
5354             op64 = aco_opcode::buffer_atomic_swap_x2;
5355             break;
5356          case nir_intrinsic_global_atomic_comp_swap:
5357             op32 = aco_opcode::buffer_atomic_cmpswap;
5358             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5359             break;
5360          default:
5361             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5362       }
5363
5364       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5365
5366       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5367
5368       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5369       mubuf->operands[0] = Operand(rsrc);
5370       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5371       mubuf->operands[2] = Operand(0u);
5372       mubuf->operands[3] = Operand(data);
5373       if (return_previous)
5374          mubuf->definitions[0] = Definition(dst);
5375       mubuf->glc = return_previous;
5376       mubuf->dlc = false;
5377       mubuf->offset = 0;
5378       mubuf->addr64 = addr.type() == RegType::vgpr;
5379       mubuf->disable_wqm = true;
5380       mubuf->barrier = barrier_buffer;
5381       ctx->program->needs_exact = true;
5382       ctx->block->instructions.emplace_back(std::move(mubuf));
5383    }
5384 }
5385
5386 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5387    Builder bld(ctx->program, ctx->block);
5388    switch(instr->intrinsic) {
5389       case nir_intrinsic_group_memory_barrier:
5390       case nir_intrinsic_memory_barrier:
5391          bld.barrier(aco_opcode::p_memory_barrier_common);
5392          break;
5393       case nir_intrinsic_memory_barrier_buffer:
5394          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5395          break;
5396       case nir_intrinsic_memory_barrier_image:
5397          bld.barrier(aco_opcode::p_memory_barrier_image);
5398          break;
5399       case nir_intrinsic_memory_barrier_shared:
5400          bld.barrier(aco_opcode::p_memory_barrier_shared);
5401          break;
5402       default:
5403          unreachable("Unimplemented memory barrier intrinsic");
5404          break;
5405    }
5406 }
5407
5408 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5409 {
5410    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5411    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5412    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5413    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5414    Builder bld(ctx->program, ctx->block);
5415
5416    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5417    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5418    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5419 }
5420
5421 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5422 {
5423    unsigned writemask = nir_intrinsic_write_mask(instr);
5424    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5425    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5426    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5427    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5428
5429    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5430    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5431 }
5432
5433 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5434 {
5435    unsigned offset = nir_intrinsic_base(instr);
5436    Operand m = load_lds_size_m0(ctx);
5437    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5438    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5439
5440    unsigned num_operands = 3;
5441    aco_opcode op32, op64, op32_rtn, op64_rtn;
5442    switch(instr->intrinsic) {
5443       case nir_intrinsic_shared_atomic_add:
5444          op32 = aco_opcode::ds_add_u32;
5445          op64 = aco_opcode::ds_add_u64;
5446          op32_rtn = aco_opcode::ds_add_rtn_u32;
5447          op64_rtn = aco_opcode::ds_add_rtn_u64;
5448          break;
5449       case nir_intrinsic_shared_atomic_imin:
5450          op32 = aco_opcode::ds_min_i32;
5451          op64 = aco_opcode::ds_min_i64;
5452          op32_rtn = aco_opcode::ds_min_rtn_i32;
5453          op64_rtn = aco_opcode::ds_min_rtn_i64;
5454          break;
5455       case nir_intrinsic_shared_atomic_umin:
5456          op32 = aco_opcode::ds_min_u32;
5457          op64 = aco_opcode::ds_min_u64;
5458          op32_rtn = aco_opcode::ds_min_rtn_u32;
5459          op64_rtn = aco_opcode::ds_min_rtn_u64;
5460          break;
5461       case nir_intrinsic_shared_atomic_imax:
5462          op32 = aco_opcode::ds_max_i32;
5463          op64 = aco_opcode::ds_max_i64;
5464          op32_rtn = aco_opcode::ds_max_rtn_i32;
5465          op64_rtn = aco_opcode::ds_max_rtn_i64;
5466          break;
5467       case nir_intrinsic_shared_atomic_umax:
5468          op32 = aco_opcode::ds_max_u32;
5469          op64 = aco_opcode::ds_max_u64;
5470          op32_rtn = aco_opcode::ds_max_rtn_u32;
5471          op64_rtn = aco_opcode::ds_max_rtn_u64;
5472          break;
5473       case nir_intrinsic_shared_atomic_and:
5474          op32 = aco_opcode::ds_and_b32;
5475          op64 = aco_opcode::ds_and_b64;
5476          op32_rtn = aco_opcode::ds_and_rtn_b32;
5477          op64_rtn = aco_opcode::ds_and_rtn_b64;
5478          break;
5479       case nir_intrinsic_shared_atomic_or:
5480          op32 = aco_opcode::ds_or_b32;
5481          op64 = aco_opcode::ds_or_b64;
5482          op32_rtn = aco_opcode::ds_or_rtn_b32;
5483          op64_rtn = aco_opcode::ds_or_rtn_b64;
5484          break;
5485       case nir_intrinsic_shared_atomic_xor:
5486          op32 = aco_opcode::ds_xor_b32;
5487          op64 = aco_opcode::ds_xor_b64;
5488          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5489          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5490          break;
5491       case nir_intrinsic_shared_atomic_exchange:
5492          op32 = aco_opcode::ds_write_b32;
5493          op64 = aco_opcode::ds_write_b64;
5494          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5495          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5496          break;
5497       case nir_intrinsic_shared_atomic_comp_swap:
5498          op32 = aco_opcode::ds_cmpst_b32;
5499          op64 = aco_opcode::ds_cmpst_b64;
5500          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5501          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5502          num_operands = 4;
5503          break;
5504       default:
5505          unreachable("Unhandled shared atomic intrinsic");
5506    }
5507
5508    /* return the previous value if dest is ever used */
5509    bool return_previous = false;
5510    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5511       return_previous = true;
5512       break;
5513    }
5514    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5515       return_previous = true;
5516       break;
5517    }
5518
5519    aco_opcode op;
5520    if (data.size() == 1) {
5521       assert(instr->dest.ssa.bit_size == 32);
5522       op = return_previous ? op32_rtn : op32;
5523    } else {
5524       assert(instr->dest.ssa.bit_size == 64);
5525       op = return_previous ? op64_rtn : op64;
5526    }
5527
5528    if (offset > 65535) {
5529       Builder bld(ctx->program, ctx->block);
5530       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5531       offset = 0;
5532    }
5533
5534    aco_ptr<DS_instruction> ds;
5535    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5536    ds->operands[0] = Operand(address);
5537    ds->operands[1] = Operand(data);
5538    if (num_operands == 4)
5539       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5540    ds->operands[num_operands - 1] = m;
5541    ds->offset0 = offset;
5542    if (return_previous)
5543       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5544    ctx->block->instructions.emplace_back(std::move(ds));
5545 }
5546
5547 Temp get_scratch_resource(isel_context *ctx)
5548 {
5549    Builder bld(ctx->program, ctx->block);
5550    Temp scratch_addr = ctx->program->private_segment_buffer;
5551    if (ctx->stage != compute_cs)
5552       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5553
5554    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5555                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5556
5557    if (ctx->program->chip_class >= GFX10) {
5558       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5559                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5560                    S_008F0C_RESOURCE_LEVEL(1);
5561    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5562       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5563                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5564    }
5565
5566    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5567    if (ctx->program->chip_class <= GFX8)
5568       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5569
5570    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5571 }
5572
5573 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5574    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5575    Builder bld(ctx->program, ctx->block);
5576    Temp rsrc = get_scratch_resource(ctx);
5577    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5578    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5579
5580    aco_opcode op;
5581    switch (dst.size()) {
5582       case 1:
5583          op = aco_opcode::buffer_load_dword;
5584          break;
5585       case 2:
5586          op = aco_opcode::buffer_load_dwordx2;
5587          break;
5588       case 3:
5589          op = aco_opcode::buffer_load_dwordx3;
5590          break;
5591       case 4:
5592          op = aco_opcode::buffer_load_dwordx4;
5593          break;
5594       case 6:
5595       case 8: {
5596          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5597          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5598                                 bld.def(v4), rsrc, offset,
5599                                 ctx->program->scratch_offset, 0, true);
5600          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5601                                                   aco_opcode::buffer_load_dwordx4,
5602                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5603                                 rsrc, offset, ctx->program->scratch_offset, 16, true);
5604          emit_split_vector(ctx, lower, 2);
5605          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5606          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5607          if (dst.size() == 8) {
5608             emit_split_vector(ctx, upper, 2);
5609             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5610             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5611          } else {
5612             elems[2] = upper;
5613          }
5614
5615          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5616                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5617          for (unsigned i = 0; i < dst.size() / 2; i++)
5618             vec->operands[i] = Operand(elems[i]);
5619          vec->definitions[0] = Definition(dst);
5620          bld.insert(std::move(vec));
5621          ctx->allocated_vec.emplace(dst.id(), elems);
5622          return;
5623       }
5624       default:
5625          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5626    }
5627
5628    bld.mubuf(op, Definition(dst), rsrc, offset, ctx->program->scratch_offset, 0, true);
5629    emit_split_vector(ctx, dst, instr->num_components);
5630 }
5631
5632 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5633    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5634    Builder bld(ctx->program, ctx->block);
5635    Temp rsrc = get_scratch_resource(ctx);
5636    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5637    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5638
5639    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5640    unsigned writemask = nir_intrinsic_write_mask(instr);
5641
5642    while (writemask) {
5643       int start, count;
5644       u_bit_scan_consecutive_range(&writemask, &start, &count);
5645       int num_bytes = count * elem_size_bytes;
5646
5647       if (num_bytes > 16) {
5648          assert(elem_size_bytes == 8);
5649          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5650          count = 2;
5651          num_bytes = 16;
5652       }
5653
5654       // TODO: check alignment of sub-dword stores
5655       // TODO: split 3 bytes. there is no store instruction for that
5656
5657       Temp write_data;
5658       if (count != instr->num_components) {
5659          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5660          for (int i = 0; i < count; i++) {
5661             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5662             vec->operands[i] = Operand(elem);
5663          }
5664          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5665          vec->definitions[0] = Definition(write_data);
5666          ctx->block->instructions.emplace_back(std::move(vec));
5667       } else {
5668          write_data = data;
5669       }
5670
5671       aco_opcode op;
5672       switch (num_bytes) {
5673          case 4:
5674             op = aco_opcode::buffer_store_dword;
5675             break;
5676          case 8:
5677             op = aco_opcode::buffer_store_dwordx2;
5678             break;
5679          case 12:
5680             op = aco_opcode::buffer_store_dwordx3;
5681             break;
5682          case 16:
5683             op = aco_opcode::buffer_store_dwordx4;
5684             break;
5685          default:
5686             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5687       }
5688
5689       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5690    }
5691 }
5692
5693 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5694    uint8_t log2_ps_iter_samples;
5695    if (ctx->program->info->ps.force_persample) {
5696       log2_ps_iter_samples =
5697          util_logbase2(ctx->options->key.fs.num_samples);
5698    } else {
5699       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5700    }
5701
5702    /* The bit pattern matches that used by fixed function fragment
5703     * processing. */
5704    static const unsigned ps_iter_masks[] = {
5705       0xffff, /* not used */
5706       0x5555,
5707       0x1111,
5708       0x0101,
5709       0x0001,
5710    };
5711    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5712
5713    Builder bld(ctx->program, ctx->block);
5714
5715    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5716                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5717    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5718    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5719    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5720    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5721 }
5722
5723 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
5724    Builder bld(ctx->program, ctx->block);
5725
5726    unsigned stream = nir_intrinsic_stream_id(instr);
5727    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5728    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
5729    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
5730
5731    /* get GSVS ring */
5732    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
5733
5734    unsigned num_components =
5735       ctx->program->info->gs.num_stream_output_components[stream];
5736    assert(num_components);
5737
5738    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
5739    unsigned stream_offset = 0;
5740    for (unsigned i = 0; i < stream; i++) {
5741       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
5742       stream_offset += prev_stride * ctx->program->wave_size;
5743    }
5744
5745    /* Limit on the stride field for <= GFX7. */
5746    assert(stride < (1 << 14));
5747
5748    Temp gsvs_dwords[4];
5749    for (unsigned i = 0; i < 4; i++)
5750       gsvs_dwords[i] = bld.tmp(s1);
5751    bld.pseudo(aco_opcode::p_split_vector,
5752               Definition(gsvs_dwords[0]),
5753               Definition(gsvs_dwords[1]),
5754               Definition(gsvs_dwords[2]),
5755               Definition(gsvs_dwords[3]),
5756               gsvs_ring);
5757
5758    if (stream_offset) {
5759       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
5760
5761       Temp carry = bld.tmp(s1);
5762       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
5763       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
5764    }
5765
5766    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
5767    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
5768
5769    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5770                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
5771
5772    unsigned offset = 0;
5773    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
5774       if (ctx->program->info->gs.output_streams[i] != stream)
5775          continue;
5776
5777       for (unsigned j = 0; j < 4; j++) {
5778          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
5779             continue;
5780
5781          if (ctx->outputs.mask[i] & (1 << j)) {
5782             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
5783             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
5784             if (const_offset >= 4096u) {
5785                if (vaddr_offset.isUndefined())
5786                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
5787                else
5788                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
5789                const_offset %= 4096u;
5790             }
5791
5792             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
5793             mtbuf->operands[0] = Operand(gsvs_ring);
5794             mtbuf->operands[1] = vaddr_offset;
5795             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
5796             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
5797             mtbuf->offen = !vaddr_offset.isUndefined();
5798             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
5799             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
5800             mtbuf->offset = const_offset;
5801             mtbuf->glc = true;
5802             mtbuf->slc = true;
5803             mtbuf->barrier = barrier_gs_data;
5804             mtbuf->can_reorder = true;
5805             bld.insert(std::move(mtbuf));
5806          }
5807
5808          offset += ctx->shader->info.gs.vertices_out;
5809       }
5810
5811       /* outputs for the next vertex are undefined and keeping them around can
5812        * create invalid IR with control flow */
5813       ctx->outputs.mask[i] = 0;
5814    }
5815
5816    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
5817 }
5818
5819 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5820 {
5821    Builder bld(ctx->program, ctx->block);
5822
5823    if (cluster_size == 1) {
5824       return src;
5825    } if (op == nir_op_iand && cluster_size == 4) {
5826       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5827       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5828       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5829                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5830    } else if (op == nir_op_ior && cluster_size == 4) {
5831       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5832       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5833                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5834    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5835       //subgroupAnd(val) -> (exec & ~val) == 0
5836       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5837       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5838       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5839    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5840       //subgroupOr(val) -> (val & exec) != 0
5841       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5842       return bool_to_vector_condition(ctx, tmp);
5843    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5844       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5845       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5846       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5847       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5848       return bool_to_vector_condition(ctx, tmp);
5849    } else {
5850       //subgroupClustered{And,Or,Xor}(val, n) ->
5851       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5852       //cluster_offset = ~(n - 1) & lane_id
5853       //cluster_mask = ((1 << n) - 1)
5854       //subgroupClusteredAnd():
5855       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5856       //subgroupClusteredOr():
5857       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5858       //subgroupClusteredXor():
5859       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5860       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5861       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5862
5863       Temp tmp;
5864       if (op == nir_op_iand)
5865          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5866       else
5867          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5868
5869       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5870
5871       if (ctx->program->chip_class <= GFX7)
5872          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5873       else if (ctx->program->wave_size == 64)
5874          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5875       else
5876          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5877       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5878       if (cluster_mask != 0xffffffff)
5879          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5880
5881       Definition cmp_def = Definition();
5882       if (op == nir_op_iand) {
5883          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5884       } else if (op == nir_op_ior) {
5885          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5886       } else if (op == nir_op_ixor) {
5887          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5888                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5889          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5890       }
5891       cmp_def.setHint(vcc);
5892       return cmp_def.getTemp();
5893    }
5894 }
5895
5896 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5897 {
5898    Builder bld(ctx->program, ctx->block);
5899
5900    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5901    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5902    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5903    Temp tmp;
5904    if (op == nir_op_iand)
5905       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5906    else
5907       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5908
5909    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5910    Temp lo = lohi.def(0).getTemp();
5911    Temp hi = lohi.def(1).getTemp();
5912    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5913
5914    Definition cmp_def = Definition();
5915    if (op == nir_op_iand)
5916       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5917    else if (op == nir_op_ior)
5918       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5919    else if (op == nir_op_ixor)
5920       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5921                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5922    cmp_def.setHint(vcc);
5923    return cmp_def.getTemp();
5924 }
5925
5926 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5927 {
5928    Builder bld(ctx->program, ctx->block);
5929
5930    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5931    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5932    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5933    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5934    if (op == nir_op_iand)
5935       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5936    else if (op == nir_op_ior)
5937       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5938    else if (op == nir_op_ixor)
5939       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5940
5941    assert(false);
5942    return Temp();
5943 }
5944
5945 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5946 {
5947    Builder bld(ctx->program, ctx->block);
5948    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5949    if (src.regClass().type() == RegType::vgpr) {
5950       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5951    } else if (src.regClass() == s1) {
5952       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5953    } else if (src.regClass() == s2) {
5954       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5955    } else {
5956       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5957       nir_print_instr(&instr->instr, stderr);
5958       fprintf(stderr, "\n");
5959    }
5960 }
5961
5962 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5963 {
5964    Builder bld(ctx->program, ctx->block);
5965    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5966    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5967    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5968
5969    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5970    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5971    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5972    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5973
5974    /* Build DD X/Y */
5975    if (ctx->program->chip_class >= GFX8) {
5976       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5977       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5978       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5979       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5980       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5981       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5982    } else {
5983       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5984       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5985       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5986       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5987       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5988       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5989       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5990       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5991       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5992       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5993    }
5994
5995    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5996    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5997    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5998    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5999    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
6000    Temp wqm1 = bld.tmp(v1);
6001    emit_wqm(ctx, tmp1, wqm1, true);
6002    Temp wqm2 = bld.tmp(v1);
6003    emit_wqm(ctx, tmp2, wqm2, true);
6004    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
6005    return;
6006 }
6007
6008 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
6009 {
6010    Builder bld(ctx->program, ctx->block);
6011    switch(instr->intrinsic) {
6012    case nir_intrinsic_load_barycentric_sample:
6013    case nir_intrinsic_load_barycentric_pixel:
6014    case nir_intrinsic_load_barycentric_centroid: {
6015       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
6016       Temp bary = Temp(0, s2);
6017       switch (mode) {
6018       case INTERP_MODE_SMOOTH:
6019       case INTERP_MODE_NONE:
6020          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6021             bary = get_arg(ctx, ctx->args->ac.persp_center);
6022          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6023             bary = ctx->persp_centroid;
6024          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6025             bary = get_arg(ctx, ctx->args->ac.persp_sample);
6026          break;
6027       case INTERP_MODE_NOPERSPECTIVE:
6028          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6029             bary = get_arg(ctx, ctx->args->ac.linear_center);
6030          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6031             bary = ctx->linear_centroid;
6032          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6033             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6034          break;
6035       default:
6036          break;
6037       }
6038       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6039       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6040       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6041       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6042                  Operand(p1), Operand(p2));
6043       emit_split_vector(ctx, dst, 2);
6044       break;
6045    }
6046    case nir_intrinsic_load_barycentric_model: {
6047       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
6048
6049       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6050       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
6051       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
6052       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
6053       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6054                  Operand(p1), Operand(p2), Operand(p3));
6055       emit_split_vector(ctx, dst, 3);
6056       break;
6057    }
6058    case nir_intrinsic_load_barycentric_at_sample: {
6059       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6060       switch (ctx->options->key.fs.num_samples) {
6061          case 2: sample_pos_offset += 1 << 3; break;
6062          case 4: sample_pos_offset += 3 << 3; break;
6063          case 8: sample_pos_offset += 7 << 3; break;
6064          default: break;
6065       }
6066       Temp sample_pos;
6067       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6068       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6069       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6070       if (addr.type() == RegType::sgpr) {
6071          Operand offset;
6072          if (const_addr) {
6073             sample_pos_offset += const_addr->u32 << 3;
6074             offset = Operand(sample_pos_offset);
6075          } else if (ctx->options->chip_class >= GFX9) {
6076             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6077          } else {
6078             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6079             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6080          }
6081
6082          Operand off = bld.copy(bld.def(s1), Operand(offset));
6083          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6084
6085       } else if (ctx->options->chip_class >= GFX9) {
6086          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6087          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6088       } else if (ctx->options->chip_class >= GFX7) {
6089          /* addr += private_segment_buffer + sample_pos_offset */
6090          Temp tmp0 = bld.tmp(s1);
6091          Temp tmp1 = bld.tmp(s1);
6092          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6093          Definition scc_tmp = bld.def(s1, scc);
6094          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6095          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6096          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6097          Temp pck0 = bld.tmp(v1);
6098          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6099          tmp1 = as_vgpr(ctx, tmp1);
6100          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6101          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6102
6103          /* sample_pos = flat_load_dwordx2 addr */
6104          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6105       } else {
6106          assert(ctx->options->chip_class == GFX6);
6107
6108          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6109                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6110          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6111
6112          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6113          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6114
6115          sample_pos = bld.tmp(v2);
6116
6117          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6118          load->definitions[0] = Definition(sample_pos);
6119          load->operands[0] = Operand(rsrc);
6120          load->operands[1] = Operand(addr);
6121          load->operands[2] = Operand(0u);
6122          load->offset = sample_pos_offset;
6123          load->offen = 0;
6124          load->addr64 = true;
6125          load->glc = false;
6126          load->dlc = false;
6127          load->disable_wqm = false;
6128          load->barrier = barrier_none;
6129          load->can_reorder = true;
6130          ctx->block->instructions.emplace_back(std::move(load));
6131       }
6132
6133       /* sample_pos -= 0.5 */
6134       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6135       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6136       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6137       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6138       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6139
6140       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6141       break;
6142    }
6143    case nir_intrinsic_load_barycentric_at_offset: {
6144       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6145       RegClass rc = RegClass(offset.type(), 1);
6146       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6147       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6148       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6149       break;
6150    }
6151    case nir_intrinsic_load_front_face: {
6152       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6153                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6154       break;
6155    }
6156    case nir_intrinsic_load_view_index:
6157    case nir_intrinsic_load_layer_id: {
6158       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & (sw_vs | sw_gs))) {
6159          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6160          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6161          break;
6162       }
6163
6164       unsigned idx = nir_intrinsic_base(instr);
6165       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6166                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6167       break;
6168    }
6169    case nir_intrinsic_load_frag_coord: {
6170       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6171       break;
6172    }
6173    case nir_intrinsic_load_sample_pos: {
6174       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6175       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6176       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6177                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6178                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6179       break;
6180    }
6181    case nir_intrinsic_load_interpolated_input:
6182       visit_load_interpolated_input(ctx, instr);
6183       break;
6184    case nir_intrinsic_store_output:
6185       visit_store_output(ctx, instr);
6186       break;
6187    case nir_intrinsic_load_input:
6188    case nir_intrinsic_load_input_vertex:
6189       visit_load_input(ctx, instr);
6190       break;
6191    case nir_intrinsic_load_per_vertex_input:
6192       visit_load_per_vertex_input(ctx, instr);
6193       break;
6194    case nir_intrinsic_load_ubo:
6195       visit_load_ubo(ctx, instr);
6196       break;
6197    case nir_intrinsic_load_push_constant:
6198       visit_load_push_constant(ctx, instr);
6199       break;
6200    case nir_intrinsic_load_constant:
6201       visit_load_constant(ctx, instr);
6202       break;
6203    case nir_intrinsic_vulkan_resource_index:
6204       visit_load_resource(ctx, instr);
6205       break;
6206    case nir_intrinsic_discard:
6207       visit_discard(ctx, instr);
6208       break;
6209    case nir_intrinsic_discard_if:
6210       visit_discard_if(ctx, instr);
6211       break;
6212    case nir_intrinsic_load_shared:
6213       visit_load_shared(ctx, instr);
6214       break;
6215    case nir_intrinsic_store_shared:
6216       visit_store_shared(ctx, instr);
6217       break;
6218    case nir_intrinsic_shared_atomic_add:
6219    case nir_intrinsic_shared_atomic_imin:
6220    case nir_intrinsic_shared_atomic_umin:
6221    case nir_intrinsic_shared_atomic_imax:
6222    case nir_intrinsic_shared_atomic_umax:
6223    case nir_intrinsic_shared_atomic_and:
6224    case nir_intrinsic_shared_atomic_or:
6225    case nir_intrinsic_shared_atomic_xor:
6226    case nir_intrinsic_shared_atomic_exchange:
6227    case nir_intrinsic_shared_atomic_comp_swap:
6228       visit_shared_atomic(ctx, instr);
6229       break;
6230    case nir_intrinsic_image_deref_load:
6231       visit_image_load(ctx, instr);
6232       break;
6233    case nir_intrinsic_image_deref_store:
6234       visit_image_store(ctx, instr);
6235       break;
6236    case nir_intrinsic_image_deref_atomic_add:
6237    case nir_intrinsic_image_deref_atomic_umin:
6238    case nir_intrinsic_image_deref_atomic_imin:
6239    case nir_intrinsic_image_deref_atomic_umax:
6240    case nir_intrinsic_image_deref_atomic_imax:
6241    case nir_intrinsic_image_deref_atomic_and:
6242    case nir_intrinsic_image_deref_atomic_or:
6243    case nir_intrinsic_image_deref_atomic_xor:
6244    case nir_intrinsic_image_deref_atomic_exchange:
6245    case nir_intrinsic_image_deref_atomic_comp_swap:
6246       visit_image_atomic(ctx, instr);
6247       break;
6248    case nir_intrinsic_image_deref_size:
6249       visit_image_size(ctx, instr);
6250       break;
6251    case nir_intrinsic_load_ssbo:
6252       visit_load_ssbo(ctx, instr);
6253       break;
6254    case nir_intrinsic_store_ssbo:
6255       visit_store_ssbo(ctx, instr);
6256       break;
6257    case nir_intrinsic_load_global:
6258       visit_load_global(ctx, instr);
6259       break;
6260    case nir_intrinsic_store_global:
6261       visit_store_global(ctx, instr);
6262       break;
6263    case nir_intrinsic_global_atomic_add:
6264    case nir_intrinsic_global_atomic_imin:
6265    case nir_intrinsic_global_atomic_umin:
6266    case nir_intrinsic_global_atomic_imax:
6267    case nir_intrinsic_global_atomic_umax:
6268    case nir_intrinsic_global_atomic_and:
6269    case nir_intrinsic_global_atomic_or:
6270    case nir_intrinsic_global_atomic_xor:
6271    case nir_intrinsic_global_atomic_exchange:
6272    case nir_intrinsic_global_atomic_comp_swap:
6273       visit_global_atomic(ctx, instr);
6274       break;
6275    case nir_intrinsic_ssbo_atomic_add:
6276    case nir_intrinsic_ssbo_atomic_imin:
6277    case nir_intrinsic_ssbo_atomic_umin:
6278    case nir_intrinsic_ssbo_atomic_imax:
6279    case nir_intrinsic_ssbo_atomic_umax:
6280    case nir_intrinsic_ssbo_atomic_and:
6281    case nir_intrinsic_ssbo_atomic_or:
6282    case nir_intrinsic_ssbo_atomic_xor:
6283    case nir_intrinsic_ssbo_atomic_exchange:
6284    case nir_intrinsic_ssbo_atomic_comp_swap:
6285       visit_atomic_ssbo(ctx, instr);
6286       break;
6287    case nir_intrinsic_load_scratch:
6288       visit_load_scratch(ctx, instr);
6289       break;
6290    case nir_intrinsic_store_scratch:
6291       visit_store_scratch(ctx, instr);
6292       break;
6293    case nir_intrinsic_get_buffer_size:
6294       visit_get_buffer_size(ctx, instr);
6295       break;
6296    case nir_intrinsic_control_barrier: {
6297       unsigned* bsize = ctx->program->info->cs.block_size;
6298       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6299       if (workgroup_size > ctx->program->wave_size)
6300          bld.sopp(aco_opcode::s_barrier);
6301       break;
6302    }
6303    case nir_intrinsic_group_memory_barrier:
6304    case nir_intrinsic_memory_barrier:
6305    case nir_intrinsic_memory_barrier_buffer:
6306    case nir_intrinsic_memory_barrier_image:
6307    case nir_intrinsic_memory_barrier_shared:
6308       emit_memory_barrier(ctx, instr);
6309       break;
6310    case nir_intrinsic_memory_barrier_tcs_patch:
6311       break;
6312    case nir_intrinsic_load_num_work_groups: {
6313       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6314       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6315       emit_split_vector(ctx, dst, 3);
6316       break;
6317    }
6318    case nir_intrinsic_load_local_invocation_id: {
6319       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6320       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6321       emit_split_vector(ctx, dst, 3);
6322       break;
6323    }
6324    case nir_intrinsic_load_work_group_id: {
6325       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6326       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6327       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6328                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6329                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6330                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6331       emit_split_vector(ctx, dst, 3);
6332       break;
6333    }
6334    case nir_intrinsic_load_local_invocation_index: {
6335       Temp id = emit_mbcnt(ctx, bld.def(v1));
6336
6337       /* The tg_size bits [6:11] contain the subgroup id,
6338        * we need this multiplied by the wave size, and then OR the thread id to it.
6339        */
6340       if (ctx->program->wave_size == 64) {
6341          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6342          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6343                                 get_arg(ctx, ctx->args->ac.tg_size));
6344          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6345       } else {
6346          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6347          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6348                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6349          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6350       }
6351       break;
6352    }
6353    case nir_intrinsic_load_subgroup_id: {
6354       if (ctx->stage == compute_cs) {
6355          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6356                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6357       } else {
6358          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6359       }
6360       break;
6361    }
6362    case nir_intrinsic_load_subgroup_invocation: {
6363       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6364       break;
6365    }
6366    case nir_intrinsic_load_num_subgroups: {
6367       if (ctx->stage == compute_cs)
6368          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6369                   get_arg(ctx, ctx->args->ac.tg_size));
6370       else
6371          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6372       break;
6373    }
6374    case nir_intrinsic_ballot: {
6375       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6376       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6377       Definition tmp = bld.def(dst.regClass());
6378       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6379       if (instr->src[0].ssa->bit_size == 1) {
6380          assert(src.regClass() == bld.lm);
6381          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6382       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6383          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6384       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6385          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6386       } else {
6387          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6388          nir_print_instr(&instr->instr, stderr);
6389          fprintf(stderr, "\n");
6390       }
6391       if (dst.size() != bld.lm.size()) {
6392          /* Wave32 with ballot size set to 64 */
6393          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6394       }
6395       emit_wqm(ctx, tmp.getTemp(), dst);
6396       break;
6397    }
6398    case nir_intrinsic_shuffle:
6399    case nir_intrinsic_read_invocation: {
6400       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6401       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6402          emit_uniform_subgroup(ctx, instr, src);
6403       } else {
6404          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6405          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6406             tid = bld.as_uniform(tid);
6407          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6408          if (src.regClass() == v1) {
6409             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6410          } else if (src.regClass() == v2) {
6411             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6412             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6413             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6414             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6415             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6416             emit_split_vector(ctx, dst, 2);
6417          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6418             assert(src.regClass() == bld.lm);
6419             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6420             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6421          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6422             assert(src.regClass() == bld.lm);
6423             Temp tmp;
6424             if (ctx->program->chip_class <= GFX7)
6425                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6426             else if (ctx->program->wave_size == 64)
6427                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6428             else
6429                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6430             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6431             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6432             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6433          } else {
6434             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6435             nir_print_instr(&instr->instr, stderr);
6436             fprintf(stderr, "\n");
6437          }
6438       }
6439       break;
6440    }
6441    case nir_intrinsic_load_sample_id: {
6442       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6443                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6444       break;
6445    }
6446    case nir_intrinsic_load_sample_mask_in: {
6447       visit_load_sample_mask_in(ctx, instr);
6448       break;
6449    }
6450    case nir_intrinsic_read_first_invocation: {
6451       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6452       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6453       if (src.regClass() == v1) {
6454          emit_wqm(ctx,
6455                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6456                   dst);
6457       } else if (src.regClass() == v2) {
6458          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6459          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6460          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6461          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6462          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6463          emit_split_vector(ctx, dst, 2);
6464       } else if (instr->dest.ssa.bit_size == 1) {
6465          assert(src.regClass() == bld.lm);
6466          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6467                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6468          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6469       } else if (src.regClass() == s1) {
6470          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6471       } else if (src.regClass() == s2) {
6472          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6473       } else {
6474          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6475          nir_print_instr(&instr->instr, stderr);
6476          fprintf(stderr, "\n");
6477       }
6478       break;
6479    }
6480    case nir_intrinsic_vote_all: {
6481       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6482       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6483       assert(src.regClass() == bld.lm);
6484       assert(dst.regClass() == bld.lm);
6485
6486       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6487       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6488       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6489       break;
6490    }
6491    case nir_intrinsic_vote_any: {
6492       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6493       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6494       assert(src.regClass() == bld.lm);
6495       assert(dst.regClass() == bld.lm);
6496
6497       Temp tmp = bool_to_scalar_condition(ctx, src);
6498       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6499       break;
6500    }
6501    case nir_intrinsic_reduce:
6502    case nir_intrinsic_inclusive_scan:
6503    case nir_intrinsic_exclusive_scan: {
6504       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6505       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6506       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6507       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6508          nir_intrinsic_cluster_size(instr) : 0;
6509       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6510
6511       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6512          emit_uniform_subgroup(ctx, instr, src);
6513       } else if (instr->dest.ssa.bit_size == 1) {
6514          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6515             op = nir_op_iand;
6516          else if (op == nir_op_iadd)
6517             op = nir_op_ixor;
6518          else if (op == nir_op_umax || op == nir_op_imax)
6519             op = nir_op_ior;
6520          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6521
6522          switch (instr->intrinsic) {
6523          case nir_intrinsic_reduce:
6524             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6525             break;
6526          case nir_intrinsic_exclusive_scan:
6527             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6528             break;
6529          case nir_intrinsic_inclusive_scan:
6530             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6531             break;
6532          default:
6533             assert(false);
6534          }
6535       } else if (cluster_size == 1) {
6536          bld.copy(Definition(dst), src);
6537       } else {
6538          src = as_vgpr(ctx, src);
6539
6540          ReduceOp reduce_op;
6541          switch (op) {
6542          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6543             CASE(iadd)
6544             CASE(imul)
6545             CASE(fadd)
6546             CASE(fmul)
6547             CASE(imin)
6548             CASE(umin)
6549             CASE(fmin)
6550             CASE(imax)
6551             CASE(umax)
6552             CASE(fmax)
6553             CASE(iand)
6554             CASE(ior)
6555             CASE(ixor)
6556             default:
6557                unreachable("unknown reduction op");
6558          #undef CASE
6559          }
6560
6561          aco_opcode aco_op;
6562          switch (instr->intrinsic) {
6563             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6564             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6565             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6566             default:
6567                unreachable("unknown reduce intrinsic");
6568          }
6569
6570          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6571          reduce->operands[0] = Operand(src);
6572          // filled in by aco_reduce_assign.cpp, used internally as part of the
6573          // reduce sequence
6574          assert(dst.size() == 1 || dst.size() == 2);
6575          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6576          reduce->operands[2] = Operand(v1.as_linear());
6577
6578          Temp tmp_dst = bld.tmp(dst.regClass());
6579          reduce->definitions[0] = Definition(tmp_dst);
6580          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6581          reduce->definitions[2] = Definition();
6582          reduce->definitions[3] = Definition(scc, s1);
6583          reduce->definitions[4] = Definition();
6584          reduce->reduce_op = reduce_op;
6585          reduce->cluster_size = cluster_size;
6586          ctx->block->instructions.emplace_back(std::move(reduce));
6587
6588          emit_wqm(ctx, tmp_dst, dst);
6589       }
6590       break;
6591    }
6592    case nir_intrinsic_quad_broadcast: {
6593       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6594       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6595          emit_uniform_subgroup(ctx, instr, src);
6596       } else {
6597          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6598          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6599          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6600
6601          if (instr->dest.ssa.bit_size == 1) {
6602             assert(src.regClass() == bld.lm);
6603             assert(dst.regClass() == bld.lm);
6604             uint32_t half_mask = 0x11111111u << lane;
6605             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6606             Temp tmp = bld.tmp(bld.lm);
6607             bld.sop1(Builder::s_wqm, Definition(tmp),
6608                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6609                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6610             emit_wqm(ctx, tmp, dst);
6611          } else if (instr->dest.ssa.bit_size == 32) {
6612             if (ctx->program->chip_class >= GFX8)
6613                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6614             else
6615                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6616          } else if (instr->dest.ssa.bit_size == 64) {
6617             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6618             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6619             if (ctx->program->chip_class >= GFX8) {
6620                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6621                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6622             } else {
6623                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6624                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6625             }
6626             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6627             emit_split_vector(ctx, dst, 2);
6628          } else {
6629             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6630             nir_print_instr(&instr->instr, stderr);
6631             fprintf(stderr, "\n");
6632          }
6633       }
6634       break;
6635    }
6636    case nir_intrinsic_quad_swap_horizontal:
6637    case nir_intrinsic_quad_swap_vertical:
6638    case nir_intrinsic_quad_swap_diagonal:
6639    case nir_intrinsic_quad_swizzle_amd: {
6640       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6641       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6642          emit_uniform_subgroup(ctx, instr, src);
6643          break;
6644       }
6645       uint16_t dpp_ctrl = 0;
6646       switch (instr->intrinsic) {
6647       case nir_intrinsic_quad_swap_horizontal:
6648          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6649          break;
6650       case nir_intrinsic_quad_swap_vertical:
6651          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6652          break;
6653       case nir_intrinsic_quad_swap_diagonal:
6654          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6655          break;
6656       case nir_intrinsic_quad_swizzle_amd:
6657          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6658          break;
6659       default:
6660          break;
6661       }
6662       if (ctx->program->chip_class < GFX8)
6663          dpp_ctrl |= (1 << 15);
6664
6665       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6666       if (instr->dest.ssa.bit_size == 1) {
6667          assert(src.regClass() == bld.lm);
6668          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6669          if (ctx->program->chip_class >= GFX8)
6670             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6671          else
6672             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6673          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6674          emit_wqm(ctx, tmp, dst);
6675       } else if (instr->dest.ssa.bit_size == 32) {
6676          Temp tmp;
6677          if (ctx->program->chip_class >= GFX8)
6678             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6679          else
6680             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6681          emit_wqm(ctx, tmp, dst);
6682       } else if (instr->dest.ssa.bit_size == 64) {
6683          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6684          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6685          if (ctx->program->chip_class >= GFX8) {
6686             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6687             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6688          } else {
6689             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6690             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6691          }
6692          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6693          emit_split_vector(ctx, dst, 2);
6694       } else {
6695          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6696          nir_print_instr(&instr->instr, stderr);
6697          fprintf(stderr, "\n");
6698       }
6699       break;
6700    }
6701    case nir_intrinsic_masked_swizzle_amd: {
6702       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6703       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6704          emit_uniform_subgroup(ctx, instr, src);
6705          break;
6706       }
6707       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6708       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6709       if (dst.regClass() == v1) {
6710          emit_wqm(ctx,
6711                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6712                   dst);
6713       } else if (dst.regClass() == v2) {
6714          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6715          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6716          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6717          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6718          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6719          emit_split_vector(ctx, dst, 2);
6720       } else {
6721          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6722          nir_print_instr(&instr->instr, stderr);
6723          fprintf(stderr, "\n");
6724       }
6725       break;
6726    }
6727    case nir_intrinsic_write_invocation_amd: {
6728       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6729       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6730       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6731       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6732       if (dst.regClass() == v1) {
6733          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6734          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6735       } else if (dst.regClass() == v2) {
6736          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6737          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6738          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6739          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6740          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6741          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6742          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6743          emit_split_vector(ctx, dst, 2);
6744       } else {
6745          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6746          nir_print_instr(&instr->instr, stderr);
6747          fprintf(stderr, "\n");
6748       }
6749       break;
6750    }
6751    case nir_intrinsic_mbcnt_amd: {
6752       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6753       RegClass rc = RegClass(src.type(), 1);
6754       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6755       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6756       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6757       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6758       emit_wqm(ctx, wqm_tmp, dst);
6759       break;
6760    }
6761    case nir_intrinsic_load_helper_invocation: {
6762       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6763       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6764       ctx->block->kind |= block_kind_needs_lowering;
6765       ctx->program->needs_exact = true;
6766       break;
6767    }
6768    case nir_intrinsic_is_helper_invocation: {
6769       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6770       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6771       ctx->block->kind |= block_kind_needs_lowering;
6772       ctx->program->needs_exact = true;
6773       break;
6774    }
6775    case nir_intrinsic_demote:
6776       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6777
6778       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6779          ctx->cf_info.exec_potentially_empty_discard = true;
6780       ctx->block->kind |= block_kind_uses_demote;
6781       ctx->program->needs_exact = true;
6782       break;
6783    case nir_intrinsic_demote_if: {
6784       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6785       assert(src.regClass() == bld.lm);
6786       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6787       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6788
6789       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6790          ctx->cf_info.exec_potentially_empty_discard = true;
6791       ctx->block->kind |= block_kind_uses_demote;
6792       ctx->program->needs_exact = true;
6793       break;
6794    }
6795    case nir_intrinsic_first_invocation: {
6796       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6797                get_ssa_temp(ctx, &instr->dest.ssa));
6798       break;
6799    }
6800    case nir_intrinsic_shader_clock:
6801       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6802       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6803       break;
6804    case nir_intrinsic_load_vertex_id_zero_base: {
6805       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6806       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6807       break;
6808    }
6809    case nir_intrinsic_load_first_vertex: {
6810       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6811       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6812       break;
6813    }
6814    case nir_intrinsic_load_base_instance: {
6815       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6816       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6817       break;
6818    }
6819    case nir_intrinsic_load_instance_id: {
6820       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6821       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6822       break;
6823    }
6824    case nir_intrinsic_load_draw_id: {
6825       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6826       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6827       break;
6828    }
6829    case nir_intrinsic_load_invocation_id: {
6830       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6831       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6832       if (ctx->options->chip_class >= GFX10)
6833          bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6834       else
6835          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6836       break;
6837    }
6838    case nir_intrinsic_load_primitive_id: {
6839       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6840       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6841       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
6842       break;
6843    }
6844    case nir_intrinsic_emit_vertex_with_counter: {
6845       visit_emit_vertex_with_counter(ctx, instr);
6846       break;
6847    }
6848    case nir_intrinsic_end_primitive_with_counter: {
6849       unsigned stream = nir_intrinsic_stream_id(instr);
6850       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
6851       break;
6852    }
6853    case nir_intrinsic_set_vertex_count: {
6854       /* unused, the HW keeps track of this for us */
6855       break;
6856    }
6857    default:
6858       fprintf(stderr, "Unimplemented intrinsic instr: ");
6859       nir_print_instr(&instr->instr, stderr);
6860       fprintf(stderr, "\n");
6861       abort();
6862
6863       break;
6864    }
6865 }
6866
6867
6868 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6869                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6870                     enum glsl_base_type *stype)
6871 {
6872    nir_deref_instr *texture_deref_instr = NULL;
6873    nir_deref_instr *sampler_deref_instr = NULL;
6874    int plane = -1;
6875
6876    for (unsigned i = 0; i < instr->num_srcs; i++) {
6877       switch (instr->src[i].src_type) {
6878       case nir_tex_src_texture_deref:
6879          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6880          break;
6881       case nir_tex_src_sampler_deref:
6882          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6883          break;
6884       case nir_tex_src_plane:
6885          plane = nir_src_as_int(instr->src[i].src);
6886          break;
6887       default:
6888          break;
6889       }
6890    }
6891
6892    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6893
6894    if (!sampler_deref_instr)
6895       sampler_deref_instr = texture_deref_instr;
6896
6897    if (plane >= 0) {
6898       assert(instr->op != nir_texop_txf_ms &&
6899              instr->op != nir_texop_samples_identical);
6900       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6901       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6902    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6903       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6904    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6905       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6906    } else {
6907       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6908    }
6909    if (samp_ptr) {
6910       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6911
6912       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6913          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6914          Builder bld(ctx->program, ctx->block);
6915
6916          /* to avoid unnecessary moves, we split and recombine sampler and image */
6917          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6918                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6919          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6920          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6921                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6922                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6923          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6924                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6925
6926          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6927          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6928                                img[0], img[1], img[2], img[3],
6929                                img[4], img[5], img[6], img[7]);
6930          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6931                                 samp[0], samp[1], samp[2], samp[3]);
6932       }
6933    }
6934    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6935                      instr->op == nir_texop_samples_identical))
6936       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6937 }
6938
6939 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6940                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6941 {
6942    Builder bld(ctx->program, ctx->block);
6943
6944    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6945    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6946    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6947
6948    Operand neg_one(0xbf800000u);
6949    Operand one(0x3f800000u);
6950    Operand two(0x40000000u);
6951    Operand four(0x40800000u);
6952
6953    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6954    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6955    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6956
6957    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6958    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6959    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6960    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6961
6962    // select sc
6963    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6964    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6965                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6966                        one, is_ma_y);
6967    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6968
6969    // select tc
6970    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6971    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6972    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6973
6974    // select ma
6975    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6976                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6977                   deriv_z, is_ma_z);
6978    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6979    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6980 }
6981
6982 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6983 {
6984    Builder bld(ctx->program, ctx->block);
6985    Temp ma, tc, sc, id;
6986
6987    if (is_array) {
6988       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
6989
6990       // see comment in ac_prepare_cube_coords()
6991       if (ctx->options->chip_class <= GFX8)
6992          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
6993    }
6994
6995    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
6996
6997    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6998    vop3a->operands[0] = Operand(ma);
6999    vop3a->abs[0] = true;
7000    Temp invma = bld.tmp(v1);
7001    vop3a->definitions[0] = Definition(invma);
7002    ctx->block->instructions.emplace_back(std::move(vop3a));
7003
7004    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7005    if (!is_deriv)
7006       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
7007
7008    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7009    if (!is_deriv)
7010       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
7011
7012    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7013
7014    if (is_deriv) {
7015       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
7016       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
7017
7018       for (unsigned i = 0; i < 2; i++) {
7019          // see comment in ac_prepare_cube_coords()
7020          Temp deriv_ma;
7021          Temp deriv_sc, deriv_tc;
7022          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
7023                            &deriv_ma, &deriv_sc, &deriv_tc);
7024
7025          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
7026
7027          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7028                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
7029                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
7030          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7031                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
7032                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
7033          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
7034       }
7035
7036       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
7037       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
7038    }
7039
7040    if (is_array)
7041       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
7042    coords.resize(3);
7043    coords[0] = sc;
7044    coords[1] = tc;
7045    coords[2] = id;
7046 }
7047
7048 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7049 {
7050    if (vec->parent_instr->type != nir_instr_type_alu)
7051       return;
7052    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7053    if (vec_instr->op != nir_op_vec(vec->num_components))
7054       return;
7055
7056    for (unsigned i = 0; i < vec->num_components; i++) {
7057       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7058               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7059    }
7060 }
7061
7062 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7063 {
7064    Builder bld(ctx->program, ctx->block);
7065    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7066         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7067    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
7068         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp();
7069    std::vector<Temp> coords;
7070    std::vector<Temp> derivs;
7071    nir_const_value *sample_index_cv = NULL;
7072    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7073    enum glsl_base_type stype;
7074    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7075
7076    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7077                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7078    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7079                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7080
7081    for (unsigned i = 0; i < instr->num_srcs; i++) {
7082       switch (instr->src[i].src_type) {
7083       case nir_tex_src_coord: {
7084          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
7085          for (unsigned i = 0; i < coord.size(); i++)
7086             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
7087          break;
7088       }
7089       case nir_tex_src_bias:
7090          if (instr->op == nir_texop_txb) {
7091             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7092             has_bias = true;
7093          }
7094          break;
7095       case nir_tex_src_lod: {
7096          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7097
7098          if (val && val->f32 <= 0.0) {
7099             level_zero = true;
7100          } else {
7101             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7102             has_lod = true;
7103          }
7104          break;
7105       }
7106       case nir_tex_src_comparator:
7107          if (instr->is_shadow) {
7108             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7109             has_compare = true;
7110          }
7111          break;
7112       case nir_tex_src_offset:
7113          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7114          get_const_vec(instr->src[i].src.ssa, const_offset);
7115          has_offset = true;
7116          break;
7117       case nir_tex_src_ddx:
7118          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7119          has_ddx = true;
7120          break;
7121       case nir_tex_src_ddy:
7122          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7123          has_ddy = true;
7124          break;
7125       case nir_tex_src_ms_index:
7126          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7127          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7128          has_sample_index = true;
7129          break;
7130       case nir_tex_src_texture_offset:
7131       case nir_tex_src_sampler_offset:
7132       default:
7133          break;
7134       }
7135    }
7136
7137    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7138       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7139
7140    if (instr->op == nir_texop_texture_samples) {
7141       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7142
7143       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7144       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7145       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7146       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7147
7148       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7149                samples, Operand(1u), bld.scc(is_msaa));
7150       return;
7151    }
7152
7153    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7154       aco_ptr<Instruction> tmp_instr;
7155       Temp acc, pack = Temp();
7156
7157       uint32_t pack_const = 0;
7158       for (unsigned i = 0; i < offset.size(); i++) {
7159          if (!const_offset[i])
7160             continue;
7161          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7162       }
7163
7164       if (offset.type() == RegType::sgpr) {
7165          for (unsigned i = 0; i < offset.size(); i++) {
7166             if (const_offset[i])
7167                continue;
7168
7169             acc = emit_extract_vector(ctx, offset, i, s1);
7170             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7171
7172             if (i) {
7173                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7174             }
7175
7176             if (pack == Temp()) {
7177                pack = acc;
7178             } else {
7179                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7180             }
7181          }
7182
7183          if (pack_const && pack != Temp())
7184             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7185       } else {
7186          for (unsigned i = 0; i < offset.size(); i++) {
7187             if (const_offset[i])
7188                continue;
7189
7190             acc = emit_extract_vector(ctx, offset, i, v1);
7191             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7192
7193             if (i) {
7194                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7195             }
7196
7197             if (pack == Temp()) {
7198                pack = acc;
7199             } else {
7200                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7201             }
7202          }
7203
7204          if (pack_const && pack != Temp())
7205             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7206       }
7207       if (pack_const && pack == Temp())
7208          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7209       else if (pack == Temp())
7210          has_offset = false;
7211       else
7212          offset = pack;
7213    }
7214
7215    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7216       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7217
7218    /* pack derivatives */
7219    if (has_ddx || has_ddy) {
7220       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7221          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
7222          Temp zero = bld.copy(bld.def(v1), Operand(0u));
7223          derivs = {ddy, zero, ddy, zero};
7224       } else {
7225          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
7226             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
7227          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
7228             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
7229       }
7230       has_derivs = true;
7231    }
7232
7233    if (instr->coord_components > 1 &&
7234        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7235        instr->is_array &&
7236        instr->op != nir_texop_txf)
7237       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
7238
7239    if (instr->coord_components > 2 &&
7240       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7241        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7242        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7243        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7244        instr->is_array &&
7245        instr->op != nir_texop_txf &&
7246        instr->op != nir_texop_txf_ms &&
7247        instr->op != nir_texop_fragment_fetch &&
7248        instr->op != nir_texop_fragment_mask_fetch)
7249       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
7250
7251    if (ctx->options->chip_class == GFX9 &&
7252        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7253        instr->op != nir_texop_lod && instr->coord_components) {
7254       assert(coords.size() > 0 && coords.size() < 3);
7255
7256       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
7257                                                                      Operand((uint32_t) 0) :
7258                                                                      Operand((uint32_t) 0x3f000000)));
7259    }
7260
7261    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7262
7263    if (instr->op == nir_texop_samples_identical)
7264       resource = fmask_ptr;
7265
7266    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7267              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7268             instr->op != nir_texop_txs &&
7269             instr->op != nir_texop_fragment_fetch &&
7270             instr->op != nir_texop_fragment_mask_fetch) {
7271       assert(has_sample_index);
7272       Operand op(sample_index);
7273       if (sample_index_cv)
7274          op = Operand(sample_index_cv->u32);
7275       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7276    }
7277
7278    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7279       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
7280          Temp off = emit_extract_vector(ctx, offset, i, v1);
7281          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
7282       }
7283       has_offset = false;
7284    }
7285
7286    /* Build tex instruction */
7287    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7288    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7289                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7290                   : 0;
7291    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7292    Temp tmp_dst = dst;
7293
7294    /* gather4 selects the component by dmask and always returns vec4 */
7295    if (instr->op == nir_texop_tg4) {
7296       assert(instr->dest.ssa.num_components == 4);
7297       if (instr->is_shadow)
7298          dmask = 1;
7299       else
7300          dmask = 1 << instr->component;
7301       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7302          tmp_dst = bld.tmp(v4);
7303    } else if (instr->op == nir_texop_samples_identical) {
7304       tmp_dst = bld.tmp(v1);
7305    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7306       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7307    }
7308
7309    aco_ptr<MIMG_instruction> tex;
7310    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7311       if (!has_lod)
7312          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7313
7314       bool div_by_6 = instr->op == nir_texop_txs &&
7315                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7316                       instr->is_array &&
7317                       (dmask & (1 << 2));
7318       if (tmp_dst.id() == dst.id() && div_by_6)
7319          tmp_dst = bld.tmp(tmp_dst.regClass());
7320
7321       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7322       tex->operands[0] = Operand(resource);
7323       tex->operands[1] = Operand(s4); /* no sampler */
7324       tex->operands[2] = Operand(as_vgpr(ctx,lod));
7325       if (ctx->options->chip_class == GFX9 &&
7326           instr->op == nir_texop_txs &&
7327           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7328           instr->is_array) {
7329          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7330       } else if (instr->op == nir_texop_query_levels) {
7331          tex->dmask = 1 << 3;
7332       } else {
7333          tex->dmask = dmask;
7334       }
7335       tex->da = da;
7336       tex->definitions[0] = Definition(tmp_dst);
7337       tex->dim = dim;
7338       tex->can_reorder = true;
7339       ctx->block->instructions.emplace_back(std::move(tex));
7340
7341       if (div_by_6) {
7342          /* divide 3rd value by 6 by multiplying with magic number */
7343          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7344          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7345          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7346          assert(instr->dest.ssa.num_components == 3);
7347          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7348          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7349                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7350                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7351                               by_6);
7352
7353       }
7354
7355       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7356       return;
7357    }
7358
7359    Temp tg4_compare_cube_wa64 = Temp();
7360
7361    if (tg4_integer_workarounds) {
7362       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7363       tex->operands[0] = Operand(resource);
7364       tex->operands[1] = Operand(s4); /* no sampler */
7365       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7366       tex->dim = dim;
7367       tex->dmask = 0x3;
7368       tex->da = da;
7369       Temp size = bld.tmp(v2);
7370       tex->definitions[0] = Definition(size);
7371       tex->can_reorder = true;
7372       ctx->block->instructions.emplace_back(std::move(tex));
7373       emit_split_vector(ctx, size, size.size());
7374
7375       Temp half_texel[2];
7376       for (unsigned i = 0; i < 2; i++) {
7377          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7378          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7379          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7380          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7381       }
7382
7383       Temp new_coords[2] = {
7384          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
7385          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
7386       };
7387
7388       if (tg4_integer_cube_workaround) {
7389          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7390          Temp desc[resource.size()];
7391          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7392                                                                            Format::PSEUDO, 1, resource.size())};
7393          split->operands[0] = Operand(resource);
7394          for (unsigned i = 0; i < resource.size(); i++) {
7395             desc[i] = bld.tmp(s1);
7396             split->definitions[i] = Definition(desc[i]);
7397          }
7398          ctx->block->instructions.emplace_back(std::move(split));
7399
7400          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7401          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7402                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7403
7404          Temp nfmt;
7405          if (stype == GLSL_TYPE_UINT) {
7406             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7407                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7408                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7409                             bld.scc(compare_cube_wa));
7410          } else {
7411             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7412                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7413                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7414                             bld.scc(compare_cube_wa));
7415          }
7416          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7417          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7418
7419          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7420
7421          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7422                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7423          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7424
7425          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7426                                                                          Format::PSEUDO, resource.size(), 1)};
7427          for (unsigned i = 0; i < resource.size(); i++)
7428             vec->operands[i] = Operand(desc[i]);
7429          resource = bld.tmp(resource.regClass());
7430          vec->definitions[0] = Definition(resource);
7431          ctx->block->instructions.emplace_back(std::move(vec));
7432
7433          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7434                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
7435          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7436                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
7437       }
7438       coords[0] = new_coords[0];
7439       coords[1] = new_coords[1];
7440    }
7441
7442    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7443       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7444
7445       assert(coords.size() == 1);
7446       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7447       aco_opcode op;
7448       switch (last_bit) {
7449       case 1:
7450          op = aco_opcode::buffer_load_format_x; break;
7451       case 2:
7452          op = aco_opcode::buffer_load_format_xy; break;
7453       case 3:
7454          op = aco_opcode::buffer_load_format_xyz; break;
7455       case 4:
7456          op = aco_opcode::buffer_load_format_xyzw; break;
7457       default:
7458          unreachable("Tex instruction loads more than 4 components.");
7459       }
7460
7461       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7462       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7463          tmp_dst = dst;
7464       else
7465          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7466
7467       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7468       mubuf->operands[0] = Operand(resource);
7469       mubuf->operands[1] = Operand(coords[0]);
7470       mubuf->operands[2] = Operand((uint32_t) 0);
7471       mubuf->definitions[0] = Definition(tmp_dst);
7472       mubuf->idxen = true;
7473       mubuf->can_reorder = true;
7474       ctx->block->instructions.emplace_back(std::move(mubuf));
7475
7476       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7477       return;
7478    }
7479
7480    /* gather MIMG address components */
7481    std::vector<Temp> args;
7482    if (has_offset)
7483       args.emplace_back(offset);
7484    if (has_bias)
7485       args.emplace_back(bias);
7486    if (has_compare)
7487       args.emplace_back(compare);
7488    if (has_derivs)
7489       args.insert(args.end(), derivs.begin(), derivs.end());
7490
7491    args.insert(args.end(), coords.begin(), coords.end());
7492    if (has_sample_index)
7493       args.emplace_back(sample_index);
7494    if (has_lod)
7495       args.emplace_back(lod);
7496
7497    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
7498    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7499    vec->definitions[0] = Definition(arg);
7500    for (unsigned i = 0; i < args.size(); i++)
7501       vec->operands[i] = Operand(args[i]);
7502    ctx->block->instructions.emplace_back(std::move(vec));
7503
7504
7505    if (instr->op == nir_texop_txf ||
7506        instr->op == nir_texop_txf_ms ||
7507        instr->op == nir_texop_samples_identical ||
7508        instr->op == nir_texop_fragment_fetch ||
7509        instr->op == nir_texop_fragment_mask_fetch) {
7510       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7511       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
7512       tex->operands[0] = Operand(resource);
7513       tex->operands[1] = Operand(s4); /* no sampler */
7514       tex->operands[2] = Operand(arg);
7515       tex->dim = dim;
7516       tex->dmask = dmask;
7517       tex->unrm = true;
7518       tex->da = da;
7519       tex->definitions[0] = Definition(tmp_dst);
7520       tex->can_reorder = true;
7521       ctx->block->instructions.emplace_back(std::move(tex));
7522
7523       if (instr->op == nir_texop_samples_identical) {
7524          assert(dmask == 1 && dst.regClass() == v1);
7525          assert(dst.id() != tmp_dst.id());
7526
7527          Temp tmp = bld.tmp(bld.lm);
7528          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7529          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7530
7531       } else {
7532          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7533       }
7534       return;
7535    }
7536
7537    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7538    aco_opcode opcode = aco_opcode::image_sample;
7539    if (has_offset) { /* image_sample_*_o */
7540       if (has_compare) {
7541          opcode = aco_opcode::image_sample_c_o;
7542          if (has_derivs)
7543             opcode = aco_opcode::image_sample_c_d_o;
7544          if (has_bias)
7545             opcode = aco_opcode::image_sample_c_b_o;
7546          if (level_zero)
7547             opcode = aco_opcode::image_sample_c_lz_o;
7548          if (has_lod)
7549             opcode = aco_opcode::image_sample_c_l_o;
7550       } else {
7551          opcode = aco_opcode::image_sample_o;
7552          if (has_derivs)
7553             opcode = aco_opcode::image_sample_d_o;
7554          if (has_bias)
7555             opcode = aco_opcode::image_sample_b_o;
7556          if (level_zero)
7557             opcode = aco_opcode::image_sample_lz_o;
7558          if (has_lod)
7559             opcode = aco_opcode::image_sample_l_o;
7560       }
7561    } else { /* no offset */
7562       if (has_compare) {
7563          opcode = aco_opcode::image_sample_c;
7564          if (has_derivs)
7565             opcode = aco_opcode::image_sample_c_d;
7566          if (has_bias)
7567             opcode = aco_opcode::image_sample_c_b;
7568          if (level_zero)
7569             opcode = aco_opcode::image_sample_c_lz;
7570          if (has_lod)
7571             opcode = aco_opcode::image_sample_c_l;
7572       } else {
7573          opcode = aco_opcode::image_sample;
7574          if (has_derivs)
7575             opcode = aco_opcode::image_sample_d;
7576          if (has_bias)
7577             opcode = aco_opcode::image_sample_b;
7578          if (level_zero)
7579             opcode = aco_opcode::image_sample_lz;
7580          if (has_lod)
7581             opcode = aco_opcode::image_sample_l;
7582       }
7583    }
7584
7585    if (instr->op == nir_texop_tg4) {
7586       if (has_offset) {
7587          opcode = aco_opcode::image_gather4_lz_o;
7588          if (has_compare)
7589             opcode = aco_opcode::image_gather4_c_lz_o;
7590       } else {
7591          opcode = aco_opcode::image_gather4_lz;
7592          if (has_compare)
7593             opcode = aco_opcode::image_gather4_c_lz;
7594       }
7595    } else if (instr->op == nir_texop_lod) {
7596       opcode = aco_opcode::image_get_lod;
7597    }
7598
7599    /* we don't need the bias, sample index, compare value or offset to be
7600     * computed in WQM but if the p_create_vector copies the coordinates, then it
7601     * needs to be in WQM */
7602    if (ctx->stage == fragment_fs &&
7603        !has_derivs && !has_lod && !level_zero &&
7604        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7605        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7606       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7607
7608    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7609    tex->operands[0] = Operand(resource);
7610    tex->operands[1] = Operand(sampler);
7611    tex->operands[2] = Operand(arg);
7612    tex->dim = dim;
7613    tex->dmask = dmask;
7614    tex->da = da;
7615    tex->definitions[0] = Definition(tmp_dst);
7616    tex->can_reorder = true;
7617    ctx->block->instructions.emplace_back(std::move(tex));
7618
7619    if (tg4_integer_cube_workaround) {
7620       assert(tmp_dst.id() != dst.id());
7621       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7622
7623       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7624       Temp val[4];
7625       for (unsigned i = 0; i < dst.size(); i++) {
7626          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7627          Temp cvt_val;
7628          if (stype == GLSL_TYPE_UINT)
7629             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7630          else
7631             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7632          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7633       }
7634       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7635       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7636                            val[0], val[1], val[2], val[3]);
7637    }
7638    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7639    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7640
7641 }
7642
7643
7644 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7645 {
7646    Temp tmp = get_ssa_temp(ctx, ssa);
7647    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7648       return Operand(tmp.regClass());
7649    else
7650       return Operand(tmp);
7651 }
7652
7653 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7654 {
7655    aco_ptr<Pseudo_instruction> phi;
7656    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7657    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7658
7659    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7660    logical |= ctx->block->kind & block_kind_merge;
7661    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7662
7663    /* we want a sorted list of sources, since the predecessor list is also sorted */
7664    std::map<unsigned, nir_ssa_def*> phi_src;
7665    nir_foreach_phi_src(src, instr)
7666       phi_src[src->pred->index] = src->src.ssa;
7667
7668    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7669    unsigned num_operands = 0;
7670    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7671    unsigned num_defined = 0;
7672    unsigned cur_pred_idx = 0;
7673    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7674       if (cur_pred_idx < preds.size()) {
7675          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7676          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7677          unsigned skipped = 0;
7678          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7679             skipped++;
7680          if (cur_pred_idx + skipped < preds.size()) {
7681             for (unsigned i = 0; i < skipped; i++)
7682                operands[num_operands++] = Operand(dst.regClass());
7683             cur_pred_idx += skipped;
7684          } else {
7685             continue;
7686          }
7687       }
7688       cur_pred_idx++;
7689       Operand op = get_phi_operand(ctx, src.second);
7690       operands[num_operands++] = op;
7691       num_defined += !op.isUndefined();
7692    }
7693    /* handle block_kind_continue_or_break at loop exit blocks */
7694    while (cur_pred_idx++ < preds.size())
7695       operands[num_operands++] = Operand(dst.regClass());
7696
7697    if (num_defined == 0) {
7698       Builder bld(ctx->program, ctx->block);
7699       if (dst.regClass() == s1) {
7700          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7701       } else if (dst.regClass() == v1) {
7702          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7703       } else {
7704          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7705          for (unsigned i = 0; i < dst.size(); i++)
7706             vec->operands[i] = Operand(0u);
7707          vec->definitions[0] = Definition(dst);
7708          ctx->block->instructions.emplace_back(std::move(vec));
7709       }
7710       return;
7711    }
7712
7713    /* we can use a linear phi in some cases if one src is undef */
7714    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7715       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7716
7717       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7718       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7719       assert(invert->kind & block_kind_invert);
7720
7721       unsigned then_block = invert->linear_preds[0];
7722
7723       Block* insert_block = NULL;
7724       for (unsigned i = 0; i < num_operands; i++) {
7725          Operand op = operands[i];
7726          if (op.isUndefined())
7727             continue;
7728          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7729          phi->operands[0] = op;
7730          break;
7731       }
7732       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7733       phi->operands[1] = Operand(dst.regClass());
7734       phi->definitions[0] = Definition(dst);
7735       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7736       return;
7737    }
7738
7739    /* try to scalarize vector phis */
7740    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7741       // TODO: scalarize linear phis on divergent ifs
7742       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7743       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7744       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7745          Operand src = operands[i];
7746          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7747             can_scalarize = false;
7748       }
7749       if (can_scalarize) {
7750          unsigned num_components = instr->dest.ssa.num_components;
7751          assert(dst.size() % num_components == 0);
7752          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7753
7754          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7755          for (unsigned k = 0; k < num_components; k++) {
7756             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7757             for (unsigned i = 0; i < num_operands; i++) {
7758                Operand src = operands[i];
7759                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7760             }
7761             Temp phi_dst = {ctx->program->allocateId(), rc};
7762             phi->definitions[0] = Definition(phi_dst);
7763             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7764             new_vec[k] = phi_dst;
7765             vec->operands[k] = Operand(phi_dst);
7766          }
7767          vec->definitions[0] = Definition(dst);
7768          ctx->block->instructions.emplace_back(std::move(vec));
7769          ctx->allocated_vec.emplace(dst.id(), new_vec);
7770          return;
7771       }
7772    }
7773
7774    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7775    for (unsigned i = 0; i < num_operands; i++)
7776       phi->operands[i] = operands[i];
7777    phi->definitions[0] = Definition(dst);
7778    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7779 }
7780
7781
7782 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7783 {
7784    Temp dst = get_ssa_temp(ctx, &instr->def);
7785
7786    assert(dst.type() == RegType::sgpr);
7787
7788    if (dst.size() == 1) {
7789       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7790    } else {
7791       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7792       for (unsigned i = 0; i < dst.size(); i++)
7793          vec->operands[i] = Operand(0u);
7794       vec->definitions[0] = Definition(dst);
7795       ctx->block->instructions.emplace_back(std::move(vec));
7796    }
7797 }
7798
7799 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7800 {
7801    Builder bld(ctx->program, ctx->block);
7802    Block *logical_target;
7803    append_logical_end(ctx->block);
7804    unsigned idx = ctx->block->index;
7805
7806    switch (instr->type) {
7807    case nir_jump_break:
7808       logical_target = ctx->cf_info.parent_loop.exit;
7809       add_logical_edge(idx, logical_target);
7810       ctx->block->kind |= block_kind_break;
7811
7812       if (!ctx->cf_info.parent_if.is_divergent &&
7813           !ctx->cf_info.parent_loop.has_divergent_continue) {
7814          /* uniform break - directly jump out of the loop */
7815          ctx->block->kind |= block_kind_uniform;
7816          ctx->cf_info.has_branch = true;
7817          bld.branch(aco_opcode::p_branch);
7818          add_linear_edge(idx, logical_target);
7819          return;
7820       }
7821       ctx->cf_info.parent_loop.has_divergent_branch = true;
7822       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7823       break;
7824    case nir_jump_continue:
7825       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7826       add_logical_edge(idx, logical_target);
7827       ctx->block->kind |= block_kind_continue;
7828
7829       if (ctx->cf_info.parent_if.is_divergent) {
7830          /* for potential uniform breaks after this continue,
7831             we must ensure that they are handled correctly */
7832          ctx->cf_info.parent_loop.has_divergent_continue = true;
7833          ctx->cf_info.parent_loop.has_divergent_branch = true;
7834          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7835       } else {
7836          /* uniform continue - directly jump to the loop header */
7837          ctx->block->kind |= block_kind_uniform;
7838          ctx->cf_info.has_branch = true;
7839          bld.branch(aco_opcode::p_branch);
7840          add_linear_edge(idx, logical_target);
7841          return;
7842       }
7843       break;
7844    default:
7845       fprintf(stderr, "Unknown NIR jump instr: ");
7846       nir_print_instr(&instr->instr, stderr);
7847       fprintf(stderr, "\n");
7848       abort();
7849    }
7850
7851    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
7852       ctx->cf_info.exec_potentially_empty_break = true;
7853       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
7854    }
7855
7856    /* remove critical edges from linear CFG */
7857    bld.branch(aco_opcode::p_branch);
7858    Block* break_block = ctx->program->create_and_insert_block();
7859    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7860    break_block->kind |= block_kind_uniform;
7861    add_linear_edge(idx, break_block);
7862    /* the loop_header pointer might be invalidated by this point */
7863    if (instr->type == nir_jump_continue)
7864       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7865    add_linear_edge(break_block->index, logical_target);
7866    bld.reset(break_block);
7867    bld.branch(aco_opcode::p_branch);
7868
7869    Block* continue_block = ctx->program->create_and_insert_block();
7870    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7871    add_linear_edge(idx, continue_block);
7872    append_logical_start(continue_block);
7873    ctx->block = continue_block;
7874    return;
7875 }
7876
7877 void visit_block(isel_context *ctx, nir_block *block)
7878 {
7879    nir_foreach_instr(instr, block) {
7880       switch (instr->type) {
7881       case nir_instr_type_alu:
7882          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7883          break;
7884       case nir_instr_type_load_const:
7885          visit_load_const(ctx, nir_instr_as_load_const(instr));
7886          break;
7887       case nir_instr_type_intrinsic:
7888          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7889          break;
7890       case nir_instr_type_tex:
7891          visit_tex(ctx, nir_instr_as_tex(instr));
7892          break;
7893       case nir_instr_type_phi:
7894          visit_phi(ctx, nir_instr_as_phi(instr));
7895          break;
7896       case nir_instr_type_ssa_undef:
7897          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7898          break;
7899       case nir_instr_type_deref:
7900          break;
7901       case nir_instr_type_jump:
7902          visit_jump(ctx, nir_instr_as_jump(instr));
7903          break;
7904       default:
7905          fprintf(stderr, "Unknown NIR instr type: ");
7906          nir_print_instr(instr, stderr);
7907          fprintf(stderr, "\n");
7908          //abort();
7909       }
7910    }
7911
7912    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7913       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7914 }
7915
7916
7917
7918 static void visit_loop(isel_context *ctx, nir_loop *loop)
7919 {
7920    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
7921    append_logical_end(ctx->block);
7922    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7923    Builder bld(ctx->program, ctx->block);
7924    bld.branch(aco_opcode::p_branch);
7925    unsigned loop_preheader_idx = ctx->block->index;
7926
7927    Block loop_exit = Block();
7928    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7929    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7930
7931    Block* loop_header = ctx->program->create_and_insert_block();
7932    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7933    loop_header->kind |= block_kind_loop_header;
7934    add_edge(loop_preheader_idx, loop_header);
7935    ctx->block = loop_header;
7936
7937    /* emit loop body */
7938    unsigned loop_header_idx = loop_header->index;
7939    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7940    append_logical_start(ctx->block);
7941    visit_cf_list(ctx, &loop->body);
7942
7943    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7944    if (!ctx->cf_info.has_branch) {
7945       append_logical_end(ctx->block);
7946       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
7947          /* Discards can result in code running with an empty exec mask.
7948           * This would result in divergent breaks not ever being taken. As a
7949           * workaround, break the loop when the loop mask is empty instead of
7950           * always continuing. */
7951          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7952          unsigned block_idx = ctx->block->index;
7953
7954          /* create helper blocks to avoid critical edges */
7955          Block *break_block = ctx->program->create_and_insert_block();
7956          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7957          break_block->kind = block_kind_uniform;
7958          bld.reset(break_block);
7959          bld.branch(aco_opcode::p_branch);
7960          add_linear_edge(block_idx, break_block);
7961          add_linear_edge(break_block->index, &loop_exit);
7962
7963          Block *continue_block = ctx->program->create_and_insert_block();
7964          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7965          continue_block->kind = block_kind_uniform;
7966          bld.reset(continue_block);
7967          bld.branch(aco_opcode::p_branch);
7968          add_linear_edge(block_idx, continue_block);
7969          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7970
7971          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7972             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7973          ctx->block = &ctx->program->blocks[block_idx];
7974       } else {
7975          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7976          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7977             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7978          else
7979             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7980       }
7981
7982       bld.reset(ctx->block);
7983       bld.branch(aco_opcode::p_branch);
7984    }
7985
7986    /* fixup phis in loop header from unreachable blocks */
7987    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7988       bool linear = ctx->cf_info.has_branch;
7989       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7990       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7991          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7992              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7993             /* the last operand should be the one that needs to be removed */
7994             instr->operands.pop_back();
7995          } else if (!is_phi(instr)) {
7996             break;
7997          }
7998       }
7999    }
8000
8001    ctx->cf_info.has_branch = false;
8002
8003    // TODO: if the loop has not a single exit, we must add one °°
8004    /* emit loop successor block */
8005    ctx->block = ctx->program->insert_block(std::move(loop_exit));
8006    append_logical_start(ctx->block);
8007
8008    #if 0
8009    // TODO: check if it is beneficial to not branch on continues
8010    /* trim linear phis in loop header */
8011    for (auto&& instr : loop_entry->instructions) {
8012       if (instr->opcode == aco_opcode::p_linear_phi) {
8013          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8014          new_phi->definitions[0] = instr->definitions[0];
8015          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8016             new_phi->operands[i] = instr->operands[i];
8017          /* check that the remaining operands are all the same */
8018          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8019             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8020          instr.swap(new_phi);
8021       } else if (instr->opcode == aco_opcode::p_phi) {
8022          continue;
8023       } else {
8024          break;
8025       }
8026    }
8027    #endif
8028 }
8029
8030 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8031 {
8032    ic->cond = cond;
8033
8034    append_logical_end(ctx->block);
8035    ctx->block->kind |= block_kind_branch;
8036
8037    /* branch to linear then block */
8038    assert(cond.regClass() == ctx->program->lane_mask);
8039    aco_ptr<Pseudo_branch_instruction> branch;
8040    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8041    branch->operands[0] = Operand(cond);
8042    ctx->block->instructions.push_back(std::move(branch));
8043
8044    ic->BB_if_idx = ctx->block->index;
8045    ic->BB_invert = Block();
8046    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8047    /* Invert blocks are intentionally not marked as top level because they
8048     * are not part of the logical cfg. */
8049    ic->BB_invert.kind |= block_kind_invert;
8050    ic->BB_endif = Block();
8051    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8052    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8053
8054    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
8055    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
8056    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
8057    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8058    ctx->cf_info.parent_if.is_divergent = true;
8059
8060    /* divergent branches use cbranch_execz */
8061    ctx->cf_info.exec_potentially_empty_discard = false;
8062    ctx->cf_info.exec_potentially_empty_break = false;
8063    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8064
8065    /** emit logical then block */
8066    Block* BB_then_logical = ctx->program->create_and_insert_block();
8067    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8068    add_edge(ic->BB_if_idx, BB_then_logical);
8069    ctx->block = BB_then_logical;
8070    append_logical_start(BB_then_logical);
8071 }
8072
8073 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8074 {
8075    Block *BB_then_logical = ctx->block;
8076    append_logical_end(BB_then_logical);
8077     /* branch from logical then block to invert block */
8078    aco_ptr<Pseudo_branch_instruction> branch;
8079    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8080    BB_then_logical->instructions.emplace_back(std::move(branch));
8081    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8082    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8083       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8084    BB_then_logical->kind |= block_kind_uniform;
8085    assert(!ctx->cf_info.has_branch);
8086    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8087    ctx->cf_info.parent_loop.has_divergent_branch = false;
8088
8089    /** emit linear then block */
8090    Block* BB_then_linear = ctx->program->create_and_insert_block();
8091    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8092    BB_then_linear->kind |= block_kind_uniform;
8093    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8094    /* branch from linear then block to invert block */
8095    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8096    BB_then_linear->instructions.emplace_back(std::move(branch));
8097    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8098
8099    /** emit invert merge block */
8100    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8101    ic->invert_idx = ctx->block->index;
8102
8103    /* branch to linear else block (skip else) */
8104    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8105    branch->operands[0] = Operand(ic->cond);
8106    ctx->block->instructions.push_back(std::move(branch));
8107
8108    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
8109    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
8110    ic->exec_potentially_empty_break_depth_old =
8111       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8112    /* divergent branches use cbranch_execz */
8113    ctx->cf_info.exec_potentially_empty_discard = false;
8114    ctx->cf_info.exec_potentially_empty_break = false;
8115    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8116
8117    /** emit logical else block */
8118    Block* BB_else_logical = ctx->program->create_and_insert_block();
8119    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8120    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8121    add_linear_edge(ic->invert_idx, BB_else_logical);
8122    ctx->block = BB_else_logical;
8123    append_logical_start(BB_else_logical);
8124 }
8125
8126 static void end_divergent_if(isel_context *ctx, if_context *ic)
8127 {
8128    Block *BB_else_logical = ctx->block;
8129    append_logical_end(BB_else_logical);
8130
8131    /* branch from logical else block to endif block */
8132    aco_ptr<Pseudo_branch_instruction> branch;
8133    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8134    BB_else_logical->instructions.emplace_back(std::move(branch));
8135    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8136    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8137       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8138    BB_else_logical->kind |= block_kind_uniform;
8139
8140    assert(!ctx->cf_info.has_branch);
8141    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8142
8143
8144    /** emit linear else block */
8145    Block* BB_else_linear = ctx->program->create_and_insert_block();
8146    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8147    BB_else_linear->kind |= block_kind_uniform;
8148    add_linear_edge(ic->invert_idx, BB_else_linear);
8149
8150    /* branch from linear else block to endif block */
8151    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8152    BB_else_linear->instructions.emplace_back(std::move(branch));
8153    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8154
8155
8156    /** emit endif merge block */
8157    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8158    append_logical_start(ctx->block);
8159
8160
8161    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8162    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
8163    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
8164    ctx->cf_info.exec_potentially_empty_break_depth =
8165       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8166    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
8167        !ctx->cf_info.parent_if.is_divergent) {
8168       ctx->cf_info.exec_potentially_empty_break = false;
8169       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8170    }
8171    /* uniform control flow never has an empty exec-mask */
8172    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
8173       ctx->cf_info.exec_potentially_empty_discard = false;
8174       ctx->cf_info.exec_potentially_empty_break = false;
8175       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8176    }
8177 }
8178
8179 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8180 {
8181    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8182    Builder bld(ctx->program, ctx->block);
8183    aco_ptr<Pseudo_branch_instruction> branch;
8184
8185    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8186       /**
8187        * Uniform conditionals are represented in the following way*) :
8188        *
8189        * The linear and logical CFG:
8190        *                        BB_IF
8191        *                        /    \
8192        *       BB_THEN (logical)      BB_ELSE (logical)
8193        *                        \    /
8194        *                        BB_ENDIF
8195        *
8196        * *) Exceptions may be due to break and continue statements within loops
8197        *    If a break/continue happens within uniform control flow, it branches
8198        *    to the loop exit/entry block. Otherwise, it branches to the next
8199        *    merge block.
8200        **/
8201       append_logical_end(ctx->block);
8202       ctx->block->kind |= block_kind_uniform;
8203
8204       /* emit branch */
8205       assert(cond.regClass() == bld.lm);
8206       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8207       cond = bool_to_scalar_condition(ctx, cond);
8208
8209       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8210       branch->operands[0] = Operand(cond);
8211       branch->operands[0].setFixed(scc);
8212       ctx->block->instructions.emplace_back(std::move(branch));
8213
8214       unsigned BB_if_idx = ctx->block->index;
8215       Block BB_endif = Block();
8216       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8217       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8218
8219       /** emit then block */
8220       Block* BB_then = ctx->program->create_and_insert_block();
8221       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8222       add_edge(BB_if_idx, BB_then);
8223       append_logical_start(BB_then);
8224       ctx->block = BB_then;
8225       visit_cf_list(ctx, &if_stmt->then_list);
8226       BB_then = ctx->block;
8227       bool then_branch = ctx->cf_info.has_branch;
8228       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8229
8230       if (!then_branch) {
8231          append_logical_end(BB_then);
8232          /* branch from then block to endif block */
8233          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8234          BB_then->instructions.emplace_back(std::move(branch));
8235          add_linear_edge(BB_then->index, &BB_endif);
8236          if (!then_branch_divergent)
8237             add_logical_edge(BB_then->index, &BB_endif);
8238          BB_then->kind |= block_kind_uniform;
8239       }
8240
8241       ctx->cf_info.has_branch = false;
8242       ctx->cf_info.parent_loop.has_divergent_branch = false;
8243
8244       /** emit else block */
8245       Block* BB_else = ctx->program->create_and_insert_block();
8246       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8247       add_edge(BB_if_idx, BB_else);
8248       append_logical_start(BB_else);
8249       ctx->block = BB_else;
8250       visit_cf_list(ctx, &if_stmt->else_list);
8251       BB_else = ctx->block;
8252
8253       if (!ctx->cf_info.has_branch) {
8254          append_logical_end(BB_else);
8255          /* branch from then block to endif block */
8256          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8257          BB_else->instructions.emplace_back(std::move(branch));
8258          add_linear_edge(BB_else->index, &BB_endif);
8259          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8260             add_logical_edge(BB_else->index, &BB_endif);
8261          BB_else->kind |= block_kind_uniform;
8262       }
8263
8264       ctx->cf_info.has_branch &= then_branch;
8265       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8266
8267       /** emit endif merge block */
8268       if (!ctx->cf_info.has_branch) {
8269          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8270          append_logical_start(ctx->block);
8271       }
8272    } else { /* non-uniform condition */
8273       /**
8274        * To maintain a logical and linear CFG without critical edges,
8275        * non-uniform conditionals are represented in the following way*) :
8276        *
8277        * The linear CFG:
8278        *                        BB_IF
8279        *                        /    \
8280        *       BB_THEN (logical)      BB_THEN (linear)
8281        *                        \    /
8282        *                        BB_INVERT (linear)
8283        *                        /    \
8284        *       BB_ELSE (logical)      BB_ELSE (linear)
8285        *                        \    /
8286        *                        BB_ENDIF
8287        *
8288        * The logical CFG:
8289        *                        BB_IF
8290        *                        /    \
8291        *       BB_THEN (logical)      BB_ELSE (logical)
8292        *                        \    /
8293        *                        BB_ENDIF
8294        *
8295        * *) Exceptions may be due to break and continue statements within loops
8296        **/
8297
8298       if_context ic;
8299
8300       begin_divergent_if_then(ctx, &ic, cond);
8301       visit_cf_list(ctx, &if_stmt->then_list);
8302
8303       begin_divergent_if_else(ctx, &ic);
8304       visit_cf_list(ctx, &if_stmt->else_list);
8305
8306       end_divergent_if(ctx, &ic);
8307    }
8308 }
8309
8310 static void visit_cf_list(isel_context *ctx,
8311                           struct exec_list *list)
8312 {
8313    foreach_list_typed(nir_cf_node, node, node, list) {
8314       switch (node->type) {
8315       case nir_cf_node_block:
8316          visit_block(ctx, nir_cf_node_as_block(node));
8317          break;
8318       case nir_cf_node_if:
8319          visit_if(ctx, nir_cf_node_as_if(node));
8320          break;
8321       case nir_cf_node_loop:
8322          visit_loop(ctx, nir_cf_node_as_loop(node));
8323          break;
8324       default:
8325          unreachable("unimplemented cf list type");
8326       }
8327    }
8328 }
8329
8330 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8331 {
8332    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8333    uint64_t mask = ctx->outputs.mask[slot];
8334    if (!is_pos && !mask)
8335       return;
8336    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8337       return;
8338    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8339    exp->enabled_mask = mask;
8340    for (unsigned i = 0; i < 4; ++i) {
8341       if (mask & (1 << i))
8342          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8343       else
8344          exp->operands[i] = Operand(v1);
8345    }
8346    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8347     * Setting valid_mask=1 prevents it and has no other effect.
8348     */
8349    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8350    exp->done = false;
8351    exp->compressed = false;
8352    if (is_pos)
8353       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8354    else
8355       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8356    ctx->block->instructions.emplace_back(std::move(exp));
8357 }
8358
8359 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8360 {
8361    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8362    exp->enabled_mask = 0;
8363    for (unsigned i = 0; i < 4; ++i)
8364       exp->operands[i] = Operand(v1);
8365    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8366       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8367       exp->enabled_mask |= 0x1;
8368    }
8369    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8370       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8371       exp->enabled_mask |= 0x4;
8372    }
8373    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8374       if (ctx->options->chip_class < GFX9) {
8375          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8376          exp->enabled_mask |= 0x8;
8377       } else {
8378          Builder bld(ctx->program, ctx->block);
8379
8380          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8381                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8382          if (exp->operands[2].isTemp())
8383             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8384
8385          exp->operands[2] = Operand(out);
8386          exp->enabled_mask |= 0x4;
8387       }
8388    }
8389    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8390    exp->done = false;
8391    exp->compressed = false;
8392    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8393    ctx->block->instructions.emplace_back(std::move(exp));
8394 }
8395
8396 static void create_vs_exports(isel_context *ctx)
8397 {
8398    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8399
8400    if (outinfo->export_prim_id) {
8401       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8402       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8403    }
8404
8405    if (ctx->options->key.has_multiview_view_index) {
8406       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8407       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8408    }
8409
8410    /* the order these position exports are created is important */
8411    int next_pos = 0;
8412    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8413    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8414       export_vs_psiz_layer_viewport(ctx, &next_pos);
8415    }
8416    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8417       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8418    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8419       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8420
8421    if (ctx->export_clip_dists) {
8422       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8423          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8424       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8425          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8426    }
8427
8428    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8429       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8430           i != VARYING_SLOT_PRIMITIVE_ID)
8431          continue;
8432
8433       export_vs_varying(ctx, i, false, NULL);
8434    }
8435 }
8436
8437 static void export_fs_mrt_z(isel_context *ctx)
8438 {
8439    Builder bld(ctx->program, ctx->block);
8440    unsigned enabled_channels = 0;
8441    bool compr = false;
8442    Operand values[4];
8443
8444    for (unsigned i = 0; i < 4; ++i) {
8445       values[i] = Operand(v1);
8446    }
8447
8448    /* Both stencil and sample mask only need 16-bits. */
8449    if (!ctx->program->info->ps.writes_z &&
8450        (ctx->program->info->ps.writes_stencil ||
8451         ctx->program->info->ps.writes_sample_mask)) {
8452       compr = true; /* COMPR flag */
8453
8454       if (ctx->program->info->ps.writes_stencil) {
8455          /* Stencil should be in X[23:16]. */
8456          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8457          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8458          enabled_channels |= 0x3;
8459       }
8460
8461       if (ctx->program->info->ps.writes_sample_mask) {
8462          /* SampleMask should be in Y[15:0]. */
8463          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8464          enabled_channels |= 0xc;
8465      }
8466    } else {
8467       if (ctx->program->info->ps.writes_z) {
8468          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8469          enabled_channels |= 0x1;
8470       }
8471
8472       if (ctx->program->info->ps.writes_stencil) {
8473          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8474          enabled_channels |= 0x2;
8475       }
8476
8477       if (ctx->program->info->ps.writes_sample_mask) {
8478          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8479          enabled_channels |= 0x4;
8480       }
8481    }
8482
8483    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8484     * writemask component.
8485     */
8486    if (ctx->options->chip_class == GFX6 &&
8487        ctx->options->family != CHIP_OLAND &&
8488        ctx->options->family != CHIP_HAINAN) {
8489             enabled_channels |= 0x1;
8490    }
8491
8492    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8493            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8494 }
8495
8496 static void export_fs_mrt_color(isel_context *ctx, int slot)
8497 {
8498    Builder bld(ctx->program, ctx->block);
8499    unsigned write_mask = ctx->outputs.mask[slot];
8500    Operand values[4];
8501
8502    for (unsigned i = 0; i < 4; ++i) {
8503       if (write_mask & (1 << i)) {
8504          values[i] = Operand(ctx->outputs.outputs[slot][i]);
8505       } else {
8506          values[i] = Operand(v1);
8507       }
8508    }
8509
8510    unsigned target, col_format;
8511    unsigned enabled_channels = 0;
8512    aco_opcode compr_op = (aco_opcode)0;
8513
8514    slot -= FRAG_RESULT_DATA0;
8515    target = V_008DFC_SQ_EXP_MRT + slot;
8516    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
8517
8518    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
8519    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
8520
8521    switch (col_format)
8522    {
8523    case V_028714_SPI_SHADER_ZERO:
8524       enabled_channels = 0; /* writemask */
8525       target = V_008DFC_SQ_EXP_NULL;
8526       break;
8527
8528    case V_028714_SPI_SHADER_32_R:
8529       enabled_channels = 1;
8530       break;
8531
8532    case V_028714_SPI_SHADER_32_GR:
8533       enabled_channels = 0x3;
8534       break;
8535
8536    case V_028714_SPI_SHADER_32_AR:
8537       if (ctx->options->chip_class >= GFX10) {
8538          /* Special case: on GFX10, the outputs are different for 32_AR */
8539          enabled_channels = 0x3;
8540          values[1] = values[3];
8541          values[3] = Operand(v1);
8542       } else {
8543          enabled_channels = 0x9;
8544       }
8545       break;
8546
8547    case V_028714_SPI_SHADER_FP16_ABGR:
8548       enabled_channels = 0x5;
8549       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
8550       break;
8551
8552    case V_028714_SPI_SHADER_UNORM16_ABGR:
8553       enabled_channels = 0x5;
8554       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
8555       break;
8556
8557    case V_028714_SPI_SHADER_SNORM16_ABGR:
8558       enabled_channels = 0x5;
8559       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
8560       break;
8561
8562    case V_028714_SPI_SHADER_UINT16_ABGR: {
8563       enabled_channels = 0x5;
8564       compr_op = aco_opcode::v_cvt_pk_u16_u32;
8565       if (is_int8 || is_int10) {
8566          /* clamp */
8567          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
8568          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8569
8570          for (unsigned i = 0; i < 4; i++) {
8571             if ((write_mask >> i) & 1) {
8572                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
8573                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
8574                                     values[i]);
8575             }
8576          }
8577       }
8578       break;
8579    }
8580
8581    case V_028714_SPI_SHADER_SINT16_ABGR:
8582       enabled_channels = 0x5;
8583       compr_op = aco_opcode::v_cvt_pk_i16_i32;
8584       if (is_int8 || is_int10) {
8585          /* clamp */
8586          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
8587          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
8588          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8589          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
8590
8591          for (unsigned i = 0; i < 4; i++) {
8592             if ((write_mask >> i) & 1) {
8593                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
8594                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
8595                                     values[i]);
8596                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
8597                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
8598                                     values[i]);
8599             }
8600          }
8601       }
8602       break;
8603
8604    case V_028714_SPI_SHADER_32_ABGR:
8605       enabled_channels = 0xF;
8606       break;
8607
8608    default:
8609       break;
8610    }
8611
8612    if (target == V_008DFC_SQ_EXP_NULL)
8613       return;
8614
8615    if ((bool) compr_op) {
8616       for (int i = 0; i < 2; i++) {
8617          /* check if at least one of the values to be compressed is enabled */
8618          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
8619          if (enabled) {
8620             enabled_channels |= enabled << (i*2);
8621             values[i] = bld.vop3(compr_op, bld.def(v1),
8622                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
8623                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
8624          } else {
8625             values[i] = Operand(v1);
8626          }
8627       }
8628       values[2] = Operand(v1);
8629       values[3] = Operand(v1);
8630    } else {
8631       for (int i = 0; i < 4; i++)
8632          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
8633    }
8634
8635    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8636            enabled_channels, target, (bool) compr_op);
8637 }
8638
8639 static void create_fs_exports(isel_context *ctx)
8640 {
8641    /* Export depth, stencil and sample mask. */
8642    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
8643        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
8644        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
8645       export_fs_mrt_z(ctx);
8646    }
8647
8648    /* Export all color render targets. */
8649    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
8650       if (ctx->outputs.mask[i])
8651          export_fs_mrt_color(ctx, i);
8652    }
8653 }
8654
8655 static void emit_stream_output(isel_context *ctx,
8656                                Temp const *so_buffers,
8657                                Temp const *so_write_offset,
8658                                const struct radv_stream_output *output)
8659 {
8660    unsigned num_comps = util_bitcount(output->component_mask);
8661    unsigned writemask = (1 << num_comps) - 1;
8662    unsigned loc = output->location;
8663    unsigned buf = output->buffer;
8664
8665    assert(num_comps && num_comps <= 4);
8666    if (!num_comps || num_comps > 4)
8667       return;
8668
8669    unsigned start = ffs(output->component_mask) - 1;
8670
8671    Temp out[4];
8672    bool all_undef = true;
8673    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
8674    for (unsigned i = 0; i < num_comps; i++) {
8675       out[i] = ctx->outputs.outputs[loc][start + i];
8676       all_undef = all_undef && !out[i].id();
8677    }
8678    if (all_undef)
8679       return;
8680
8681    while (writemask) {
8682       int start, count;
8683       u_bit_scan_consecutive_range(&writemask, &start, &count);
8684       if (count == 3 && ctx->options->chip_class == GFX6) {
8685          /* GFX6 doesn't support storing vec3, split it. */
8686          writemask |= 1u << (start + 2);
8687          count = 2;
8688       }
8689
8690       unsigned offset = output->offset + start * 4;
8691
8692       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8693       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8694       for (int i = 0; i < count; ++i)
8695          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8696       vec->definitions[0] = Definition(write_data);
8697       ctx->block->instructions.emplace_back(std::move(vec));
8698
8699       aco_opcode opcode;
8700       switch (count) {
8701       case 1:
8702          opcode = aco_opcode::buffer_store_dword;
8703          break;
8704       case 2:
8705          opcode = aco_opcode::buffer_store_dwordx2;
8706          break;
8707       case 3:
8708          opcode = aco_opcode::buffer_store_dwordx3;
8709          break;
8710       case 4:
8711          opcode = aco_opcode::buffer_store_dwordx4;
8712          break;
8713       default:
8714          unreachable("Unsupported dword count.");
8715       }
8716
8717       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8718       store->operands[0] = Operand(so_buffers[buf]);
8719       store->operands[1] = Operand(so_write_offset[buf]);
8720       store->operands[2] = Operand((uint32_t) 0);
8721       store->operands[3] = Operand(write_data);
8722       if (offset > 4095) {
8723          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8724          Builder bld(ctx->program, ctx->block);
8725          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8726       } else {
8727          store->offset = offset;
8728       }
8729       store->offen = true;
8730       store->glc = true;
8731       store->dlc = false;
8732       store->slc = true;
8733       store->can_reorder = true;
8734       ctx->block->instructions.emplace_back(std::move(store));
8735    }
8736 }
8737
8738 static void emit_streamout(isel_context *ctx, unsigned stream)
8739 {
8740    Builder bld(ctx->program, ctx->block);
8741
8742    Temp so_buffers[4];
8743    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8744    for (unsigned i = 0; i < 4; i++) {
8745       unsigned stride = ctx->program->info->so.strides[i];
8746       if (!stride)
8747          continue;
8748
8749       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
8750       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
8751    }
8752
8753    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8754                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8755
8756    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8757
8758    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8759
8760    if_context ic;
8761    begin_divergent_if_then(ctx, &ic, can_emit);
8762
8763    bld.reset(ctx->block);
8764
8765    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8766
8767    Temp so_write_offset[4];
8768
8769    for (unsigned i = 0; i < 4; i++) {
8770       unsigned stride = ctx->program->info->so.strides[i];
8771       if (!stride)
8772          continue;
8773
8774       if (stride == 1) {
8775          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8776                                 get_arg(ctx, ctx->args->streamout_write_idx),
8777                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8778          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8779
8780          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8781       } else {
8782          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8783          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8784                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8785          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8786       }
8787    }
8788
8789    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8790       struct radv_stream_output *output =
8791          &ctx->program->info->so.outputs[i];
8792       if (stream != output->stream)
8793          continue;
8794
8795       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8796    }
8797
8798    begin_divergent_if_else(ctx, &ic);
8799    end_divergent_if(ctx, &ic);
8800 }
8801
8802 } /* end namespace */
8803
8804 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8805 {
8806    /* Split all arguments except for the first (ring_offsets) and the last
8807     * (exec) so that the dead channels don't stay live throughout the program.
8808     */
8809    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8810       if (startpgm->definitions[i].regClass().size() > 1) {
8811          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8812                            startpgm->definitions[i].regClass().size());
8813       }
8814    }
8815 }
8816
8817 void handle_bc_optimize(isel_context *ctx)
8818 {
8819    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8820    Builder bld(ctx->program, ctx->block);
8821    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8822    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8823    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8824    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8825    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8826    if (uses_center && uses_centroid) {
8827       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8828                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8829
8830       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8831          Temp new_coord[2];
8832          for (unsigned i = 0; i < 2; i++) {
8833             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8834             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8835             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8836                                     persp_centroid, persp_center, sel);
8837          }
8838          ctx->persp_centroid = bld.tmp(v2);
8839          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8840                     Operand(new_coord[0]), Operand(new_coord[1]));
8841          emit_split_vector(ctx, ctx->persp_centroid, 2);
8842       }
8843
8844       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8845          Temp new_coord[2];
8846          for (unsigned i = 0; i < 2; i++) {
8847             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8848             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8849             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8850                                     linear_centroid, linear_center, sel);
8851          }
8852          ctx->linear_centroid = bld.tmp(v2);
8853          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8854                     Operand(new_coord[0]), Operand(new_coord[1]));
8855          emit_split_vector(ctx, ctx->linear_centroid, 2);
8856       }
8857    }
8858 }
8859
8860 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8861 {
8862    Program *program = ctx->program;
8863
8864    unsigned float_controls = shader->info.float_controls_execution_mode;
8865
8866    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8867       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8868    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8869       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8870                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8871
8872    program->next_fp_mode.must_flush_denorms32 =
8873       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8874    program->next_fp_mode.must_flush_denorms16_64 =
8875       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8876                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8877
8878    program->next_fp_mode.care_about_round32 =
8879       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8880
8881    program->next_fp_mode.care_about_round16_64 =
8882       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8883                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8884
8885    /* default to preserving fp16 and fp64 denorms, since it's free */
8886    if (program->next_fp_mode.must_flush_denorms16_64)
8887       program->next_fp_mode.denorm16_64 = 0;
8888    else
8889       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8890
8891    /* preserving fp32 denorms is expensive, so only do it if asked */
8892    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8893       program->next_fp_mode.denorm32 = fp_denorm_keep;
8894    else
8895       program->next_fp_mode.denorm32 = 0;
8896
8897    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8898       program->next_fp_mode.round32 = fp_round_tz;
8899    else
8900       program->next_fp_mode.round32 = fp_round_ne;
8901
8902    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8903       program->next_fp_mode.round16_64 = fp_round_tz;
8904    else
8905       program->next_fp_mode.round16_64 = fp_round_ne;
8906
8907    ctx->block->fp_mode = program->next_fp_mode;
8908 }
8909
8910 void cleanup_cfg(Program *program)
8911 {
8912    /* create linear_succs/logical_succs */
8913    for (Block& BB : program->blocks) {
8914       for (unsigned idx : BB.linear_preds)
8915          program->blocks[idx].linear_succs.emplace_back(BB.index);
8916       for (unsigned idx : BB.logical_preds)
8917          program->blocks[idx].logical_succs.emplace_back(BB.index);
8918    }
8919 }
8920
8921 void select_program(Program *program,
8922                     unsigned shader_count,
8923                     struct nir_shader *const *shaders,
8924                     ac_shader_config* config,
8925                     struct radv_shader_args *args)
8926 {
8927    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
8928
8929    for (unsigned i = 0; i < shader_count; i++) {
8930       nir_shader *nir = shaders[i];
8931       init_context(&ctx, nir);
8932
8933       setup_fp_mode(&ctx, nir);
8934
8935       if (!i) {
8936          /* needs to be after init_context() for FS */
8937          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8938          append_logical_start(ctx.block);
8939          split_arguments(&ctx, startpgm);
8940       }
8941
8942       if_context ic;
8943       if (shader_count >= 2) {
8944          Builder bld(ctx.program, ctx.block);
8945          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
8946          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8947          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8948
8949          begin_divergent_if_then(&ctx, &ic, cond);
8950       }
8951
8952       if (i) {
8953          Builder bld(ctx.program, ctx.block);
8954          assert(ctx.stage == vertex_geometry_gs);
8955          bld.barrier(aco_opcode::p_memory_barrier_shared);
8956          bld.sopp(aco_opcode::s_barrier);
8957
8958          ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
8959       } else if (ctx.stage == geometry_gs)
8960          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
8961
8962       if (ctx.stage == fragment_fs)
8963          handle_bc_optimize(&ctx);
8964
8965       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8966       visit_cf_list(&ctx, &func->body);
8967
8968       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
8969          emit_streamout(&ctx, 0);
8970
8971       if (ctx.stage == vertex_vs) {
8972          create_vs_exports(&ctx);
8973       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
8974          Builder bld(ctx.program, ctx.block);
8975          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
8976          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
8977       }
8978
8979       if (ctx.stage == fragment_fs)
8980          create_fs_exports(&ctx);
8981
8982       if (shader_count >= 2) {
8983          begin_divergent_if_else(&ctx, &ic);
8984          end_divergent_if(&ctx, &ic);
8985       }
8986
8987       ralloc_free(ctx.divergent_vals);
8988    }
8989
8990    program->config->float_mode = program->blocks[0].fp_mode.val;
8991
8992    append_logical_end(ctx.block);
8993    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
8994    Builder bld(ctx.program, ctx.block);
8995    if (ctx.program->wb_smem_l1_on_end)
8996       bld.smem(aco_opcode::s_dcache_wb, false);
8997    bld.sopp(aco_opcode::s_endpgm);
8998
8999    cleanup_cfg(program);
9000 }
9001
9002 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
9003                            ac_shader_config* config,
9004                            struct radv_shader_args *args)
9005 {
9006    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
9007
9008    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
9009    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
9010    program->next_fp_mode.must_flush_denorms32 = false;
9011    program->next_fp_mode.must_flush_denorms16_64 = false;
9012    program->next_fp_mode.care_about_round32 = false;
9013    program->next_fp_mode.care_about_round16_64 = false;
9014    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9015    program->next_fp_mode.denorm32 = 0;
9016    program->next_fp_mode.round32 = fp_round_ne;
9017    program->next_fp_mode.round16_64 = fp_round_ne;
9018    ctx.block->fp_mode = program->next_fp_mode;
9019
9020    add_startpgm(&ctx);
9021    append_logical_start(ctx.block);
9022
9023    Builder bld(ctx.program, ctx.block);
9024
9025    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
9026
9027    Operand stream_id(0u);
9028    if (args->shader_info->so.num_outputs)
9029       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9030                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
9031
9032    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9033
9034    std::stack<Block> endif_blocks;
9035
9036    for (unsigned stream = 0; stream < 4; stream++) {
9037       if (stream_id.isConstant() && stream != stream_id.constantValue())
9038          continue;
9039
9040       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9041       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9042          continue;
9043
9044       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9045
9046       unsigned BB_if_idx = ctx.block->index;
9047       Block BB_endif = Block();
9048       if (!stream_id.isConstant()) {
9049          /* begin IF */
9050          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9051          append_logical_end(ctx.block);
9052          ctx.block->kind |= block_kind_uniform;
9053          bld.branch(aco_opcode::p_cbranch_z, cond);
9054
9055          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9056
9057          ctx.block = ctx.program->create_and_insert_block();
9058          add_edge(BB_if_idx, ctx.block);
9059          bld.reset(ctx.block);
9060          append_logical_start(ctx.block);
9061       }
9062
9063       unsigned offset = 0;
9064       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9065          if (args->shader_info->gs.output_streams[i] != stream)
9066             continue;
9067
9068          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9069          unsigned length = util_last_bit(output_usage_mask);
9070          for (unsigned j = 0; j < length; ++j) {
9071             if (!(output_usage_mask & (1 << j)))
9072                continue;
9073
9074             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9075             Temp voffset = vtx_offset;
9076             if (const_offset >= 4096u) {
9077                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9078                const_offset %= 4096u;
9079             }
9080
9081             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9082             mubuf->definitions[0] = bld.def(v1);
9083             mubuf->operands[0] = Operand(gsvs_ring);
9084             mubuf->operands[1] = Operand(voffset);
9085             mubuf->operands[2] = Operand(0u);
9086             mubuf->offen = true;
9087             mubuf->offset = const_offset;
9088             mubuf->glc = true;
9089             mubuf->slc = true;
9090             mubuf->dlc = args->options->chip_class >= GFX10;
9091             mubuf->barrier = barrier_none;
9092             mubuf->can_reorder = true;
9093
9094             ctx.outputs.mask[i] |= 1 << j;
9095             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
9096
9097             bld.insert(std::move(mubuf));
9098
9099             offset++;
9100          }
9101       }
9102
9103       if (args->shader_info->so.num_outputs) {
9104          emit_streamout(&ctx, stream);
9105          bld.reset(ctx.block);
9106       }
9107
9108       if (stream == 0) {
9109          create_vs_exports(&ctx);
9110          ctx.block->kind |= block_kind_export_end;
9111       }
9112
9113       if (!stream_id.isConstant()) {
9114          append_logical_end(ctx.block);
9115
9116          /* branch from then block to endif block */
9117          bld.branch(aco_opcode::p_branch);
9118          add_edge(ctx.block->index, &BB_endif);
9119          ctx.block->kind |= block_kind_uniform;
9120
9121          /* emit else block */
9122          ctx.block = ctx.program->create_and_insert_block();
9123          add_edge(BB_if_idx, ctx.block);
9124          bld.reset(ctx.block);
9125          append_logical_start(ctx.block);
9126
9127          endif_blocks.push(std::move(BB_endif));
9128       }
9129    }
9130
9131    while (!endif_blocks.empty()) {
9132       Block BB_endif = std::move(endif_blocks.top());
9133       endif_blocks.pop();
9134
9135       Block *BB_else = ctx.block;
9136
9137       append_logical_end(BB_else);
9138       /* branch from else block to endif block */
9139       bld.branch(aco_opcode::p_branch);
9140       add_edge(BB_else->index, &BB_endif);
9141       BB_else->kind |= block_kind_uniform;
9142
9143       /** emit endif merge block */
9144       ctx.block = program->insert_block(std::move(BB_endif));
9145       bld.reset(ctx.block);
9146       append_logical_start(ctx.block);
9147    }
9148
9149    program->config->float_mode = program->blocks[0].fp_mode.val;
9150
9151    append_logical_end(ctx.block);
9152    ctx.block->kind |= block_kind_uniform;
9153    bld.sopp(aco_opcode::s_endpgm);
9154
9155    cleanup_cfg(program);
9156 }
9157 }