src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_old;
  83
  84    unsigned BB_if_idx;
  85    unsigned invert_idx;
  86    bool then_branch_divergent;
  87    Block BB_invert;
  88    Block BB_endif;
  89 };
  90
  91 static void visit_cf_list(struct isel_context *ctx,
  92                           struct exec_list *list);
  93
  94 static void add_logical_edge(unsigned pred_idx, Block *succ)
  95 {
  96    succ->logical_preds.emplace_back(pred_idx);
  97 }
  98
  99
 100 static void add_linear_edge(unsigned pred_idx, Block *succ)
 101 {
 102    succ->linear_preds.emplace_back(pred_idx);
 103 }
 104
 105 static void add_edge(unsigned pred_idx, Block *succ)
 106 {
 107    add_logical_edge(pred_idx, succ);
 108    add_linear_edge(pred_idx, succ);
 109 }
 110
 111 static void append_logical_start(Block *b)
 112 {
 113    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 114 }
 115
 116 static void append_logical_end(Block *b)
 117 {
 118    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 119 }
 120
 121 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 122 {
 123    assert(ctx->allocated[def->index].id());
 124    return ctx->allocated[def->index];
 125 }
 126
 127 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 128                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 129 {
 130    Builder bld(ctx->program, ctx->block);
 131    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 132    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 133
 134    if (ctx->program->wave_size == 32) {
 135       return thread_id_lo;
 136    } else {
 137       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 138       return thread_id_hi;
 139    }
 140 }
 141
 142 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 143 {
 144    Builder bld(ctx->program, ctx->block);
 145
 146    if (!dst.id())
 147       dst = bld.tmp(src.regClass());
 148
 149    assert(src.size() == dst.size());
 150
 151    if (ctx->stage != fragment_fs) {
 152       if (!dst.id())
 153          return src;
 154
 155       bld.copy(Definition(dst), src);
 156       return dst;
 157    }
 158
 159    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 160    ctx->program->needs_wqm |= program_needs_wqm;
 161    return dst;
 162 }
 163
 164 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 165 {
 166    if (index.regClass() == s1)
 167       return bld.readlane(bld.def(s1), data, index);
 168
 169    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 170
 171    /* Currently not implemented on GFX6-7 */
 172    assert(ctx->options->chip_class >= GFX8);
 173
 174    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 175       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 176    }
 177
 178    /* GFX10, wave64 mode:
 179     * The bpermute instruction is limited to half-wave operation, which means that it can't
 180     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 181     * emulate it here.
 182     */
 183    if (!ctx->has_gfx10_wave64_bpermute) {
 184       ctx->has_gfx10_wave64_bpermute = true;
 185       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 186       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 187    }
 188
 189    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 190    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 191    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 192    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 193
 194    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 195                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 196 }
 197
 198 Temp as_vgpr(isel_context *ctx, Temp val)
 199 {
 200    if (val.type() == RegType::sgpr) {
 201       Builder bld(ctx->program, ctx->block);
 202       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 203    }
 204    assert(val.type() == RegType::vgpr);
 205    return val;
 206 }
 207
 208 //assumes a != 0xffffffff
 209 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 210 {
 211    assert(b != 0);
 212    Builder bld(ctx->program, ctx->block);
 213
 214    if (util_is_power_of_two_or_zero(b)) {
 215       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 216       return;
 217    }
 218
 219    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 220
 221    assert(info.multiplier <= 0xffffffff);
 222
 223    bool pre_shift = info.pre_shift != 0;
 224    bool increment = info.increment != 0;
 225    bool multiply = true;
 226    bool post_shift = info.post_shift != 0;
 227
 228    if (!pre_shift && !increment && !multiply && !post_shift) {
 229       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 230       return;
 231    }
 232
 233    Temp pre_shift_dst = a;
 234    if (pre_shift) {
 235       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 237    }
 238
 239    Temp increment_dst = pre_shift_dst;
 240    if (increment) {
 241       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 242       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 243    }
 244
 245    Temp multiply_dst = increment_dst;
 246    if (multiply) {
 247       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 248       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 249                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 250    }
 251
 252    if (post_shift) {
 253       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 254    }
 255 }
 256
 257 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 258 {
 259    Builder bld(ctx->program, ctx->block);
 260    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 261 }
 262
 263
 264 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 265 {
 266    /* no need to extract the whole vector */
 267    if (src.regClass() == dst_rc) {
 268       assert(idx == 0);
 269       return src;
 270    }
 271    assert(src.size() > idx);
 272    Builder bld(ctx->program, ctx->block);
 273    auto it = ctx->allocated_vec.find(src.id());
 274    /* the size check needs to be early because elements other than 0 may be garbage */
 275    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 276       if (it->second[idx].regClass() == dst_rc) {
 277          return it->second[idx];
 278       } else {
 279          assert(dst_rc.size() == it->second[idx].regClass().size());
 280          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 281          return bld.copy(bld.def(dst_rc), it->second[idx]);
 282       }
 283    }
 284
 285    if (src.size() == dst_rc.size()) {
 286       assert(idx == 0);
 287       return bld.copy(bld.def(dst_rc), src);
 288    } else {
 289       Temp dst = bld.tmp(dst_rc);
 290       emit_extract_vector(ctx, src, idx, dst);
 291       return dst;
 292    }
 293 }
 294
 295 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 296 {
 297    if (num_components == 1)
 298       return;
 299    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 300       return;
 301    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 302    split->operands[0] = Operand(vec_src);
 303    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 304    for (unsigned i = 0; i < num_components; i++) {
 305       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 306       split->definitions[i] = Definition(elems[i]);
 307    }
 308    ctx->block->instructions.emplace_back(std::move(split));
 309    ctx->allocated_vec.emplace(vec_src.id(), elems);
 310 }
 311
 312 /* This vector expansion uses a mask to determine which elements in the new vector
 313  * come from the original vector. The other elements are undefined. */
 314 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 315 {
 316    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 317
 318    if (vec_src == dst)
 319       return;
 320
 321    Builder bld(ctx->program, ctx->block);
 322    if (num_components == 1) {
 323       if (dst.type() == RegType::sgpr)
 324          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 325       else
 326          bld.copy(Definition(dst), vec_src);
 327       return;
 328    }
 329
 330    unsigned component_size = dst.size() / num_components;
 331    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 332
 333    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 334    vec->definitions[0] = Definition(dst);
 335    unsigned k = 0;
 336    for (unsigned i = 0; i < num_components; i++) {
 337       if (mask & (1 << i)) {
 338          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 339          if (dst.type() == RegType::sgpr)
 340             src = bld.as_uniform(src);
 341          vec->operands[i] = Operand(src);
 342       } else {
 343          vec->operands[i] = Operand(0u);
 344       }
 345       elems[i] = vec->operands[i].getTemp();
 346    }
 347    ctx->block->instructions.emplace_back(std::move(vec));
 348    ctx->allocated_vec.emplace(dst.id(), elems);
 349 }
 350
 351 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 352 {
 353    Builder bld(ctx->program, ctx->block);
 354    if (!dst.id())
 355       dst = bld.tmp(bld.lm);
 356
 357    assert(val.regClass() == s1);
 358    assert(dst.regClass() == bld.lm);
 359
 360    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 361 }
 362
 363 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 364 {
 365    Builder bld(ctx->program, ctx->block);
 366    if (!dst.id())
 367       dst = bld.tmp(s1);
 368
 369    assert(val.regClass() == bld.lm);
 370    assert(dst.regClass() == s1);
 371
 372    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 373    Temp tmp = bld.tmp(s1);
 374    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 375    return emit_wqm(ctx, tmp, dst);
 376 }
 377
 378 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 379 {
 380    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 381       return get_ssa_temp(ctx, src.src.ssa);
 382
 383    if (src.src.ssa->num_components == size) {
 384       bool identity_swizzle = true;
 385       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 386          if (src.swizzle[i] != i)
 387             identity_swizzle = false;
 388       }
 389       if (identity_swizzle)
 390          return get_ssa_temp(ctx, src.src.ssa);
 391    }
 392
 393    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 394    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 395    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 396    assert(vec.size() % elem_size == 0);
 397
 398    RegClass elem_rc = RegClass(vec.type(), elem_size);
 399    if (size == 1) {
 400       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 401    } else {
 402       assert(size <= 4);
 403       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 404       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 405       for (unsigned i = 0; i < size; ++i) {
 406          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 407          vec_instr->operands[i] = Operand{elems[i]};
 408       }
 409       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 410       vec_instr->definitions[0] = Definition(dst);
 411       ctx->block->instructions.emplace_back(std::move(vec_instr));
 412       ctx->allocated_vec.emplace(dst.id(), elems);
 413       return dst;
 414    }
 415 }
 416
 417 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 418 {
 419    if (ptr.size() == 2)
 420       return ptr;
 421    Builder bld(ctx->program, ctx->block);
 422    if (ptr.type() == RegType::vgpr)
 423       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 424    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 425                      ptr, Operand((unsigned)ctx->options->address32_hi));
 426 }
 427
 428 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 429 {
 430    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 431    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 432    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 433    sop2->definitions[0] = Definition(dst);
 434    if (writes_scc)
 435       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 436    ctx->block->instructions.emplace_back(std::move(sop2));
 437 }
 438
 439 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 440                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 441 {
 442    Builder bld(ctx->program, ctx->block);
 443    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 444    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 445    if (src1.type() == RegType::sgpr) {
 446       if (commutative && src0.type() == RegType::vgpr) {
 447          Temp t = src0;
 448          src0 = src1;
 449          src1 = t;
 450       } else if (src0.type() == RegType::vgpr &&
 451                  op != aco_opcode::v_madmk_f32 &&
 452                  op != aco_opcode::v_madak_f32 &&
 453                  op != aco_opcode::v_madmk_f16 &&
 454                  op != aco_opcode::v_madak_f16) {
 455          /* If the instruction is not commutative, we emit a VOP3A instruction */
 456          bld.vop2_e64(op, Definition(dst), src0, src1);
 457          return;
 458       } else {
 459          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 460       }
 461    }
 462
 463    if (flush_denorms && ctx->program->chip_class < GFX9) {
 464       assert(dst.size() == 1);
 465       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 466       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 467    } else {
 468       bld.vop2(op, Definition(dst), src0, src1);
 469    }
 470 }
 471
 472 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 473                             bool flush_denorms = false)
 474 {
 475    Temp src0 = get_alu_src(ctx, instr->src[0]);
 476    Temp src1 = get_alu_src(ctx, instr->src[1]);
 477    Temp src2 = get_alu_src(ctx, instr->src[2]);
 478
 479    /* ensure that the instruction has at most 1 sgpr operand
 480     * The optimizer will inline constants for us */
 481    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 482       src0 = as_vgpr(ctx, src0);
 483    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 484       src1 = as_vgpr(ctx, src1);
 485    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 486       src2 = as_vgpr(ctx, src2);
 487
 488    Builder bld(ctx->program, ctx->block);
 489    if (flush_denorms && ctx->program->chip_class < GFX9) {
 490       assert(dst.size() == 1);
 491       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 492       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 493    } else {
 494       bld.vop3(op, Definition(dst), src0, src1, src2);
 495    }
 496 }
 497
 498 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 499 {
 500    Builder bld(ctx->program, ctx->block);
 501    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 502 }
 503
 504 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 505 {
 506    Temp src0 = get_alu_src(ctx, instr->src[0]);
 507    Temp src1 = get_alu_src(ctx, instr->src[1]);
 508    assert(src0.size() == src1.size());
 509
 510    aco_ptr<Instruction> vopc;
 511    if (src1.type() == RegType::sgpr) {
 512       if (src0.type() == RegType::vgpr) {
 513          /* to swap the operands, we might also have to change the opcode */
 514          switch (op) {
 515             case aco_opcode::v_cmp_lt_f32:
 516                op = aco_opcode::v_cmp_gt_f32;
 517                break;
 518             case aco_opcode::v_cmp_ge_f32:
 519                op = aco_opcode::v_cmp_le_f32;
 520                break;
 521             case aco_opcode::v_cmp_lt_i32:
 522                op = aco_opcode::v_cmp_gt_i32;
 523                break;
 524             case aco_opcode::v_cmp_ge_i32:
 525                op = aco_opcode::v_cmp_le_i32;
 526                break;
 527             case aco_opcode::v_cmp_lt_u32:
 528                op = aco_opcode::v_cmp_gt_u32;
 529                break;
 530             case aco_opcode::v_cmp_ge_u32:
 531                op = aco_opcode::v_cmp_le_u32;
 532                break;
 533             case aco_opcode::v_cmp_lt_f64:
 534                op = aco_opcode::v_cmp_gt_f64;
 535                break;
 536             case aco_opcode::v_cmp_ge_f64:
 537                op = aco_opcode::v_cmp_le_f64;
 538                break;
 539             case aco_opcode::v_cmp_lt_i64:
 540                op = aco_opcode::v_cmp_gt_i64;
 541                break;
 542             case aco_opcode::v_cmp_ge_i64:
 543                op = aco_opcode::v_cmp_le_i64;
 544                break;
 545             case aco_opcode::v_cmp_lt_u64:
 546                op = aco_opcode::v_cmp_gt_u64;
 547                break;
 548             case aco_opcode::v_cmp_ge_u64:
 549                op = aco_opcode::v_cmp_le_u64;
 550                break;
 551             default: /* eq and ne are commutative */
 552                break;
 553          }
 554          Temp t = src0;
 555          src0 = src1;
 556          src1 = t;
 557       } else {
 558          src1 = as_vgpr(ctx, src1);
 559       }
 560    }
 561
 562    Builder bld(ctx->program, ctx->block);
 563    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 564 }
 565
 566 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 567 {
 568    Temp src0 = get_alu_src(ctx, instr->src[0]);
 569    Temp src1 = get_alu_src(ctx, instr->src[1]);
 570    Builder bld(ctx->program, ctx->block);
 571
 572    assert(dst.regClass() == bld.lm);
 573    assert(src0.type() == RegType::sgpr);
 574    assert(src1.type() == RegType::sgpr);
 575    assert(src0.regClass() == src1.regClass());
 576
 577    /* Emit the SALU comparison instruction */
 578    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 579    /* Turn the result into a per-lane bool */
 580    bool_to_vector_condition(ctx, cmp, dst);
 581 }
 582
 583 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 584                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 585 {
 586    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 587    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 588    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 589    bool use_valu = s_op == aco_opcode::num_opcodes ||
 590                    divergent_vals ||
 591                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 592                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 593    aco_opcode op = use_valu ? v_op : s_op;
 594    assert(op != aco_opcode::num_opcodes);
 595    assert(dst.regClass() == ctx->program->lane_mask);
 596
 597    if (use_valu)
 598       emit_vopc_instruction(ctx, instr, op, dst);
 599    else
 600       emit_sopc_instruction(ctx, instr, op, dst);
 601 }
 602
 603 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 604 {
 605    Builder bld(ctx->program, ctx->block);
 606    Temp src0 = get_alu_src(ctx, instr->src[0]);
 607    Temp src1 = get_alu_src(ctx, instr->src[1]);
 608
 609    assert(dst.regClass() == bld.lm);
 610    assert(src0.regClass() == bld.lm);
 611    assert(src1.regClass() == bld.lm);
 612
 613    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 614 }
 615
 616 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 617 {
 618    Builder bld(ctx->program, ctx->block);
 619    Temp cond = get_alu_src(ctx, instr->src[0]);
 620    Temp then = get_alu_src(ctx, instr->src[1]);
 621    Temp els = get_alu_src(ctx, instr->src[2]);
 622
 623    assert(cond.regClass() == bld.lm);
 624
 625    if (dst.type() == RegType::vgpr) {
 626       aco_ptr<Instruction> bcsel;
 627       if (dst.size() == 1) {
 628          then = as_vgpr(ctx, then);
 629          els = as_vgpr(ctx, els);
 630
 631          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 632       } else if (dst.size() == 2) {
 633          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 634          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 635          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 637
 638          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 639          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 640
 641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 642       } else {
 643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 644          nir_print_instr(&instr->instr, stderr);
 645          fprintf(stderr, "\n");
 646       }
 647       return;
 648    }
 649
 650    if (instr->dest.dest.ssa.bit_size == 1) {
 651       assert(dst.regClass() == bld.lm);
 652       assert(then.regClass() == bld.lm);
 653       assert(els.regClass() == bld.lm);
 654    }
 655
 656    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 657       if (dst.regClass() == s1 || dst.regClass() == s2) {
 658          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 659          assert(dst.size() == then.size());
 660          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 661          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 662       } else {
 663          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 664          nir_print_instr(&instr->instr, stderr);
 665          fprintf(stderr, "\n");
 666       }
 667       return;
 668    }
 669
 670    /* divergent boolean bcsel
 671     * this implements bcsel on bools: dst = s0 ? s1 : s2
 672     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 673    assert(instr->dest.dest.ssa.bit_size == 1);
 674
 675    if (cond.id() != then.id())
 676       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 677
 678    if (cond.id() == els.id())
 679       bld.sop1(Builder::s_mov, Definition(dst), then);
 680    else
 681       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 682                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 683 }
 684
 685 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 686                     aco_opcode op, uint32_t undo)
 687 {
 688    /* multiply by 16777216 to handle denormals */
 689    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 690                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 691    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 692    scaled = bld.vop1(op, bld.def(v1), scaled);
 693    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 694
 695    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 696
 697    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 698 }
 699
 700 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 701 {
 702    if (ctx->block->fp_mode.denorm32 == 0) {
 703       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 704       return;
 705    }
 706
 707    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 708 }
 709
 710 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 711 {
 712    if (ctx->block->fp_mode.denorm32 == 0) {
 713       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 714       return;
 715    }
 716
 717    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 718 }
 719
 720 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 721 {
 722    if (ctx->block->fp_mode.denorm32 == 0) {
 723       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 724       return;
 725    }
 726
 727    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 728 }
 729
 730 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 731 {
 732    if (ctx->block->fp_mode.denorm32 == 0) {
 733       bld.vop1(aco_opcode::v_log_f32, dst, val);
 734       return;
 735    }
 736
 737    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 738 }
 739
 740 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 741 {
 742    if (ctx->options->chip_class >= GFX7)
 743       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 744
 745    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 746    /* TODO: create more efficient code! */
 747    if (val.type() == RegType::sgpr)
 748       val = as_vgpr(ctx, val);
 749
 750    /* Split the input value. */
 751    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 752    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 753
 754    /* Extract the exponent and compute the unbiased value. */
 755    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 756
 757    /* Extract the fractional part. */
 758    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 759    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 760
 761    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 762    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 763
 764    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 765    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 766    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 767    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 768    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 769
 770    /* Get the sign bit. */
 771    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 772
 773    /* Decide the operation to apply depending on the unbiased exponent. */
 774    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 775    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 776    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 777    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 778    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 779    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 780
 781    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 782 }
 783
 784 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 785 {
 786    if (ctx->options->chip_class >= GFX7)
 787       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 788
 789    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 790    Temp src0 = as_vgpr(ctx, val);
 791
 792    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 793    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 794
 795    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 796    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 797    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 798
 799    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 800    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 801    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 803
 804    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 805    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 806
 807    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 808
 809    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 810    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 811
 812    return add->definitions[0].getTemp();
 813 }
 814
 815 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 816 {
 817    if (!instr->dest.dest.is_ssa) {
 818       fprintf(stderr, "nir alu dst not in ssa: ");
 819       nir_print_instr(&instr->instr, stderr);
 820       fprintf(stderr, "\n");
 821       abort();
 822    }
 823    Builder bld(ctx->program, ctx->block);
 824    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 825    switch(instr->op) {
 826    case nir_op_vec2:
 827    case nir_op_vec3:
 828    case nir_op_vec4: {
 829       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 830       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 831       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 832          elems[i] = get_alu_src(ctx, instr->src[i]);
 833          vec->operands[i] = Operand{elems[i]};
 834       }
 835       vec->definitions[0] = Definition(dst);
 836       ctx->block->instructions.emplace_back(std::move(vec));
 837       ctx->allocated_vec.emplace(dst.id(), elems);
 838       break;
 839    }
 840    case nir_op_mov: {
 841       Temp src = get_alu_src(ctx, instr->src[0]);
 842       aco_ptr<Instruction> mov;
 843       if (dst.type() == RegType::sgpr) {
 844          if (src.type() == RegType::vgpr)
 845             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 846          else if (src.regClass() == s1)
 847             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 848          else if (src.regClass() == s2)
 849             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 850          else
 851             unreachable("wrong src register class for nir_op_imov");
 852       } else if (dst.regClass() == v1) {
 853          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 854       } else if (dst.regClass() == v2) {
 855          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 856       } else {
 857          nir_print_instr(&instr->instr, stderr);
 858          unreachable("Should have been lowered to scalar.");
 859       }
 860       break;
 861    }
 862    case nir_op_inot: {
 863       Temp src = get_alu_src(ctx, instr->src[0]);
 864       if (instr->dest.dest.ssa.bit_size == 1) {
 865          assert(src.regClass() == bld.lm);
 866          assert(dst.regClass() == bld.lm);
 867          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 868          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 869          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 870       } else if (dst.regClass() == v1) {
 871          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 872       } else if (dst.type() == RegType::sgpr) {
 873          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 874          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_ineg: {
 883       Temp src = get_alu_src(ctx, instr->src[0]);
 884       if (dst.regClass() == v1) {
 885          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 886       } else if (dst.regClass() == s1) {
 887          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 888       } else if (dst.size() == 2) {
 889          Temp src0 = bld.tmp(dst.type(), 1);
 890          Temp src1 = bld.tmp(dst.type(), 1);
 891          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 892
 893          if (dst.regClass() == s2) {
 894             Temp carry = bld.tmp(s1);
 895             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 896             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 897             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 898          } else {
 899             Temp lower = bld.tmp(v1);
 900             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 901             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 902             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 903          }
 904       } else {
 905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 906          nir_print_instr(&instr->instr, stderr);
 907          fprintf(stderr, "\n");
 908       }
 909       break;
 910    }
 911    case nir_op_iabs: {
 912       if (dst.regClass() == s1) {
 913          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 914       } else if (dst.regClass() == v1) {
 915          Temp src = get_alu_src(ctx, instr->src[0]);
 916          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 917       } else {
 918          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 919          nir_print_instr(&instr->instr, stderr);
 920          fprintf(stderr, "\n");
 921       }
 922       break;
 923    }
 924    case nir_op_isign: {
 925       Temp src = get_alu_src(ctx, instr->src[0]);
 926       if (dst.regClass() == s1) {
 927          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 928          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 929          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 930       } else if (dst.regClass() == s2) {
 931          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 932          Temp neqz;
 933          if (ctx->program->chip_class >= GFX8)
 934             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 935          else
 936             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 937          /* SCC gets zero-extended to 64 bit */
 938          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 939       } else if (dst.regClass() == v1) {
 940          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 941          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 942          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 943       } else if (dst.regClass() == v2) {
 944          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 945          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 946          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 947          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 948          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 949          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 950       } else {
 951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 952          nir_print_instr(&instr->instr, stderr);
 953          fprintf(stderr, "\n");
 954       }
 955       break;
 956    }
 957    case nir_op_imax: {
 958       if (dst.regClass() == v1) {
 959          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 960       } else if (dst.regClass() == s1) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 962       } else {
 963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 964          nir_print_instr(&instr->instr, stderr);
 965          fprintf(stderr, "\n");
 966       }
 967       break;
 968    }
 969    case nir_op_umax: {
 970       if (dst.regClass() == v1) {
 971          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 972       } else if (dst.regClass() == s1) {
 973          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 974       } else {
 975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 976          nir_print_instr(&instr->instr, stderr);
 977          fprintf(stderr, "\n");
 978       }
 979       break;
 980    }
 981    case nir_op_imin: {
 982       if (dst.regClass() == v1) {
 983          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 984       } else if (dst.regClass() == s1) {
 985          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 986       } else {
 987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 988          nir_print_instr(&instr->instr, stderr);
 989          fprintf(stderr, "\n");
 990       }
 991       break;
 992    }
 993    case nir_op_umin: {
 994       if (dst.regClass() == v1) {
 995          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 996       } else if (dst.regClass() == s1) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_ior: {
1006       if (instr->dest.dest.ssa.bit_size == 1) {
1007          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1008       } else if (dst.regClass() == v1) {
1009          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1010       } else if (dst.regClass() == s1) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1012       } else if (dst.regClass() == s2) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1014       } else {
1015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1016          nir_print_instr(&instr->instr, stderr);
1017          fprintf(stderr, "\n");
1018       }
1019       break;
1020    }
1021    case nir_op_iand: {
1022       if (instr->dest.dest.ssa.bit_size == 1) {
1023          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1024       } else if (dst.regClass() == v1) {
1025          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1026       } else if (dst.regClass() == s1) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1028       } else if (dst.regClass() == s2) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1030       } else {
1031          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1032          nir_print_instr(&instr->instr, stderr);
1033          fprintf(stderr, "\n");
1034       }
1035       break;
1036    }
1037    case nir_op_ixor: {
1038       if (instr->dest.dest.ssa.bit_size == 1) {
1039          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1040       } else if (dst.regClass() == v1) {
1041          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1042       } else if (dst.regClass() == s1) {
1043          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1044       } else if (dst.regClass() == s2) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1046       } else {
1047          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1048          nir_print_instr(&instr->instr, stderr);
1049          fprintf(stderr, "\n");
1050       }
1051       break;
1052    }
1053    case nir_op_ushr: {
1054       if (dst.regClass() == v1) {
1055          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1056       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1057          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1058                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1059       } else if (dst.regClass() == v2) {
1060          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1061                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1062       } else if (dst.regClass() == s2) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1064       } else if (dst.regClass() == s1) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1066       } else {
1067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1068          nir_print_instr(&instr->instr, stderr);
1069          fprintf(stderr, "\n");
1070       }
1071       break;
1072    }
1073    case nir_op_ishl: {
1074       if (dst.regClass() == v1) {
1075          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1076       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1077          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1078                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1079       } else if (dst.regClass() == v2) {
1080          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1081                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1082       } else if (dst.regClass() == s1) {
1083          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1084       } else if (dst.regClass() == s2) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_ishr: {
1094       if (dst.regClass() == v1) {
1095          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1096       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1097          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1098                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1099       } else if (dst.regClass() == v2) {
1100          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1101                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1102       } else if (dst.regClass() == s1) {
1103          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1104       } else if (dst.regClass() == s2) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1106       } else {
1107          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1108          nir_print_instr(&instr->instr, stderr);
1109          fprintf(stderr, "\n");
1110       }
1111       break;
1112    }
1113    case nir_op_find_lsb: {
1114       Temp src = get_alu_src(ctx, instr->src[0]);
1115       if (src.regClass() == s1) {
1116          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1117       } else if (src.regClass() == v1) {
1118          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1119       } else if (src.regClass() == s2) {
1120          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1121       } else {
1122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1123          nir_print_instr(&instr->instr, stderr);
1124          fprintf(stderr, "\n");
1125       }
1126       break;
1127    }
1128    case nir_op_ufind_msb:
1129    case nir_op_ifind_msb: {
1130       Temp src = get_alu_src(ctx, instr->src[0]);
1131       if (src.regClass() == s1 || src.regClass() == s2) {
1132          aco_opcode op = src.regClass() == s2 ?
1133                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1134                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1135          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1136
1137          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1138                                         Operand(src.size() * 32u - 1u), msb_rev);
1139          Temp msb = sub.def(0).getTemp();
1140          Temp carry = sub.def(1).getTemp();
1141
1142          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1143       } else if (src.regClass() == v1) {
1144          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1145          Temp msb_rev = bld.tmp(v1);
1146          emit_vop1_instruction(ctx, instr, op, msb_rev);
1147          Temp msb = bld.tmp(v1);
1148          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1149          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1150       } else {
1151          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1152          nir_print_instr(&instr->instr, stderr);
1153          fprintf(stderr, "\n");
1154       }
1155       break;
1156    }
1157    case nir_op_bitfield_reverse: {
1158       if (dst.regClass() == s1) {
1159          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1160       } else if (dst.regClass() == v1) {
1161          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else {
1163          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1164          nir_print_instr(&instr->instr, stderr);
1165          fprintf(stderr, "\n");
1166       }
1167       break;
1168    }
1169    case nir_op_iadd: {
1170       if (dst.regClass() == s1) {
1171          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1172          break;
1173       }
1174
1175       Temp src0 = get_alu_src(ctx, instr->src[0]);
1176       Temp src1 = get_alu_src(ctx, instr->src[1]);
1177       if (dst.regClass() == v1) {
1178          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1179          break;
1180       }
1181
1182       assert(src0.size() == 2 && src1.size() == 2);
1183       Temp src00 = bld.tmp(src0.type(), 1);
1184       Temp src01 = bld.tmp(dst.type(), 1);
1185       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1186       Temp src10 = bld.tmp(src1.type(), 1);
1187       Temp src11 = bld.tmp(dst.type(), 1);
1188       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1189
1190       if (dst.regClass() == s2) {
1191          Temp carry = bld.tmp(s1);
1192          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1193          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1195       } else if (dst.regClass() == v2) {
1196          Temp dst0 = bld.tmp(v1);
1197          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1198          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1199          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1200       } else {
1201          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1202          nir_print_instr(&instr->instr, stderr);
1203          fprintf(stderr, "\n");
1204       }
1205       break;
1206    }
1207    case nir_op_uadd_sat: {
1208       Temp src0 = get_alu_src(ctx, instr->src[0]);
1209       Temp src1 = get_alu_src(ctx, instr->src[1]);
1210       if (dst.regClass() == s1) {
1211          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1212          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1213                   src0, src1);
1214          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1215       } else if (dst.regClass() == v1) {
1216          if (ctx->options->chip_class >= GFX9) {
1217             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1218             add->operands[0] = Operand(src0);
1219             add->operands[1] = Operand(src1);
1220             add->definitions[0] = Definition(dst);
1221             add->clamp = 1;
1222             ctx->block->instructions.emplace_back(std::move(add));
1223          } else {
1224             if (src1.regClass() != v1)
1225                std::swap(src0, src1);
1226             assert(src1.regClass() == v1);
1227             Temp tmp = bld.tmp(v1);
1228             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1229             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1230          }
1231       } else {
1232          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1233          nir_print_instr(&instr->instr, stderr);
1234          fprintf(stderr, "\n");
1235       }
1236       break;
1237    }
1238    case nir_op_uadd_carry: {
1239       Temp src0 = get_alu_src(ctx, instr->src[0]);
1240       Temp src1 = get_alu_src(ctx, instr->src[1]);
1241       if (dst.regClass() == s1) {
1242          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1243          break;
1244       }
1245       if (dst.regClass() == v1) {
1246          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1247          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1248          break;
1249       }
1250
1251       Temp src00 = bld.tmp(src0.type(), 1);
1252       Temp src01 = bld.tmp(dst.type(), 1);
1253       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1254       Temp src10 = bld.tmp(src1.type(), 1);
1255       Temp src11 = bld.tmp(dst.type(), 1);
1256       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1257       if (dst.regClass() == s2) {
1258          Temp carry = bld.tmp(s1);
1259          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1260          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1261          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1262       } else if (dst.regClass() == v2) {
1263          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1264          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1265          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1266          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1267       } else {
1268          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1269          nir_print_instr(&instr->instr, stderr);
1270          fprintf(stderr, "\n");
1271       }
1272       break;
1273    }
1274    case nir_op_isub: {
1275       if (dst.regClass() == s1) {
1276          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1277          break;
1278       }
1279
1280       Temp src0 = get_alu_src(ctx, instr->src[0]);
1281       Temp src1 = get_alu_src(ctx, instr->src[1]);
1282       if (dst.regClass() == v1) {
1283          bld.vsub32(Definition(dst), src0, src1);
1284          break;
1285       }
1286
1287       Temp src00 = bld.tmp(src0.type(), 1);
1288       Temp src01 = bld.tmp(dst.type(), 1);
1289       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1290       Temp src10 = bld.tmp(src1.type(), 1);
1291       Temp src11 = bld.tmp(dst.type(), 1);
1292       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1293       if (dst.regClass() == s2) {
1294          Temp carry = bld.tmp(s1);
1295          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1296          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1297          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1298       } else if (dst.regClass() == v2) {
1299          Temp lower = bld.tmp(v1);
1300          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1301          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1302          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1303       } else {
1304          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1305          nir_print_instr(&instr->instr, stderr);
1306          fprintf(stderr, "\n");
1307       }
1308       break;
1309    }
1310    case nir_op_usub_borrow: {
1311       Temp src0 = get_alu_src(ctx, instr->src[0]);
1312       Temp src1 = get_alu_src(ctx, instr->src[1]);
1313       if (dst.regClass() == s1) {
1314          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1315          break;
1316       } else if (dst.regClass() == v1) {
1317          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1318          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1319          break;
1320       }
1321
1322       Temp src00 = bld.tmp(src0.type(), 1);
1323       Temp src01 = bld.tmp(dst.type(), 1);
1324       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1325       Temp src10 = bld.tmp(src1.type(), 1);
1326       Temp src11 = bld.tmp(dst.type(), 1);
1327       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1328       if (dst.regClass() == s2) {
1329          Temp borrow = bld.tmp(s1);
1330          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1331          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1332          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1333       } else if (dst.regClass() == v2) {
1334          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1335          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1336          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1337          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_imul: {
1346       if (dst.regClass() == v1) {
1347          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1349       } else if (dst.regClass() == s1) {
1350          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_umul_high: {
1359       if (dst.regClass() == v1) {
1360          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1361       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1362          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1) {
1364          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1365                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1366          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_imul_high: {
1375       if (dst.regClass() == v1) {
1376          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1377       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1378          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1) {
1380          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1381                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1382          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1383       } else {
1384          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1385          nir_print_instr(&instr->instr, stderr);
1386          fprintf(stderr, "\n");
1387       }
1388       break;
1389    }
1390    case nir_op_fmul: {
1391       if (dst.size() == 1) {
1392          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1393       } else if (dst.size() == 2) {
1394          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1395                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1396       } else {
1397          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1398          nir_print_instr(&instr->instr, stderr);
1399          fprintf(stderr, "\n");
1400       }
1401       break;
1402    }
1403    case nir_op_fadd: {
1404       if (dst.size() == 1) {
1405          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1406       } else if (dst.size() == 2) {
1407          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1408                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1409       } else {
1410          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1411          nir_print_instr(&instr->instr, stderr);
1412          fprintf(stderr, "\n");
1413       }
1414       break;
1415    }
1416    case nir_op_fsub: {
1417       Temp src0 = get_alu_src(ctx, instr->src[0]);
1418       Temp src1 = get_alu_src(ctx, instr->src[1]);
1419       if (dst.size() == 1) {
1420          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1421             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1422          else
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1424       } else if (dst.size() == 2) {
1425          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1426                                      get_alu_src(ctx, instr->src[0]),
1427                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1428          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1429          sub->neg[1] = true;
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_fmax: {
1438       if (dst.size() == 1) {
1439          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1440       } else if (dst.size() == 2) {
1441          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1442             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1443                                 get_alu_src(ctx, instr->src[0]),
1444                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1445             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1446          } else {
1447             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1448                      get_alu_src(ctx, instr->src[0]),
1449                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1450          }
1451       } else {
1452          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1453          nir_print_instr(&instr->instr, stderr);
1454          fprintf(stderr, "\n");
1455       }
1456       break;
1457    }
1458    case nir_op_fmin: {
1459       if (dst.size() == 1) {
1460          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1461       } else if (dst.size() == 2) {
1462          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1463             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1464                                 get_alu_src(ctx, instr->src[0]),
1465                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1466             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1467          } else {
1468             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1469                      get_alu_src(ctx, instr->src[0]),
1470                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1471          }
1472       } else {
1473          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1474          nir_print_instr(&instr->instr, stderr);
1475          fprintf(stderr, "\n");
1476       }
1477       break;
1478    }
1479    case nir_op_fmax3: {
1480       if (dst.size() == 1) {
1481          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1482       } else {
1483          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1484          nir_print_instr(&instr->instr, stderr);
1485          fprintf(stderr, "\n");
1486       }
1487       break;
1488    }
1489    case nir_op_fmin3: {
1490       if (dst.size() == 1) {
1491          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fmed3: {
1500       if (dst.size() == 1) {
1501          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1502       } else {
1503          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1504          nir_print_instr(&instr->instr, stderr);
1505          fprintf(stderr, "\n");
1506       }
1507       break;
1508    }
1509    case nir_op_umax3: {
1510       if (dst.size() == 1) {
1511          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1512       } else {
1513          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1514          nir_print_instr(&instr->instr, stderr);
1515          fprintf(stderr, "\n");
1516       }
1517       break;
1518    }
1519    case nir_op_umin3: {
1520       if (dst.size() == 1) {
1521          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1522       } else {
1523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1524          nir_print_instr(&instr->instr, stderr);
1525          fprintf(stderr, "\n");
1526       }
1527       break;
1528    }
1529    case nir_op_umed3: {
1530       if (dst.size() == 1) {
1531          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_imax3: {
1540       if (dst.size() == 1) {
1541          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_imin3: {
1550       if (dst.size() == 1) {
1551          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_imed3: {
1560       if (dst.size() == 1) {
1561          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1562       } else {
1563          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1564          nir_print_instr(&instr->instr, stderr);
1565          fprintf(stderr, "\n");
1566       }
1567       break;
1568    }
1569    case nir_op_cube_face_coord: {
1570       Temp in = get_alu_src(ctx, instr->src[0], 3);
1571       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1572                       emit_extract_vector(ctx, in, 1, v1),
1573                       emit_extract_vector(ctx, in, 2, v1) };
1574       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1575       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1576       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1577       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1578       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1579       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1580       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1581       break;
1582    }
1583    case nir_op_cube_face_index: {
1584       Temp in = get_alu_src(ctx, instr->src[0], 3);
1585       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1586                       emit_extract_vector(ctx, in, 1, v1),
1587                       emit_extract_vector(ctx, in, 2, v1) };
1588       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1589       break;
1590    }
1591    case nir_op_bcsel: {
1592       emit_bcsel(ctx, instr, dst);
1593       break;
1594    }
1595    case nir_op_frsq: {
1596       if (dst.size() == 1) {
1597          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1598       } else if (dst.size() == 2) {
1599          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1600       } else {
1601          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1602          nir_print_instr(&instr->instr, stderr);
1603          fprintf(stderr, "\n");
1604       }
1605       break;
1606    }
1607    case nir_op_fneg: {
1608       Temp src = get_alu_src(ctx, instr->src[0]);
1609       if (dst.size() == 1) {
1610          if (ctx->block->fp_mode.must_flush_denorms32)
1611             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1612          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1613       } else if (dst.size() == 2) {
1614          if (ctx->block->fp_mode.must_flush_denorms16_64)
1615             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1616          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1617          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1618          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1619          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1620       } else {
1621          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1622          nir_print_instr(&instr->instr, stderr);
1623          fprintf(stderr, "\n");
1624       }
1625       break;
1626    }
1627    case nir_op_fabs: {
1628       Temp src = get_alu_src(ctx, instr->src[0]);
1629       if (dst.size() == 1) {
1630          if (ctx->block->fp_mode.must_flush_denorms32)
1631             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1632          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1633       } else if (dst.size() == 2) {
1634          if (ctx->block->fp_mode.must_flush_denorms16_64)
1635             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1636          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1637          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1638          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1640       } else {
1641          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1642          nir_print_instr(&instr->instr, stderr);
1643          fprintf(stderr, "\n");
1644       }
1645       break;
1646    }
1647    case nir_op_fsat: {
1648       Temp src = get_alu_src(ctx, instr->src[0]);
1649       if (dst.size() == 1) {
1650          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1651          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1652          // TODO: confirm that this holds under any circumstances
1653       } else if (dst.size() == 2) {
1654          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1655          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1656          vop3->clamp = true;
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_flog2: {
1665       if (dst.size() == 1) {
1666          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_frcp: {
1675       if (dst.size() == 1) {
1676          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1677       } else if (dst.size() == 2) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1679       } else {
1680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1681          nir_print_instr(&instr->instr, stderr);
1682          fprintf(stderr, "\n");
1683       }
1684       break;
1685    }
1686    case nir_op_fexp2: {
1687       if (dst.size() == 1) {
1688          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1689       } else {
1690          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1691          nir_print_instr(&instr->instr, stderr);
1692          fprintf(stderr, "\n");
1693       }
1694       break;
1695    }
1696    case nir_op_fsqrt: {
1697       if (dst.size() == 1) {
1698          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1699       } else if (dst.size() == 2) {
1700          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1701       } else {
1702          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1703          nir_print_instr(&instr->instr, stderr);
1704          fprintf(stderr, "\n");
1705       }
1706       break;
1707    }
1708    case nir_op_ffract: {
1709       if (dst.size() == 1) {
1710          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1711       } else if (dst.size() == 2) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1713       } else {
1714          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1715          nir_print_instr(&instr->instr, stderr);
1716          fprintf(stderr, "\n");
1717       }
1718       break;
1719    }
1720    case nir_op_ffloor: {
1721       if (dst.size() == 1) {
1722          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1723       } else if (dst.size() == 2) {
1724          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1725       } else {
1726          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1727          nir_print_instr(&instr->instr, stderr);
1728          fprintf(stderr, "\n");
1729       }
1730       break;
1731    }
1732    case nir_op_fceil: {
1733       if (dst.size() == 1) {
1734          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1735       } else if (dst.size() == 2) {
1736          if (ctx->options->chip_class >= GFX7) {
1737             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1738          } else {
1739             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1740             Temp src0 = get_alu_src(ctx, instr->src[0]);
1741
1742             /* trunc = trunc(src0)
1743              * if (src0 > 0.0 && src0 != trunc)
1744              *    trunc += 1.0
1745              */
1746             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1747             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1748             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1749             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1750             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1751             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1752             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1753          }
1754       } else {
1755          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1756          nir_print_instr(&instr->instr, stderr);
1757          fprintf(stderr, "\n");
1758       }
1759       break;
1760    }
1761    case nir_op_ftrunc: {
1762       if (dst.size() == 1) {
1763          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1764       } else if (dst.size() == 2) {
1765          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1766       } else {
1767          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1768          nir_print_instr(&instr->instr, stderr);
1769          fprintf(stderr, "\n");
1770       }
1771       break;
1772    }
1773    case nir_op_fround_even: {
1774       if (dst.size() == 1) {
1775          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1776       } else if (dst.size() == 2) {
1777          if (ctx->options->chip_class >= GFX7) {
1778             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1779          } else {
1780             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1781             Temp src0 = get_alu_src(ctx, instr->src[0]);
1782
1783             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1784             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1785
1786             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1787             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1788             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1789             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1790             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1791             tmp = sub->definitions[0].getTemp();
1792
1793             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1794             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1795             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1796             Temp cond = vop3->definitions[0].getTemp();
1797
1798             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1799             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1800             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1801             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1802
1803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1804          }
1805       } else {
1806          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1807          nir_print_instr(&instr->instr, stderr);
1808          fprintf(stderr, "\n");
1809       }
1810       break;
1811    }
1812    case nir_op_fsin:
1813    case nir_op_fcos: {
1814       Temp src = get_alu_src(ctx, instr->src[0]);
1815       aco_ptr<Instruction> norm;
1816       if (dst.size() == 1) {
1817          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1818          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1819
1820          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1821          if (ctx->options->chip_class < GFX9)
1822             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1823
1824          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1825          bld.vop1(opcode, Definition(dst), tmp);
1826       } else {
1827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1828          nir_print_instr(&instr->instr, stderr);
1829          fprintf(stderr, "\n");
1830       }
1831       break;
1832    }
1833    case nir_op_ldexp: {
1834       if (dst.size() == 1) {
1835          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1836                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1837                   get_alu_src(ctx, instr->src[1]));
1838       } else if (dst.size() == 2) {
1839          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1840                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1841                   get_alu_src(ctx, instr->src[1]));
1842       } else {
1843          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1844          nir_print_instr(&instr->instr, stderr);
1845          fprintf(stderr, "\n");
1846       }
1847       break;
1848    }
1849    case nir_op_frexp_sig: {
1850       if (dst.size() == 1) {
1851          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1852                   get_alu_src(ctx, instr->src[0]));
1853       } else if (dst.size() == 2) {
1854          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1855                   get_alu_src(ctx, instr->src[0]));
1856       } else {
1857          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1858          nir_print_instr(&instr->instr, stderr);
1859          fprintf(stderr, "\n");
1860       }
1861       break;
1862    }
1863    case nir_op_frexp_exp: {
1864       if (instr->src[0].src.ssa->bit_size == 32) {
1865          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1866                   get_alu_src(ctx, instr->src[0]));
1867       } else if (instr->src[0].src.ssa->bit_size == 64) {
1868          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1869                   get_alu_src(ctx, instr->src[0]));
1870       } else {
1871          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1872          nir_print_instr(&instr->instr, stderr);
1873          fprintf(stderr, "\n");
1874       }
1875       break;
1876    }
1877    case nir_op_fsign: {
1878       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1879       if (dst.size() == 1) {
1880          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1881          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1882          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1884       } else if (dst.size() == 2) {
1885          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1886          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1887          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1888
1889          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1890          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1891          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1892
1893          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1894       } else {
1895          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1896          nir_print_instr(&instr->instr, stderr);
1897          fprintf(stderr, "\n");
1898       }
1899       break;
1900    }
1901    case nir_op_f2f32: {
1902       if (instr->src[0].src.ssa->bit_size == 64) {
1903          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_f2f64: {
1912       if (instr->src[0].src.ssa->bit_size == 32) {
1913          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1914       } else {
1915          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1916          nir_print_instr(&instr->instr, stderr);
1917          fprintf(stderr, "\n");
1918       }
1919       break;
1920    }
1921    case nir_op_i2f32: {
1922       assert(dst.size() == 1);
1923       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1924       break;
1925    }
1926    case nir_op_i2f64: {
1927       if (instr->src[0].src.ssa->bit_size == 32) {
1928          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1929       } else if (instr->src[0].src.ssa->bit_size == 64) {
1930          Temp src = get_alu_src(ctx, instr->src[0]);
1931          RegClass rc = RegClass(src.type(), 1);
1932          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1933          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1934          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1935          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1936          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1937          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1938
1939       } else {
1940          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1941          nir_print_instr(&instr->instr, stderr);
1942          fprintf(stderr, "\n");
1943       }
1944       break;
1945    }
1946    case nir_op_u2f32: {
1947       assert(dst.size() == 1);
1948       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1949       break;
1950    }
1951    case nir_op_u2f64: {
1952       if (instr->src[0].src.ssa->bit_size == 32) {
1953          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1954       } else if (instr->src[0].src.ssa->bit_size == 64) {
1955          Temp src = get_alu_src(ctx, instr->src[0]);
1956          RegClass rc = RegClass(src.type(), 1);
1957          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1958          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1959          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1960          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1961          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1962          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_f2i32: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32) {
1973          if (dst.type() == RegType::vgpr)
1974             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1975          else
1976             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1977                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1978
1979       } else if (instr->src[0].src.ssa->bit_size == 64) {
1980          if (dst.type() == RegType::vgpr)
1981             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1982          else
1983             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1984                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1985
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_f2u32: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       if (instr->src[0].src.ssa->bit_size == 32) {
1996          if (dst.type() == RegType::vgpr)
1997             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1998          else
1999             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2000                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2001
2002       } else if (instr->src[0].src.ssa->bit_size == 64) {
2003          if (dst.type() == RegType::vgpr)
2004             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2005          else
2006             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2007                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2008
2009       } else {
2010          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2011          nir_print_instr(&instr->instr, stderr);
2012          fprintf(stderr, "\n");
2013       }
2014       break;
2015    }
2016    case nir_op_f2i64: {
2017       Temp src = get_alu_src(ctx, instr->src[0]);
2018       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2019          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2020          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2021          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2022          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2023          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2024          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2025          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2026          Temp new_exponent = bld.tmp(v1);
2027          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2028          if (ctx->program->chip_class >= GFX8)
2029             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2030          else
2031             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2032          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2033          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2034          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2035          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2036          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2037          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2038          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2039          Temp new_lower = bld.tmp(v1);
2040          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2041          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2042          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2043
2044       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2045          if (src.type() == RegType::vgpr)
2046             src = bld.as_uniform(src);
2047          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2048          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2049          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2050          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2051          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2052          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2053          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2054          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2055          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2056          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2057          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2058          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2059          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2060          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2061          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2062          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2063          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2064          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2065          Temp borrow = bld.tmp(s1);
2066          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2067          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2068          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2069
2070       } else if (instr->src[0].src.ssa->bit_size == 64) {
2071          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2072          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2073          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2074          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2075          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2076          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2077          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2078          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2079          if (dst.type() == RegType::sgpr) {
2080             lower = bld.as_uniform(lower);
2081             upper = bld.as_uniform(upper);
2082          }
2083          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2084
2085       } else {
2086          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2087          nir_print_instr(&instr->instr, stderr);
2088          fprintf(stderr, "\n");
2089       }
2090       break;
2091    }
2092    case nir_op_f2u64: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2095          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2096          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2097          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2098          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2099          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2100          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2101          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2102          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2103          Temp new_exponent = bld.tmp(v1);
2104          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2105          if (ctx->program->chip_class >= GFX8)
2106             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2107          else
2108             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2109          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2110          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2111          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2112          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2114          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2116
2117       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2118          if (src.type() == RegType::vgpr)
2119             src = bld.as_uniform(src);
2120          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2121          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2122          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2123          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2124          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2125          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2126          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2127          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2128          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2129          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2130          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2131          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2132          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2133          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2134          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2135          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2136          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2137          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2138
2139       } else if (instr->src[0].src.ssa->bit_size == 64) {
2140          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2141          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2142          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2143          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2144          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2145          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2146          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2147          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2148          if (dst.type() == RegType::sgpr) {
2149             lower = bld.as_uniform(lower);
2150             upper = bld.as_uniform(upper);
2151          }
2152          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2153
2154       } else {
2155          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2156          nir_print_instr(&instr->instr, stderr);
2157          fprintf(stderr, "\n");
2158       }
2159       break;
2160    }
2161    case nir_op_b2f32: {
2162       Temp src = get_alu_src(ctx, instr->src[0]);
2163       assert(src.regClass() == bld.lm);
2164
2165       if (dst.regClass() == s1) {
2166          src = bool_to_scalar_condition(ctx, src);
2167          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2168       } else if (dst.regClass() == v1) {
2169          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2170       } else {
2171          unreachable("Wrong destination register class for nir_op_b2f32.");
2172       }
2173       break;
2174    }
2175    case nir_op_b2f64: {
2176       Temp src = get_alu_src(ctx, instr->src[0]);
2177       assert(src.regClass() == bld.lm);
2178
2179       if (dst.regClass() == s2) {
2180          src = bool_to_scalar_condition(ctx, src);
2181          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2182       } else if (dst.regClass() == v2) {
2183          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2184          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2185          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2186       } else {
2187          unreachable("Wrong destination register class for nir_op_b2f64.");
2188       }
2189       break;
2190    }
2191    case nir_op_i2i32: {
2192       Temp src = get_alu_src(ctx, instr->src[0]);
2193       if (instr->src[0].src.ssa->bit_size == 64) {
2194          /* we can actually just say dst = src, as it would map the lower register */
2195          emit_extract_vector(ctx, src, 0, dst);
2196       } else {
2197          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2198          nir_print_instr(&instr->instr, stderr);
2199          fprintf(stderr, "\n");
2200       }
2201       break;
2202    }
2203    case nir_op_u2u32: {
2204       Temp src = get_alu_src(ctx, instr->src[0]);
2205       if (instr->src[0].src.ssa->bit_size == 16) {
2206          if (dst.regClass() == s1) {
2207             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2208          } else {
2209             // TODO: do better with SDWA
2210             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2211          }
2212       } else if (instr->src[0].src.ssa->bit_size == 64) {
2213          /* we can actually just say dst = src, as it would map the lower register */
2214          emit_extract_vector(ctx, src, 0, dst);
2215       } else {
2216          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2217          nir_print_instr(&instr->instr, stderr);
2218          fprintf(stderr, "\n");
2219       }
2220       break;
2221    }
2222    case nir_op_i2i64: {
2223       Temp src = get_alu_src(ctx, instr->src[0]);
2224       if (src.regClass() == s1) {
2225          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2226          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2227       } else if (src.regClass() == v1) {
2228          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2230       } else {
2231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2232          nir_print_instr(&instr->instr, stderr);
2233          fprintf(stderr, "\n");
2234       }
2235       break;
2236    }
2237    case nir_op_u2u64: {
2238       Temp src = get_alu_src(ctx, instr->src[0]);
2239       if (instr->src[0].src.ssa->bit_size == 32) {
2240          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2241       } else {
2242          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2243          nir_print_instr(&instr->instr, stderr);
2244          fprintf(stderr, "\n");
2245       }
2246       break;
2247    }
2248    case nir_op_b2i32: {
2249       Temp src = get_alu_src(ctx, instr->src[0]);
2250       assert(src.regClass() == bld.lm);
2251
2252       if (dst.regClass() == s1) {
2253          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2254          bool_to_scalar_condition(ctx, src, dst);
2255       } else if (dst.regClass() == v1) {
2256          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2257       } else {
2258          unreachable("Invalid register class for b2i32");
2259       }
2260       break;
2261    }
2262    case nir_op_i2b1: {
2263       Temp src = get_alu_src(ctx, instr->src[0]);
2264       assert(dst.regClass() == bld.lm);
2265
2266       if (src.type() == RegType::vgpr) {
2267          assert(src.regClass() == v1 || src.regClass() == v2);
2268          assert(dst.regClass() == bld.lm);
2269          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2270                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2271       } else {
2272          assert(src.regClass() == s1 || src.regClass() == s2);
2273          Temp tmp;
2274          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2275             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2276          } else {
2277             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2278                            bld.scc(bld.def(s1)), Operand(0u), src);
2279          }
2280          bool_to_vector_condition(ctx, tmp, dst);
2281       }
2282       break;
2283    }
2284    case nir_op_pack_64_2x32_split: {
2285       Temp src0 = get_alu_src(ctx, instr->src[0]);
2286       Temp src1 = get_alu_src(ctx, instr->src[1]);
2287
2288       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2289       break;
2290    }
2291    case nir_op_unpack_64_2x32_split_x:
2292       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2293       break;
2294    case nir_op_unpack_64_2x32_split_y:
2295       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2296       break;
2297    case nir_op_pack_half_2x16: {
2298       Temp src = get_alu_src(ctx, instr->src[0], 2);
2299
2300       if (dst.regClass() == v1) {
2301          Temp src0 = bld.tmp(v1);
2302          Temp src1 = bld.tmp(v1);
2303          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2304          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2305             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2306          else
2307             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2308                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2309                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2310       } else {
2311          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2312          nir_print_instr(&instr->instr, stderr);
2313          fprintf(stderr, "\n");
2314       }
2315       break;
2316    }
2317    case nir_op_unpack_half_2x16_split_x: {
2318       if (dst.regClass() == v1) {
2319          Builder bld(ctx->program, ctx->block);
2320          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_unpack_half_2x16_split_y: {
2329       if (dst.regClass() == v1) {
2330          Builder bld(ctx->program, ctx->block);
2331          /* TODO: use SDWA here */
2332          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2333                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2334       } else {
2335          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2336          nir_print_instr(&instr->instr, stderr);
2337          fprintf(stderr, "\n");
2338       }
2339       break;
2340    }
2341    case nir_op_fquantize2f16: {
2342       Temp src = get_alu_src(ctx, instr->src[0]);
2343       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2344       Temp f32, cmp_res;
2345
2346       if (ctx->program->chip_class >= GFX8) {
2347          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2348          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2349          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2350       } else {
2351          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2352           * so compare the result and flush to 0 if it's smaller.
2353           */
2354          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2355          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2356          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2357          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2358          cmp_res = vop3->definitions[0].getTemp();
2359       }
2360
2361       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2362          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2363          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2364       } else {
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2366       }
2367       break;
2368    }
2369    case nir_op_bfm: {
2370       Temp bits = get_alu_src(ctx, instr->src[0]);
2371       Temp offset = get_alu_src(ctx, instr->src[1]);
2372
2373       if (dst.regClass() == s1) {
2374          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2375       } else if (dst.regClass() == v1) {
2376          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2377       } else {
2378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2379          nir_print_instr(&instr->instr, stderr);
2380          fprintf(stderr, "\n");
2381       }
2382       break;
2383    }
2384    case nir_op_bitfield_select: {
2385       /* (mask & insert) | (~mask & base) */
2386       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2387       Temp insert = get_alu_src(ctx, instr->src[1]);
2388       Temp base = get_alu_src(ctx, instr->src[2]);
2389
2390       /* dst = (insert & bitmask) | (base & ~bitmask) */
2391       if (dst.regClass() == s1) {
2392          aco_ptr<Instruction> sop2;
2393          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2394          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2395          Operand lhs;
2396          if (const_insert && const_bitmask) {
2397             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2398          } else {
2399             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2400             lhs = Operand(insert);
2401          }
2402
2403          Operand rhs;
2404          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2405          if (const_base && const_bitmask) {
2406             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2407          } else {
2408             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2409             rhs = Operand(base);
2410          }
2411
2412          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2413
2414       } else if (dst.regClass() == v1) {
2415          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2416             base = as_vgpr(ctx, base);
2417          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2418             insert = as_vgpr(ctx, insert);
2419
2420          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2421
2422       } else {
2423          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2424          nir_print_instr(&instr->instr, stderr);
2425          fprintf(stderr, "\n");
2426       }
2427       break;
2428    }
2429    case nir_op_ubfe:
2430    case nir_op_ibfe: {
2431       Temp base = get_alu_src(ctx, instr->src[0]);
2432       Temp offset = get_alu_src(ctx, instr->src[1]);
2433       Temp bits = get_alu_src(ctx, instr->src[2]);
2434
2435       if (dst.type() == RegType::sgpr) {
2436          Operand extract;
2437          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2438          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2439          if (const_offset && const_bits) {
2440             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2441             extract = Operand(const_extract);
2442          } else {
2443             Operand width;
2444             if (const_bits) {
2445                width = Operand(const_bits->u32 << 16);
2446             } else {
2447                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2448             }
2449             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2450          }
2451
2452          aco_opcode opcode;
2453          if (dst.regClass() == s1) {
2454             if (instr->op == nir_op_ubfe)
2455                opcode = aco_opcode::s_bfe_u32;
2456             else
2457                opcode = aco_opcode::s_bfe_i32;
2458          } else if (dst.regClass() == s2) {
2459             if (instr->op == nir_op_ubfe)
2460                opcode = aco_opcode::s_bfe_u64;
2461             else
2462                opcode = aco_opcode::s_bfe_i64;
2463          } else {
2464             unreachable("Unsupported BFE bit size");
2465          }
2466
2467          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2468
2469       } else {
2470          aco_opcode opcode;
2471          if (dst.regClass() == v1) {
2472             if (instr->op == nir_op_ubfe)
2473                opcode = aco_opcode::v_bfe_u32;
2474             else
2475                opcode = aco_opcode::v_bfe_i32;
2476          } else {
2477             unreachable("Unsupported BFE bit size");
2478          }
2479
2480          emit_vop3a_instruction(ctx, instr, opcode, dst);
2481       }
2482       break;
2483    }
2484    case nir_op_bit_count: {
2485       Temp src = get_alu_src(ctx, instr->src[0]);
2486       if (src.regClass() == s1) {
2487          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2488       } else if (src.regClass() == v1) {
2489          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2490       } else if (src.regClass() == v2) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2492                   emit_extract_vector(ctx, src, 1, v1),
2493                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2494                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2495       } else if (src.regClass() == s2) {
2496          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2497       } else {
2498          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2499          nir_print_instr(&instr->instr, stderr);
2500          fprintf(stderr, "\n");
2501       }
2502       break;
2503    }
2504    case nir_op_flt: {
2505       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2506       break;
2507    }
2508    case nir_op_fge: {
2509       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2510       break;
2511    }
2512    case nir_op_feq: {
2513       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2514       break;
2515    }
2516    case nir_op_fne: {
2517       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2518       break;
2519    }
2520    case nir_op_ilt: {
2521       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2522       break;
2523    }
2524    case nir_op_ige: {
2525       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2526       break;
2527    }
2528    case nir_op_ieq: {
2529       if (instr->src[0].src.ssa->bit_size == 1)
2530          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2531       else
2532          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2533                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2534       break;
2535    }
2536    case nir_op_ine: {
2537       if (instr->src[0].src.ssa->bit_size == 1)
2538          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2539       else
2540          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2541                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2542       break;
2543    }
2544    case nir_op_ult: {
2545       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2546       break;
2547    }
2548    case nir_op_uge: {
2549       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2550       break;
2551    }
2552    case nir_op_fddx:
2553    case nir_op_fddy:
2554    case nir_op_fddx_fine:
2555    case nir_op_fddy_fine:
2556    case nir_op_fddx_coarse:
2557    case nir_op_fddy_coarse: {
2558       Temp src = get_alu_src(ctx, instr->src[0]);
2559       uint16_t dpp_ctrl1, dpp_ctrl2;
2560       if (instr->op == nir_op_fddx_fine) {
2561          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2562          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2563       } else if (instr->op == nir_op_fddy_fine) {
2564          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2565          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2566       } else {
2567          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2568          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2569             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2570          else
2571             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2572       }
2573
2574       Temp tmp;
2575       if (ctx->program->chip_class >= GFX8) {
2576          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2577          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2578       } else {
2579          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2580          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2581          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2582       }
2583       emit_wqm(ctx, tmp, dst, true);
2584       break;
2585    }
2586    default:
2587       fprintf(stderr, "Unknown NIR ALU instr: ");
2588       nir_print_instr(&instr->instr, stderr);
2589       fprintf(stderr, "\n");
2590    }
2591 }
2592
2593 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2594 {
2595    Temp dst = get_ssa_temp(ctx, &instr->def);
2596
2597    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2598    // which get truncated the lsb if double and msb if int
2599    // for now, we only use s_mov_b64 with 64bit inline constants
2600    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2601    assert(dst.type() == RegType::sgpr);
2602
2603    Builder bld(ctx->program, ctx->block);
2604
2605    if (instr->def.bit_size == 1) {
2606       assert(dst.regClass() == bld.lm);
2607       int val = instr->value[0].b ? -1 : 0;
2608       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2609       bld.sop1(Builder::s_mov, Definition(dst), op);
2610    } else if (dst.size() == 1) {
2611       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2612    } else {
2613       assert(dst.size() != 1);
2614       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2615       if (instr->def.bit_size == 64)
2616          for (unsigned i = 0; i < dst.size(); i++)
2617             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2618       else {
2619          for (unsigned i = 0; i < dst.size(); i++)
2620             vec->operands[i] = Operand{instr->value[i].u32};
2621       }
2622       vec->definitions[0] = Definition(dst);
2623       ctx->block->instructions.emplace_back(std::move(vec));
2624    }
2625 }
2626
2627 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2628 {
2629    uint32_t new_mask = 0;
2630    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2631       if (mask & (1u << i))
2632          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2633    return new_mask;
2634 }
2635
2636 Operand load_lds_size_m0(isel_context *ctx)
2637 {
2638    /* TODO: m0 does not need to be initialized on GFX9+ */
2639    Builder bld(ctx->program, ctx->block);
2640    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2641 }
2642
2643 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2644               Temp address, unsigned base_offset, unsigned align)
2645 {
2646    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2647
2648    Builder bld(ctx->program, ctx->block);
2649
2650    Operand m = load_lds_size_m0(ctx);
2651
2652    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2653    unsigned bytes_read = 0;
2654    unsigned result_size = 0;
2655    unsigned total_bytes = num_components * elem_size_bytes;
2656    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2657    bool large_ds_read = ctx->options->chip_class >= GFX7;
2658
2659    while (bytes_read < total_bytes) {
2660       unsigned todo = total_bytes - bytes_read;
2661       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2662       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2663
2664       aco_opcode op = aco_opcode::last_opcode;
2665       bool read2 = false;
2666       if (todo >= 16 && aligned16 && large_ds_read) {
2667          op = aco_opcode::ds_read_b128;
2668          todo = 16;
2669       } else if (todo >= 16 && aligned8) {
2670          op = aco_opcode::ds_read2_b64;
2671          read2 = true;
2672          todo = 16;
2673       } else if (todo >= 12 && aligned16 && large_ds_read) {
2674          op = aco_opcode::ds_read_b96;
2675          todo = 12;
2676       } else if (todo >= 8 && aligned8) {
2677          op = aco_opcode::ds_read_b64;
2678          todo = 8;
2679       } else if (todo >= 8) {
2680          op = aco_opcode::ds_read2_b32;
2681          read2 = true;
2682          todo = 8;
2683       } else if (todo >= 4) {
2684          op = aco_opcode::ds_read_b32;
2685          todo = 4;
2686       } else {
2687          assert(false);
2688       }
2689       assert(todo % elem_size_bytes == 0);
2690       unsigned num_elements = todo / elem_size_bytes;
2691       unsigned offset = base_offset + bytes_read;
2692       unsigned max_offset = read2 ? 1019 : 65535;
2693
2694       Temp address_offset = address;
2695       if (offset > max_offset) {
2696          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2697          offset = bytes_read;
2698       }
2699       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2700
2701       Temp res;
2702       if (num_components == 1 && dst.type() == RegType::vgpr)
2703          res = dst;
2704       else
2705          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2706
2707       if (read2)
2708          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2709       else
2710          res = bld.ds(op, Definition(res), address_offset, m, offset);
2711
2712       if (num_components == 1) {
2713          assert(todo == total_bytes);
2714          if (dst.type() == RegType::sgpr)
2715             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2716          return;
2717       }
2718
2719       if (dst.type() == RegType::sgpr) {
2720          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2721          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2722          res = new_res;
2723       }
2724
2725       if (num_elements == 1) {
2726          result[result_size++] = res;
2727       } else {
2728          assert(res != dst && res.size() % num_elements == 0);
2729          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2730          split->operands[0] = Operand(res);
2731          for (unsigned i = 0; i < num_elements; i++)
2732             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2733          ctx->block->instructions.emplace_back(std::move(split));
2734       }
2735
2736       bytes_read += todo;
2737    }
2738
2739    assert(result_size == num_components && result_size > 1);
2740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2741    for (unsigned i = 0; i < result_size; i++)
2742       vec->operands[i] = Operand(result[i]);
2743    vec->definitions[0] = Definition(dst);
2744    ctx->block->instructions.emplace_back(std::move(vec));
2745    ctx->allocated_vec.emplace(dst.id(), result);
2746 }
2747
2748 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2749 {
2750    if (start == 0 && size == data.size())
2751       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2752
2753    unsigned size_hint = 1;
2754    auto it = ctx->allocated_vec.find(data.id());
2755    if (it != ctx->allocated_vec.end())
2756       size_hint = it->second[0].size();
2757    if (size % size_hint || start % size_hint)
2758       size_hint = 1;
2759
2760    start /= size_hint;
2761    size /= size_hint;
2762
2763    Temp elems[size];
2764    for (unsigned i = 0; i < size; i++)
2765       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2766
2767    if (size == 1)
2768       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2769
2770    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2771    for (unsigned i = 0; i < size; i++)
2772       vec->operands[i] = Operand(elems[i]);
2773    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2774    vec->definitions[0] = Definition(res);
2775    ctx->block->instructions.emplace_back(std::move(vec));
2776    return res;
2777 }
2778
2779 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2780 {
2781    Builder bld(ctx->program, ctx->block);
2782    unsigned bytes_written = 0;
2783    bool large_ds_write = ctx->options->chip_class >= GFX7;
2784
2785    while (bytes_written < total_size * 4) {
2786       unsigned todo = total_size * 4 - bytes_written;
2787       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2788       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2789
2790       aco_opcode op = aco_opcode::last_opcode;
2791       bool write2 = false;
2792       unsigned size = 0;
2793       if (todo >= 16 && aligned16 && large_ds_write) {
2794          op = aco_opcode::ds_write_b128;
2795          size = 4;
2796       } else if (todo >= 16 && aligned8) {
2797          op = aco_opcode::ds_write2_b64;
2798          write2 = true;
2799          size = 4;
2800       } else if (todo >= 12 && aligned16 && large_ds_write) {
2801          op = aco_opcode::ds_write_b96;
2802          size = 3;
2803       } else if (todo >= 8 && aligned8) {
2804          op = aco_opcode::ds_write_b64;
2805          size = 2;
2806       } else if (todo >= 8) {
2807          op = aco_opcode::ds_write2_b32;
2808          write2 = true;
2809          size = 2;
2810       } else if (todo >= 4) {
2811          op = aco_opcode::ds_write_b32;
2812          size = 1;
2813       } else {
2814          assert(false);
2815       }
2816
2817       unsigned offset = offset0 + offset1 + bytes_written;
2818       unsigned max_offset = write2 ? 1020 : 65535;
2819       Temp address_offset = address;
2820       if (offset > max_offset) {
2821          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2822          offset = offset1 + bytes_written;
2823       }
2824       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2825
2826       if (write2) {
2827          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2828          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2829          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2830       } else {
2831          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2832          bld.ds(op, address_offset, val, m, offset);
2833       }
2834
2835       bytes_written += size * 4;
2836    }
2837 }
2838
2839 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2840                Temp address, unsigned base_offset, unsigned align)
2841 {
2842    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2843
2844    Operand m = load_lds_size_m0(ctx);
2845
2846    /* we need at most two stores for 32bit variables */
2847    int start[2], count[2];
2848    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2849    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2850    assert(wrmask == 0);
2851
2852    /* one combined store is sufficient */
2853    if (count[0] == count[1]) {
2854       Builder bld(ctx->program, ctx->block);
2855
2856       Temp address_offset = address;
2857       if ((base_offset >> 2) + start[1] > 255) {
2858          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2859          base_offset = 0;
2860       }
2861
2862       assert(count[0] == 1);
2863       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2864       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2865       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2866       base_offset = base_offset / elem_size_bytes;
2867       bld.ds(op, address_offset, val0, val1, m,
2868              base_offset + start[0], base_offset + start[1]);
2869       return;
2870    }
2871
2872    for (unsigned i = 0; i < 2; i++) {
2873       if (count[i] == 0)
2874          continue;
2875
2876       unsigned elem_size_words = elem_size_bytes / 4;
2877       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2878                       base_offset, start[i] * elem_size_bytes, align);
2879    }
2880    return;
2881 }
2882
2883 void visit_store_vsgs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2884 {
2885    unsigned write_mask = nir_intrinsic_write_mask(instr);
2886    unsigned component = nir_intrinsic_component(instr);
2887    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2888    unsigned idx = (nir_intrinsic_base(instr) + component) * 4u;
2889    Operand offset(s1);
2890    Builder bld(ctx->program, ctx->block);
2891
2892    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2893    if (off_instr->type != nir_instr_type_load_const)
2894       offset = bld.v_mul24_imm(bld.def(v1), get_ssa_temp(ctx, instr->src[1].ssa), 16u);
2895    else
2896       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 16u;
2897
2898    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
2899    if (ctx->stage == vertex_es) {
2900       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
2901
2902       Temp elems[NIR_MAX_VEC_COMPONENTS * 2];
2903       if (elem_size_bytes == 8) {
2904          for (unsigned i = 0; i < src.size() / 2; i++) {
2905             Temp elem = emit_extract_vector(ctx, src, i, v2);
2906             elems[i*2] = bld.tmp(v1);
2907             elems[i*2+1] = bld.tmp(v1);
2908             bld.pseudo(aco_opcode::p_split_vector, Definition(elems[i*2]), Definition(elems[i*2+1]), elem);
2909          }
2910          write_mask = widen_mask(write_mask, 2);
2911          elem_size_bytes /= 2u;
2912       } else {
2913          for (unsigned i = 0; i < src.size(); i++)
2914             elems[i] = emit_extract_vector(ctx, src, i, v1);
2915       }
2916
2917       while (write_mask) {
2918          unsigned index = u_bit_scan(&write_mask);
2919          unsigned offset = index * elem_size_bytes;
2920          Temp elem = emit_extract_vector(ctx, src, index, RegClass(RegType::vgpr, elem_size_bytes / 4));
2921
2922          Operand vaddr_offset(v1);
2923          unsigned const_offset = idx + offset;
2924          if (const_offset >= 4096u) {
2925             vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
2926             const_offset %= 4096u;
2927          }
2928
2929          aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
2930          mtbuf->operands[0] = vaddr_offset;
2931          mtbuf->operands[1] = Operand(esgs_ring);
2932          mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->es2gs_offset));
2933          mtbuf->operands[3] = Operand(elem);
2934          mtbuf->offen = !vaddr_offset.isUndefined();
2935          mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
2936          mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
2937          mtbuf->offset = const_offset;
2938          mtbuf->glc = true;
2939          mtbuf->slc = true;
2940          mtbuf->barrier = barrier_none;
2941          mtbuf->can_reorder = true;
2942          bld.insert(std::move(mtbuf));
2943       }
2944    } else {
2945       unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
2946
2947       Temp vertex_idx = emit_mbcnt(ctx, bld.def(v1));
2948       Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
2949       vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), vertex_idx,
2950                             bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
2951
2952       Temp lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
2953       if (!offset.isUndefined())
2954          lds_base = bld.vadd32(bld.def(v1), offset, lds_base);
2955
2956       unsigned align = 1 << (ffs(itemsize) - 1);
2957       if (idx)
2958          align = std::min(align, 1u << (ffs(idx) - 1));
2959
2960       store_lds(ctx, elem_size_bytes, src, write_mask, lds_base, idx, align);
2961    }
2962 }
2963
2964 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2965 {
2966    if (ctx->stage == vertex_vs ||
2967        ctx->stage == fragment_fs ||
2968        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
2969       unsigned write_mask = nir_intrinsic_write_mask(instr);
2970       unsigned component = nir_intrinsic_component(instr);
2971       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2972       unsigned idx = nir_intrinsic_base(instr) + component;
2973
2974       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2975       if (off_instr->type != nir_instr_type_load_const) {
2976          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2977          nir_print_instr(off_instr, stderr);
2978          fprintf(stderr, "\n");
2979       }
2980       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2981
2982       if (instr->src[0].ssa->bit_size == 64)
2983          write_mask = widen_mask(write_mask, 2);
2984
2985       for (unsigned i = 0; i < 8; ++i) {
2986          if (write_mask & (1 << i)) {
2987             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
2988             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2989          }
2990          idx++;
2991       }
2992    } else if (ctx->stage == vertex_es ||
2993               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
2994       visit_store_vsgs_output(ctx, instr);
2995    } else {
2996       unreachable("Shader stage not implemented");
2997    }
2998 }
2999
3000 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3001 {
3002    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3003    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3004
3005    Builder bld(ctx->program, ctx->block);
3006    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3007    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3008 }
3009
3010 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3011 {
3012    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3013    for (unsigned i = 0; i < num_components; i++)
3014       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3015    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3016       assert(num_components == 4);
3017       Builder bld(ctx->program, ctx->block);
3018       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3019    }
3020
3021    for (Operand& op : vec->operands)
3022       op = op.isUndefined() ? Operand(0u) : op;
3023
3024    vec->definitions[0] = Definition(dst);
3025    ctx->block->instructions.emplace_back(std::move(vec));
3026    emit_split_vector(ctx, dst, num_components);
3027    return;
3028 }
3029
3030 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3031 {
3032    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3033    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3034    unsigned idx = nir_intrinsic_base(instr);
3035    unsigned component = nir_intrinsic_component(instr);
3036    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3037
3038    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3039    if (offset) {
3040       assert(offset->u32 == 0);
3041    } else {
3042       /* the lower 15bit of the prim_mask contain the offset into LDS
3043        * while the upper bits contain the number of prims */
3044       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3045       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3046       Builder bld(ctx->program, ctx->block);
3047       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3048       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3049       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3050       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3051       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3052    }
3053
3054    if (instr->dest.ssa.num_components == 1) {
3055       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3056    } else {
3057       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3058       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3059       {
3060          Temp tmp = {ctx->program->allocateId(), v1};
3061          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3062          vec->operands[i] = Operand(tmp);
3063       }
3064       vec->definitions[0] = Definition(dst);
3065       ctx->block->instructions.emplace_back(std::move(vec));
3066    }
3067 }
3068
3069 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3070                              unsigned offset, unsigned stride, unsigned channels)
3071 {
3072    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3073    if (vtx_info->chan_byte_size != 4 && channels == 3)
3074       return false;
3075    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3076           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3077 }
3078
3079 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3080                               unsigned offset, unsigned stride, unsigned *channels)
3081 {
3082    if (!vtx_info->chan_byte_size) {
3083       *channels = vtx_info->num_channels;
3084       return vtx_info->chan_format;
3085    }
3086
3087    unsigned num_channels = *channels;
3088    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3089       unsigned new_channels = num_channels + 1;
3090       /* first, assume more loads is worse and try using a larger data format */
3091       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3092          new_channels++;
3093          /* don't make the attribute potentially out-of-bounds */
3094          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3095             new_channels = 5;
3096       }
3097
3098       if (new_channels == 5) {
3099          /* then try decreasing load size (at the cost of more loads) */
3100          new_channels = *channels;
3101          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3102             new_channels--;
3103       }
3104
3105       if (new_channels < *channels)
3106          *channels = new_channels;
3107       num_channels = new_channels;
3108    }
3109
3110    switch (vtx_info->chan_format) {
3111    case V_008F0C_BUF_DATA_FORMAT_8:
3112       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3113                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3114    case V_008F0C_BUF_DATA_FORMAT_16:
3115       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3116                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3117    case V_008F0C_BUF_DATA_FORMAT_32:
3118       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3119                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3120    }
3121    unreachable("shouldn't reach here");
3122    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3123 }
3124
3125 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3126  * so we may need to fix it up. */
3127 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3128 {
3129    Builder bld(ctx->program, ctx->block);
3130
3131    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3132       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3133
3134    /* For the integer-like cases, do a natural sign extension.
3135     *
3136     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3137     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3138     * exponent.
3139     */
3140    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3141    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3142
3143    /* Convert back to the right type. */
3144    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3145       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3146       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3147       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3148    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3149       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3150    }
3151
3152    return alpha;
3153 }
3154
3155 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3156 {
3157    Builder bld(ctx->program, ctx->block);
3158    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3159    if (ctx->stage & sw_vs) {
3160
3161       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3162       if (off_instr->type != nir_instr_type_load_const) {
3163          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3164          nir_print_instr(off_instr, stderr);
3165          fprintf(stderr, "\n");
3166       }
3167       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3168
3169       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3170
3171       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3172       unsigned component = nir_intrinsic_component(instr);
3173       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3174       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3175       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3176       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3177
3178       unsigned dfmt = attrib_format & 0xf;
3179       unsigned nfmt = (attrib_format >> 4) & 0x7;
3180       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3181
3182       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3183       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3184       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3185       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3186       if (post_shuffle)
3187          num_channels = MAX2(num_channels, 3);
3188
3189       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3190       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3191
3192       Temp index;
3193       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3194          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3195          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3196          if (divisor) {
3197             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3198             if (divisor != 1) {
3199                Temp divided = bld.tmp(v1);
3200                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3201                index = bld.vadd32(bld.def(v1), start_instance, divided);
3202             } else {
3203                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3204             }
3205          } else {
3206             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3207          }
3208       } else {
3209          index = bld.vadd32(bld.def(v1),
3210                             get_arg(ctx, ctx->args->ac.base_vertex),
3211                             get_arg(ctx, ctx->args->ac.vertex_id));
3212       }
3213
3214       Temp channels[num_channels];
3215       unsigned channel_start = 0;
3216       bool direct_fetch = false;
3217
3218       /* skip unused channels at the start */
3219       if (vtx_info->chan_byte_size && !post_shuffle) {
3220          channel_start = ffs(mask) - 1;
3221          for (unsigned i = 0; i < channel_start; i++)
3222             channels[i] = Temp(0, s1);
3223       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3224          num_channels = 3 - (ffs(mask) - 1);
3225       }
3226
3227       /* load channels */
3228       while (channel_start < num_channels) {
3229          unsigned fetch_size = num_channels - channel_start;
3230          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3231
3232          /* use MUBUF when possible to avoid possible alignment issues */
3233          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3234          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3235                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3236                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3237                           vtx_info->chan_byte_size == 4;
3238          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3239          if (!use_mubuf)
3240             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3241
3242          Temp fetch_index = index;
3243          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3244             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3245             fetch_offset = fetch_offset % attrib_stride;
3246          }
3247
3248          Operand soffset(0u);
3249          if (fetch_offset >= 4096) {
3250             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3251             fetch_offset %= 4096;
3252          }
3253
3254          aco_opcode opcode;
3255          switch (fetch_size) {
3256          case 1:
3257             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3258             break;
3259          case 2:
3260             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3261             break;
3262          case 3:
3263             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3264             break;
3265          case 4:
3266             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3267             break;
3268          default:
3269             unreachable("Unimplemented load_input vector size");
3270          }
3271
3272          Temp fetch_dst;
3273          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3274              (alpha_adjust == RADV_ALPHA_ADJUST_NONE || num_channels <= 3)) {
3275             direct_fetch = true;
3276             fetch_dst = dst;
3277          } else {
3278             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3279          }
3280
3281          if (use_mubuf) {
3282             Instruction *mubuf = bld.mubuf(opcode,
3283                                            Definition(fetch_dst), fetch_index, list, soffset,
3284                                            fetch_offset, false, true).instr;
3285             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3286          } else {
3287             Instruction *mtbuf = bld.mtbuf(opcode,
3288                                            Definition(fetch_dst), fetch_index, list, soffset,
3289                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3290             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3291          }
3292
3293          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3294
3295          if (fetch_size == 1) {
3296             channels[channel_start] = fetch_dst;
3297          } else {
3298             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3299                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3300          }
3301
3302          channel_start += fetch_size;
3303       }
3304
3305       if (!direct_fetch) {
3306          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3307                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3308
3309          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3310          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3311          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3312
3313          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3314          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3315          unsigned num_temp = 0;
3316          for (unsigned i = 0; i < dst.size(); i++) {
3317             unsigned idx = i + component;
3318             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3319                Temp channel = channels[swizzle[idx]];
3320                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3321                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3322                vec->operands[i] = Operand(channel);
3323
3324                num_temp++;
3325                elems[i] = channel;
3326             } else if (is_float && idx == 3) {
3327                vec->operands[i] = Operand(0x3f800000u);
3328             } else if (!is_float && idx == 3) {
3329                vec->operands[i] = Operand(1u);
3330             } else {
3331                vec->operands[i] = Operand(0u);
3332             }
3333          }
3334          vec->definitions[0] = Definition(dst);
3335          ctx->block->instructions.emplace_back(std::move(vec));
3336          emit_split_vector(ctx, dst, dst.size());
3337
3338          if (num_temp == dst.size())
3339             ctx->allocated_vec.emplace(dst.id(), elems);
3340       }
3341    } else if (ctx->stage == fragment_fs) {
3342       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3343       if (off_instr->type != nir_instr_type_load_const ||
3344           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3345          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3346          nir_print_instr(off_instr, stderr);
3347          fprintf(stderr, "\n");
3348       }
3349
3350       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3351       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3352       if (offset) {
3353          assert(offset->u32 == 0);
3354       } else {
3355          /* the lower 15bit of the prim_mask contain the offset into LDS
3356           * while the upper bits contain the number of prims */
3357          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3358          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3359          Builder bld(ctx->program, ctx->block);
3360          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3361          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3362          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3363          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3364          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3365       }
3366
3367       unsigned idx = nir_intrinsic_base(instr);
3368       unsigned component = nir_intrinsic_component(instr);
3369
3370       if (dst.size() == 1) {
3371          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3372       } else {
3373          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3374          for (unsigned i = 0; i < dst.size(); i++)
3375             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3376          vec->definitions[0] = Definition(dst);
3377          bld.insert(std::move(vec));
3378       }
3379
3380    } else {
3381       unreachable("Shader stage not implemented");
3382    }
3383 }
3384
3385 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3386 {
3387    assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
3388    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3389
3390    Builder bld(ctx->program, ctx->block);
3391    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3392
3393    Temp offset = Temp();
3394    if (instr->src[0].ssa->parent_instr->type != nir_instr_type_load_const) {
3395       /* better code could be created, but this case probably doesn't happen
3396        * much in practice */
3397       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
3398       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3399          Temp elem;
3400          if (ctx->stage == vertex_geometry_gs) {
3401             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3402             if (i % 2u)
3403                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3404          } else {
3405             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3406          }
3407          if (offset.id()) {
3408             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(s2)),
3409                                  Operand(i), indirect_vertex);
3410             offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), offset, elem, cond);
3411          } else {
3412             offset = elem;
3413          }
3414       }
3415       if (ctx->stage == vertex_geometry_gs)
3416          offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), offset);
3417    } else {
3418       unsigned vertex = nir_src_as_uint(instr->src[0]);
3419       if (ctx->stage == vertex_geometry_gs)
3420          offset = bld.vop3(
3421          aco_opcode::v_bfe_u32, bld.def(v1), get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3422             Operand((vertex % 2u) * 16u), Operand(16u));
3423       else
3424          offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3425    }
3426
3427    unsigned const_offset = nir_intrinsic_base(instr);
3428    const_offset += nir_intrinsic_component(instr);
3429
3430    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3431    if (off_instr->type != nir_instr_type_load_const) {
3432       Temp indirect_offset = get_ssa_temp(ctx, instr->src[1].ssa);
3433       offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u),
3434                         bld.vadd32(bld.def(v1), indirect_offset, offset));
3435    } else {
3436       const_offset += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3437    }
3438    const_offset *= 4u;
3439
3440    offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), offset);
3441
3442    unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3443
3444    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3445    if (ctx->stage == geometry_gs) {
3446       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3447
3448       const_offset *= ctx->program->wave_size;
3449
3450       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3451       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3452          aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1)};
3453       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++) {
3454          Temp subelems[2];
3455          for (unsigned j = 0; j < elem_size_bytes / 4; j++) {
3456             Operand soffset(0u);
3457             if (const_offset >= 4096u)
3458                soffset = bld.copy(bld.def(s1), Operand(const_offset / 4096u * 4096u));
3459
3460             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
3461             mubuf->definitions[0] = bld.def(v1);
3462             subelems[j] = mubuf->definitions[0].getTemp();
3463             mubuf->operands[0] = Operand(offset);
3464             mubuf->operands[1] = Operand(esgs_ring);
3465             mubuf->operands[2] = Operand(soffset);
3466             mubuf->offen = true;
3467             mubuf->offset = const_offset % 4096u;
3468             mubuf->glc = true;
3469             mubuf->dlc = ctx->options->chip_class >= GFX10;
3470             mubuf->barrier = barrier_none;
3471             mubuf->can_reorder = true;
3472             bld.insert(std::move(mubuf));
3473
3474             const_offset += ctx->program->wave_size * 4u;
3475          }
3476
3477          if (elem_size_bytes == 4)
3478             elems[i] = subelems[0];
3479          else
3480             elems[i] = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), subelems[0], subelems[1]);
3481          vec->operands[i] = Operand(elems[i]);
3482       }
3483       vec->definitions[0] = Definition(dst);
3484       ctx->block->instructions.emplace_back(std::move(vec));
3485       ctx->allocated_vec.emplace(dst.id(), elems);
3486    } else {
3487       unsigned align = 16; /* alignment of indirect offset */
3488       align = std::min(align, 1u << (ffs(itemsize) - 1));
3489       if (const_offset)
3490          align = std::min(align, 1u << (ffs(const_offset) - 1));
3491
3492       load_lds(ctx, elem_size_bytes, dst, offset, const_offset, align);
3493    }
3494 }
3495
3496 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3497 {
3498    if (ctx->program->info->need_indirect_descriptor_sets) {
3499       Builder bld(ctx->program, ctx->block);
3500       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3501       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3502       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3503    }
3504
3505    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3506 }
3507
3508
3509 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3510 {
3511    Builder bld(ctx->program, ctx->block);
3512    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3513    if (!ctx->divergent_vals[instr->dest.ssa.index])
3514       index = bld.as_uniform(index);
3515    unsigned desc_set = nir_intrinsic_desc_set(instr);
3516    unsigned binding = nir_intrinsic_binding(instr);
3517
3518    Temp desc_ptr;
3519    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3520    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3521    unsigned offset = layout->binding[binding].offset;
3522    unsigned stride;
3523    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3524        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3525       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3526       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3527       offset = pipeline_layout->push_constant_size + 16 * idx;
3528       stride = 16;
3529    } else {
3530       desc_ptr = load_desc_ptr(ctx, desc_set);
3531       stride = layout->binding[binding].size;
3532    }
3533
3534    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3535    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3536    if (stride != 1) {
3537       if (nir_const_index) {
3538          const_index = const_index * stride;
3539       } else if (index.type() == RegType::vgpr) {
3540          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3541          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3542       } else {
3543          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3544       }
3545    }
3546    if (offset) {
3547       if (nir_const_index) {
3548          const_index = const_index + offset;
3549       } else if (index.type() == RegType::vgpr) {
3550          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3551       } else {
3552          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3553       }
3554    }
3555
3556    if (nir_const_index && const_index == 0) {
3557       index = desc_ptr;
3558    } else if (index.type() == RegType::vgpr) {
3559       index = bld.vadd32(bld.def(v1),
3560                          nir_const_index ? Operand(const_index) : Operand(index),
3561                          Operand(desc_ptr));
3562    } else {
3563       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3564                        nir_const_index ? Operand(const_index) : Operand(index),
3565                        Operand(desc_ptr));
3566    }
3567
3568    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3569 }
3570
3571 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3572                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3573 {
3574    Builder bld(ctx->program, ctx->block);
3575
3576    unsigned num_bytes = dst.size() * 4;
3577    bool dlc = glc && ctx->options->chip_class >= GFX10;
3578
3579    aco_opcode op;
3580    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3581       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3582       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3583       unsigned const_offset = 0;
3584
3585       Temp lower = Temp();
3586       if (num_bytes > 16) {
3587          assert(num_components == 3 || num_components == 4);
3588          op = aco_opcode::buffer_load_dwordx4;
3589          lower = bld.tmp(v4);
3590          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3591          mubuf->definitions[0] = Definition(lower);
3592          mubuf->operands[0] = vaddr;
3593          mubuf->operands[1] = Operand(rsrc);
3594          mubuf->operands[2] = soffset;
3595          mubuf->offen = (offset.type() == RegType::vgpr);
3596          mubuf->glc = glc;
3597          mubuf->dlc = dlc;
3598          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3599          mubuf->can_reorder = readonly;
3600          bld.insert(std::move(mubuf));
3601          emit_split_vector(ctx, lower, 2);
3602          num_bytes -= 16;
3603          const_offset = 16;
3604       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3605          /* GFX6 doesn't support loading vec3, expand to vec4. */
3606          num_bytes = 16;
3607       }
3608
3609       switch (num_bytes) {
3610          case 4:
3611             op = aco_opcode::buffer_load_dword;
3612             break;
3613          case 8:
3614             op = aco_opcode::buffer_load_dwordx2;
3615             break;
3616          case 12:
3617             assert(ctx->options->chip_class > GFX6);
3618             op = aco_opcode::buffer_load_dwordx3;
3619             break;
3620          case 16:
3621             op = aco_opcode::buffer_load_dwordx4;
3622             break;
3623          default:
3624             unreachable("Load SSBO not implemented for this size.");
3625       }
3626       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3627       mubuf->operands[0] = vaddr;
3628       mubuf->operands[1] = Operand(rsrc);
3629       mubuf->operands[2] = soffset;
3630       mubuf->offen = (offset.type() == RegType::vgpr);
3631       mubuf->glc = glc;
3632       mubuf->dlc = dlc;
3633       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3634       mubuf->can_reorder = readonly;
3635       mubuf->offset = const_offset;
3636       aco_ptr<Instruction> instr = std::move(mubuf);
3637
3638       if (dst.size() > 4) {
3639          assert(lower != Temp());
3640          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3641          instr->definitions[0] = Definition(upper);
3642          bld.insert(std::move(instr));
3643          if (dst.size() == 8)
3644             emit_split_vector(ctx, upper, 2);
3645          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3646          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3647          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3648          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3649          if (dst.size() == 8)
3650             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3651       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3652          Temp vec = bld.tmp(v4);
3653          instr->definitions[0] = Definition(vec);
3654          bld.insert(std::move(instr));
3655          emit_split_vector(ctx, vec, 4);
3656
3657          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3658          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3659          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3660          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3661       }
3662
3663       if (dst.type() == RegType::sgpr) {
3664          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3665          instr->definitions[0] = Definition(vec);
3666          bld.insert(std::move(instr));
3667          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3668       } else {
3669          instr->definitions[0] = Definition(dst);
3670          bld.insert(std::move(instr));
3671          emit_split_vector(ctx, dst, num_components);
3672       }
3673    } else {
3674       switch (num_bytes) {
3675          case 4:
3676             op = aco_opcode::s_buffer_load_dword;
3677             break;
3678          case 8:
3679             op = aco_opcode::s_buffer_load_dwordx2;
3680             break;
3681          case 12:
3682          case 16:
3683             op = aco_opcode::s_buffer_load_dwordx4;
3684             break;
3685          case 24:
3686          case 32:
3687             op = aco_opcode::s_buffer_load_dwordx8;
3688             break;
3689          default:
3690             unreachable("Load SSBO not implemented for this size.");
3691       }
3692       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3693       load->operands[0] = Operand(rsrc);
3694       load->operands[1] = Operand(bld.as_uniform(offset));
3695       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3696       load->definitions[0] = Definition(dst);
3697       load->glc = glc;
3698       load->dlc = dlc;
3699       load->barrier = readonly ? barrier_none : barrier_buffer;
3700       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3701       assert(ctx->options->chip_class >= GFX8 || !glc);
3702
3703       /* trim vector */
3704       if (dst.size() == 3) {
3705          Temp vec = bld.tmp(s4);
3706          load->definitions[0] = Definition(vec);
3707          bld.insert(std::move(load));
3708          emit_split_vector(ctx, vec, 4);
3709
3710          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3711                     emit_extract_vector(ctx, vec, 0, s1),
3712                     emit_extract_vector(ctx, vec, 1, s1),
3713                     emit_extract_vector(ctx, vec, 2, s1));
3714       } else if (dst.size() == 6) {
3715          Temp vec = bld.tmp(s8);
3716          load->definitions[0] = Definition(vec);
3717          bld.insert(std::move(load));
3718          emit_split_vector(ctx, vec, 4);
3719
3720          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3721                     emit_extract_vector(ctx, vec, 0, s2),
3722                     emit_extract_vector(ctx, vec, 1, s2),
3723                     emit_extract_vector(ctx, vec, 2, s2));
3724       } else {
3725          bld.insert(std::move(load));
3726       }
3727       emit_split_vector(ctx, dst, num_components);
3728    }
3729 }
3730
3731 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3732 {
3733    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3734    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3735
3736    Builder bld(ctx->program, ctx->block);
3737
3738    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3739    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3740    unsigned binding = nir_intrinsic_binding(idx_instr);
3741    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3742
3743    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3744       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3745                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3746                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3747                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3748       if (ctx->options->chip_class >= GFX10) {
3749          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3750                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3751                       S_008F0C_RESOURCE_LEVEL(1);
3752       } else {
3753          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3754                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3755       }
3756       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3757                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3758                                      Operand(0xFFFFFFFFu),
3759                                      Operand(desc_type));
3760       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3761                         rsrc, upper_dwords);
3762    } else {
3763       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3764       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3765    }
3766
3767    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3768 }
3769
3770 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3771 {
3772    Builder bld(ctx->program, ctx->block);
3773    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3774
3775    unsigned offset = nir_intrinsic_base(instr);
3776    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3777    if (index_cv && instr->dest.ssa.bit_size == 32) {
3778
3779       unsigned count = instr->dest.ssa.num_components;
3780       unsigned start = (offset + index_cv->u32) / 4u;
3781       start -= ctx->args->ac.base_inline_push_consts;
3782       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3783          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3784          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3785          for (unsigned i = 0; i < count; ++i) {
3786             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3787             vec->operands[i] = Operand{elems[i]};
3788          }
3789          vec->definitions[0] = Definition(dst);
3790          ctx->block->instructions.emplace_back(std::move(vec));
3791          ctx->allocated_vec.emplace(dst.id(), elems);
3792          return;
3793       }
3794    }
3795
3796    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3797    if (offset != 0) // TODO check if index != 0 as well
3798       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3799    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3800    Temp vec = dst;
3801    bool trim = false;
3802    aco_opcode op;
3803
3804    switch (dst.size()) {
3805    case 1:
3806       op = aco_opcode::s_load_dword;
3807       break;
3808    case 2:
3809       op = aco_opcode::s_load_dwordx2;
3810       break;
3811    case 3:
3812       vec = bld.tmp(s4);
3813       trim = true;
3814    case 4:
3815       op = aco_opcode::s_load_dwordx4;
3816       break;
3817    case 6:
3818       vec = bld.tmp(s8);
3819       trim = true;
3820    case 8:
3821       op = aco_opcode::s_load_dwordx8;
3822       break;
3823    default:
3824       unreachable("unimplemented or forbidden load_push_constant.");
3825    }
3826
3827    bld.smem(op, Definition(vec), ptr, index);
3828
3829    if (trim) {
3830       emit_split_vector(ctx, vec, 4);
3831       RegClass rc = dst.size() == 3 ? s1 : s2;
3832       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3833                  emit_extract_vector(ctx, vec, 0, rc),
3834                  emit_extract_vector(ctx, vec, 1, rc),
3835                  emit_extract_vector(ctx, vec, 2, rc));
3836
3837    }
3838    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3839 }
3840
3841 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3842 {
3843    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3844
3845    Builder bld(ctx->program, ctx->block);
3846
3847    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3848                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3849                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3850                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3851    if (ctx->options->chip_class >= GFX10) {
3852       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3853                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3854                    S_008F0C_RESOURCE_LEVEL(1);
3855    } else {
3856       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3857                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3858    }
3859
3860    unsigned base = nir_intrinsic_base(instr);
3861    unsigned range = nir_intrinsic_range(instr);
3862
3863    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3864    if (base && offset.type() == RegType::sgpr)
3865       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3866    else if (base && offset.type() == RegType::vgpr)
3867       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3868
3869    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3870                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3871                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3872                           Operand(desc_type));
3873
3874    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3875 }
3876
3877 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3878 {
3879    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3880       ctx->cf_info.exec_potentially_empty = true;
3881
3882    ctx->program->needs_exact = true;
3883
3884    // TODO: optimize uniform conditions
3885    Builder bld(ctx->program, ctx->block);
3886    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3887    assert(src.regClass() == bld.lm);
3888    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3889    bld.pseudo(aco_opcode::p_discard_if, src);
3890    ctx->block->kind |= block_kind_uses_discard_if;
3891    return;
3892 }
3893
3894 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3895 {
3896    Builder bld(ctx->program, ctx->block);
3897
3898    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3899       ctx->cf_info.exec_potentially_empty = true;
3900
3901    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3902                     ctx->cf_info.parent_loop.has_divergent_continue;
3903
3904    if (ctx->block->loop_nest_depth &&
3905        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3906       /* we handle discards the same way as jump instructions */
3907       append_logical_end(ctx->block);
3908
3909       /* in loops, discard behaves like break */
3910       Block *linear_target = ctx->cf_info.parent_loop.exit;
3911       ctx->block->kind |= block_kind_discard;
3912
3913       if (!divergent) {
3914          /* uniform discard - loop ends here */
3915          assert(nir_instr_is_last(&instr->instr));
3916          ctx->block->kind |= block_kind_uniform;
3917          ctx->cf_info.has_branch = true;
3918          bld.branch(aco_opcode::p_branch);
3919          add_linear_edge(ctx->block->index, linear_target);
3920          return;
3921       }
3922
3923       /* we add a break right behind the discard() instructions */
3924       ctx->block->kind |= block_kind_break;
3925       unsigned idx = ctx->block->index;
3926
3927       /* remove critical edges from linear CFG */
3928       bld.branch(aco_opcode::p_branch);
3929       Block* break_block = ctx->program->create_and_insert_block();
3930       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3931       break_block->kind |= block_kind_uniform;
3932       add_linear_edge(idx, break_block);
3933       add_linear_edge(break_block->index, linear_target);
3934       bld.reset(break_block);
3935       bld.branch(aco_opcode::p_branch);
3936
3937       Block* continue_block = ctx->program->create_and_insert_block();
3938       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3939       add_linear_edge(idx, continue_block);
3940       append_logical_start(continue_block);
3941       ctx->block = continue_block;
3942
3943       return;
3944    }
3945
3946    /* it can currently happen that NIR doesn't remove the unreachable code */
3947    if (!nir_instr_is_last(&instr->instr)) {
3948       ctx->program->needs_exact = true;
3949       /* save exec somewhere temporarily so that it doesn't get
3950        * overwritten before the discard from outer exec masks */
3951       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3952       bld.pseudo(aco_opcode::p_discard_if, cond);
3953       ctx->block->kind |= block_kind_uses_discard_if;
3954       return;
3955    }
3956
3957    /* This condition is incorrect for uniformly branched discards in a loop
3958     * predicated by a divergent condition, but the above code catches that case
3959     * and the discard would end up turning into a discard_if.
3960     * For example:
3961     * if (divergent) {
3962     *    while (...) {
3963     *       if (uniform) {
3964     *          discard;
3965     *       }
3966     *    }
3967     * }
3968     */
3969    if (!ctx->cf_info.parent_if.is_divergent) {
3970       /* program just ends here */
3971       ctx->block->kind |= block_kind_uniform;
3972       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3973               0 /* enabled mask */, 9 /* dest */,
3974               false /* compressed */, true/* done */, true /* valid mask */);
3975       bld.sopp(aco_opcode::s_endpgm);
3976       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3977    } else {
3978       ctx->block->kind |= block_kind_discard;
3979       /* branch and linear edge is added by visit_if() */
3980    }
3981 }
3982
3983 enum aco_descriptor_type {
3984    ACO_DESC_IMAGE,
3985    ACO_DESC_FMASK,
3986    ACO_DESC_SAMPLER,
3987    ACO_DESC_BUFFER,
3988    ACO_DESC_PLANE_0,
3989    ACO_DESC_PLANE_1,
3990    ACO_DESC_PLANE_2,
3991 };
3992
3993 static bool
3994 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3995    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3996       return false;
3997    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3998    return dim == ac_image_cube ||
3999           dim == ac_image_1darray ||
4000           dim == ac_image_2darray ||
4001           dim == ac_image_2darraymsaa;
4002 }
4003
4004 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4005                       enum aco_descriptor_type desc_type,
4006                       const nir_tex_instr *tex_instr, bool image, bool write)
4007 {
4008 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4009    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4010    if (it != ctx->tex_desc.end())
4011       return it->second;
4012 */
4013    Temp index = Temp();
4014    bool index_set = false;
4015    unsigned constant_index = 0;
4016    unsigned descriptor_set;
4017    unsigned base_index;
4018    Builder bld(ctx->program, ctx->block);
4019
4020    if (!deref_instr) {
4021       assert(tex_instr && !image);
4022       descriptor_set = 0;
4023       base_index = tex_instr->sampler_index;
4024    } else {
4025       while(deref_instr->deref_type != nir_deref_type_var) {
4026          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4027          if (!array_size)
4028             array_size = 1;
4029
4030          assert(deref_instr->deref_type == nir_deref_type_array);
4031          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4032          if (const_value) {
4033             constant_index += array_size * const_value->u32;
4034          } else {
4035             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4036             if (indirect.type() == RegType::vgpr)
4037                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4038
4039             if (array_size != 1)
4040                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4041
4042             if (!index_set) {
4043                index = indirect;
4044                index_set = true;
4045             } else {
4046                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4047             }
4048          }
4049
4050          deref_instr = nir_src_as_deref(deref_instr->parent);
4051       }
4052       descriptor_set = deref_instr->var->data.descriptor_set;
4053       base_index = deref_instr->var->data.binding;
4054    }
4055
4056    Temp list = load_desc_ptr(ctx, descriptor_set);
4057    list = convert_pointer_to_64_bit(ctx, list);
4058
4059    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4060    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4061    unsigned offset = binding->offset;
4062    unsigned stride = binding->size;
4063    aco_opcode opcode;
4064    RegClass type;
4065
4066    assert(base_index < layout->binding_count);
4067
4068    switch (desc_type) {
4069    case ACO_DESC_IMAGE:
4070       type = s8;
4071       opcode = aco_opcode::s_load_dwordx8;
4072       break;
4073    case ACO_DESC_FMASK:
4074       type = s8;
4075       opcode = aco_opcode::s_load_dwordx8;
4076       offset += 32;
4077       break;
4078    case ACO_DESC_SAMPLER:
4079       type = s4;
4080       opcode = aco_opcode::s_load_dwordx4;
4081       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4082          offset += radv_combined_image_descriptor_sampler_offset(binding);
4083       break;
4084    case ACO_DESC_BUFFER:
4085       type = s4;
4086       opcode = aco_opcode::s_load_dwordx4;
4087       break;
4088    case ACO_DESC_PLANE_0:
4089    case ACO_DESC_PLANE_1:
4090       type = s8;
4091       opcode = aco_opcode::s_load_dwordx8;
4092       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4093       break;
4094    case ACO_DESC_PLANE_2:
4095       type = s4;
4096       opcode = aco_opcode::s_load_dwordx4;
4097       offset += 64;
4098       break;
4099    default:
4100       unreachable("invalid desc_type\n");
4101    }
4102
4103    offset += constant_index * stride;
4104
4105    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4106       (!index_set || binding->immutable_samplers_equal)) {
4107       if (binding->immutable_samplers_equal)
4108          constant_index = 0;
4109
4110       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4111       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4112                         Operand(samplers[constant_index * 4 + 0]),
4113                         Operand(samplers[constant_index * 4 + 1]),
4114                         Operand(samplers[constant_index * 4 + 2]),
4115                         Operand(samplers[constant_index * 4 + 3]));
4116    }
4117
4118    Operand off;
4119    if (!index_set) {
4120       off = bld.copy(bld.def(s1), Operand(offset));
4121    } else {
4122       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4123                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4124    }
4125
4126    Temp res = bld.smem(opcode, bld.def(type), list, off);
4127
4128    if (desc_type == ACO_DESC_PLANE_2) {
4129       Temp components[8];
4130       for (unsigned i = 0; i < 8; i++)
4131          components[i] = bld.tmp(s1);
4132       bld.pseudo(aco_opcode::p_split_vector,
4133                  Definition(components[0]),
4134                  Definition(components[1]),
4135                  Definition(components[2]),
4136                  Definition(components[3]),
4137                  res);
4138
4139       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4140       bld.pseudo(aco_opcode::p_split_vector,
4141                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4142                  Definition(components[4]),
4143                  Definition(components[5]),
4144                  Definition(components[6]),
4145                  Definition(components[7]),
4146                  desc2);
4147
4148       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4149                        components[0], components[1], components[2], components[3],
4150                        components[4], components[5], components[6], components[7]);
4151    }
4152
4153    return res;
4154 }
4155
4156 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4157 {
4158    switch (dim) {
4159    case GLSL_SAMPLER_DIM_BUF:
4160       return 1;
4161    case GLSL_SAMPLER_DIM_1D:
4162       return array ? 2 : 1;
4163    case GLSL_SAMPLER_DIM_2D:
4164       return array ? 3 : 2;
4165    case GLSL_SAMPLER_DIM_MS:
4166       return array ? 4 : 3;
4167    case GLSL_SAMPLER_DIM_3D:
4168    case GLSL_SAMPLER_DIM_CUBE:
4169       return 3;
4170    case GLSL_SAMPLER_DIM_RECT:
4171    case GLSL_SAMPLER_DIM_SUBPASS:
4172       return 2;
4173    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4174       return 3;
4175    default:
4176       break;
4177    }
4178    return 0;
4179 }
4180
4181
4182 /* Adjust the sample index according to FMASK.
4183  *
4184  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4185  * which is the identity mapping. Each nibble says which physical sample
4186  * should be fetched to get that sample.
4187  *
4188  * For example, 0x11111100 means there are only 2 samples stored and
4189  * the second sample covers 3/4 of the pixel. When reading samples 0
4190  * and 1, return physical sample 0 (determined by the first two 0s
4191  * in FMASK), otherwise return physical sample 1.
4192  *
4193  * The sample index should be adjusted as follows:
4194  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4195  */
4196 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4197 {
4198    Builder bld(ctx->program, ctx->block);
4199    Temp fmask = bld.tmp(v1);
4200    unsigned dim = ctx->options->chip_class >= GFX10
4201                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4202                   : 0;
4203
4204    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4205    load->operands[0] = Operand(coords);
4206    load->operands[1] = Operand(fmask_desc_ptr);
4207    load->definitions[0] = Definition(fmask);
4208    load->glc = false;
4209    load->dlc = false;
4210    load->dmask = 0x1;
4211    load->unrm = true;
4212    load->da = da;
4213    load->dim = dim;
4214    load->can_reorder = true; /* fmask images shouldn't be modified */
4215    ctx->block->instructions.emplace_back(std::move(load));
4216
4217    Operand sample_index4;
4218    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4219       sample_index4 = Operand(sample_index.constantValue() << 2);
4220    } else if (sample_index.regClass() == s1) {
4221       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4222    } else {
4223       assert(sample_index.regClass() == v1);
4224       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4225    }
4226
4227    Temp final_sample;
4228    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4229       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4230    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4231       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4232    else
4233       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4234
4235    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4236     * resource descriptor is 0 (invalid),
4237     */
4238    Temp compare = bld.tmp(bld.lm);
4239    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4240                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4241
4242    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4243
4244    /* Replace the MSAA sample index. */
4245    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4246 }
4247
4248 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4249 {
4250
4251    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4252    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4253    bool is_array = glsl_sampler_type_is_array(type);
4254    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4255    assert(!add_frag_pos && "Input attachments should be lowered.");
4256    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4257    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4258    int count = image_type_to_components_count(dim, is_array);
4259    std::vector<Operand> coords(count);
4260
4261    if (is_ms) {
4262       Operand sample_index;
4263       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4264       if (sample_cv)
4265          sample_index = Operand(sample_cv->u32);
4266       else
4267          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4268
4269       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4270          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4271          for (unsigned i = 0; i < vec->operands.size(); i++)
4272             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4273          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4274          vec->definitions[0] = Definition(fmask_load_address);
4275          ctx->block->instructions.emplace_back(std::move(vec));
4276
4277          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4278          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4279       }
4280       count--;
4281       coords[count] = sample_index;
4282    }
4283
4284    if (count == 1 && !gfx9_1d)
4285       return emit_extract_vector(ctx, src0, 0, v1);
4286
4287    if (gfx9_1d) {
4288       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4289       coords.resize(coords.size() + 1);
4290       coords[1] = Operand((uint32_t) 0);
4291       if (is_array)
4292          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4293    } else {
4294       for (int i = 0; i < count; i++)
4295          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4296    }
4297
4298    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4299        instr->intrinsic == nir_intrinsic_image_deref_store) {
4300       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4301       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4302
4303       if (!level_zero)
4304          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4305    }
4306
4307    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4308    for (unsigned i = 0; i < coords.size(); i++)
4309       vec->operands[i] = coords[i];
4310    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4311    vec->definitions[0] = Definition(res);
4312    ctx->block->instructions.emplace_back(std::move(vec));
4313    return res;
4314 }
4315
4316
4317 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4318 {
4319    Builder bld(ctx->program, ctx->block);
4320    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4321    const struct glsl_type *type = glsl_without_array(var->type);
4322    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4323    bool is_array = glsl_sampler_type_is_array(type);
4324    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4325
4326    if (dim == GLSL_SAMPLER_DIM_BUF) {
4327       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4328       unsigned num_channels = util_last_bit(mask);
4329       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4330       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4331
4332       aco_opcode opcode;
4333       switch (num_channels) {
4334       case 1:
4335          opcode = aco_opcode::buffer_load_format_x;
4336          break;
4337       case 2:
4338          opcode = aco_opcode::buffer_load_format_xy;
4339          break;
4340       case 3:
4341          opcode = aco_opcode::buffer_load_format_xyz;
4342          break;
4343       case 4:
4344          opcode = aco_opcode::buffer_load_format_xyzw;
4345          break;
4346       default:
4347          unreachable(">4 channel buffer image load");
4348       }
4349       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4350       load->operands[0] = Operand(vindex);
4351       load->operands[1] = Operand(rsrc);
4352       load->operands[2] = Operand((uint32_t) 0);
4353       Temp tmp;
4354       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4355          tmp = dst;
4356       else
4357          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4358       load->definitions[0] = Definition(tmp);
4359       load->idxen = true;
4360       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4361       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4362       load->barrier = barrier_image;
4363       ctx->block->instructions.emplace_back(std::move(load));
4364
4365       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4366       return;
4367    }
4368
4369    Temp coords = get_image_coords(ctx, instr, type);
4370    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4371
4372    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4373    unsigned num_components = util_bitcount(dmask);
4374    Temp tmp;
4375    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4376       tmp = dst;
4377    else
4378       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4379
4380    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4381    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4382
4383    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4384    load->operands[0] = Operand(coords);
4385    load->operands[1] = Operand(resource);
4386    load->definitions[0] = Definition(tmp);
4387    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4388    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4389    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4390    load->dmask = dmask;
4391    load->unrm = true;
4392    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4393    load->barrier = barrier_image;
4394    ctx->block->instructions.emplace_back(std::move(load));
4395
4396    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4397    return;
4398 }
4399
4400 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4401 {
4402    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4403    const struct glsl_type *type = glsl_without_array(var->type);
4404    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4405    bool is_array = glsl_sampler_type_is_array(type);
4406    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4407
4408    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4409
4410    if (dim == GLSL_SAMPLER_DIM_BUF) {
4411       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4412       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4413       aco_opcode opcode;
4414       switch (data.size()) {
4415       case 1:
4416          opcode = aco_opcode::buffer_store_format_x;
4417          break;
4418       case 2:
4419          opcode = aco_opcode::buffer_store_format_xy;
4420          break;
4421       case 3:
4422          opcode = aco_opcode::buffer_store_format_xyz;
4423          break;
4424       case 4:
4425          opcode = aco_opcode::buffer_store_format_xyzw;
4426          break;
4427       default:
4428          unreachable(">4 channel buffer image store");
4429       }
4430       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4431       store->operands[0] = Operand(vindex);
4432       store->operands[1] = Operand(rsrc);
4433       store->operands[2] = Operand((uint32_t) 0);
4434       store->operands[3] = Operand(data);
4435       store->idxen = true;
4436       store->glc = glc;
4437       store->dlc = false;
4438       store->disable_wqm = true;
4439       store->barrier = barrier_image;
4440       ctx->program->needs_exact = true;
4441       ctx->block->instructions.emplace_back(std::move(store));
4442       return;
4443    }
4444
4445    assert(data.type() == RegType::vgpr);
4446    Temp coords = get_image_coords(ctx, instr, type);
4447    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4448
4449    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4450    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4451
4452    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4453    store->operands[0] = Operand(coords);
4454    store->operands[1] = Operand(resource);
4455    store->operands[2] = Operand(s4);
4456    store->operands[3] = Operand(data);
4457    store->glc = glc;
4458    store->dlc = false;
4459    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4460    store->dmask = (1 << data.size()) - 1;
4461    store->unrm = true;
4462    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4463    store->disable_wqm = true;
4464    store->barrier = barrier_image;
4465    ctx->program->needs_exact = true;
4466    ctx->block->instructions.emplace_back(std::move(store));
4467    return;
4468 }
4469
4470 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4471 {
4472    /* return the previous value if dest is ever used */
4473    bool return_previous = false;
4474    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4475       return_previous = true;
4476       break;
4477    }
4478    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4479       return_previous = true;
4480       break;
4481    }
4482
4483    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4484    const struct glsl_type *type = glsl_without_array(var->type);
4485    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4486    bool is_array = glsl_sampler_type_is_array(type);
4487    Builder bld(ctx->program, ctx->block);
4488
4489    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4490    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4491
4492    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4493       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4494
4495    aco_opcode buf_op, image_op;
4496    switch (instr->intrinsic) {
4497       case nir_intrinsic_image_deref_atomic_add:
4498          buf_op = aco_opcode::buffer_atomic_add;
4499          image_op = aco_opcode::image_atomic_add;
4500          break;
4501       case nir_intrinsic_image_deref_atomic_umin:
4502          buf_op = aco_opcode::buffer_atomic_umin;
4503          image_op = aco_opcode::image_atomic_umin;
4504          break;
4505       case nir_intrinsic_image_deref_atomic_imin:
4506          buf_op = aco_opcode::buffer_atomic_smin;
4507          image_op = aco_opcode::image_atomic_smin;
4508          break;
4509       case nir_intrinsic_image_deref_atomic_umax:
4510          buf_op = aco_opcode::buffer_atomic_umax;
4511          image_op = aco_opcode::image_atomic_umax;
4512          break;
4513       case nir_intrinsic_image_deref_atomic_imax:
4514          buf_op = aco_opcode::buffer_atomic_smax;
4515          image_op = aco_opcode::image_atomic_smax;
4516          break;
4517       case nir_intrinsic_image_deref_atomic_and:
4518          buf_op = aco_opcode::buffer_atomic_and;
4519          image_op = aco_opcode::image_atomic_and;
4520          break;
4521       case nir_intrinsic_image_deref_atomic_or:
4522          buf_op = aco_opcode::buffer_atomic_or;
4523          image_op = aco_opcode::image_atomic_or;
4524          break;
4525       case nir_intrinsic_image_deref_atomic_xor:
4526          buf_op = aco_opcode::buffer_atomic_xor;
4527          image_op = aco_opcode::image_atomic_xor;
4528          break;
4529       case nir_intrinsic_image_deref_atomic_exchange:
4530          buf_op = aco_opcode::buffer_atomic_swap;
4531          image_op = aco_opcode::image_atomic_swap;
4532          break;
4533       case nir_intrinsic_image_deref_atomic_comp_swap:
4534          buf_op = aco_opcode::buffer_atomic_cmpswap;
4535          image_op = aco_opcode::image_atomic_cmpswap;
4536          break;
4537       default:
4538          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4539    }
4540
4541    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4542
4543    if (dim == GLSL_SAMPLER_DIM_BUF) {
4544       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4545       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4546       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4547       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4548       mubuf->operands[0] = Operand(vindex);
4549       mubuf->operands[1] = Operand(resource);
4550       mubuf->operands[2] = Operand((uint32_t)0);
4551       mubuf->operands[3] = Operand(data);
4552       if (return_previous)
4553          mubuf->definitions[0] = Definition(dst);
4554       mubuf->offset = 0;
4555       mubuf->idxen = true;
4556       mubuf->glc = return_previous;
4557       mubuf->dlc = false; /* Not needed for atomics */
4558       mubuf->disable_wqm = true;
4559       mubuf->barrier = barrier_image;
4560       ctx->program->needs_exact = true;
4561       ctx->block->instructions.emplace_back(std::move(mubuf));
4562       return;
4563    }
4564
4565    Temp coords = get_image_coords(ctx, instr, type);
4566    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4567    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4568    mimg->operands[0] = Operand(coords);
4569    mimg->operands[1] = Operand(resource);
4570    mimg->operands[2] = Operand(s4); /* no sampler */
4571    mimg->operands[3] = Operand(data);
4572    if (return_previous)
4573       mimg->definitions[0] = Definition(dst);
4574    mimg->glc = return_previous;
4575    mimg->dlc = false; /* Not needed for atomics */
4576    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4577    mimg->dmask = (1 << data.size()) - 1;
4578    mimg->unrm = true;
4579    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4580    mimg->disable_wqm = true;
4581    mimg->barrier = barrier_image;
4582    ctx->program->needs_exact = true;
4583    ctx->block->instructions.emplace_back(std::move(mimg));
4584    return;
4585 }
4586
4587 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4588 {
4589    if (in_elements && ctx->options->chip_class == GFX8) {
4590       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4591       Builder bld(ctx->program, ctx->block);
4592
4593       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4594
4595       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4596       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4597
4598       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4599       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4600
4601       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4602       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4603
4604       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4605       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4606                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4607       if (dst.type() == RegType::vgpr)
4608          bld.copy(Definition(dst), shr_dst);
4609
4610       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4611    } else {
4612       emit_extract_vector(ctx, desc, 2, dst);
4613    }
4614 }
4615
4616 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4617 {
4618    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4619    const struct glsl_type *type = glsl_without_array(var->type);
4620    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4621    bool is_array = glsl_sampler_type_is_array(type);
4622    Builder bld(ctx->program, ctx->block);
4623
4624    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4625       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4626       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4627    }
4628
4629    /* LOD */
4630    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4631
4632    /* Resource */
4633    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4634
4635    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4636
4637    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4638    mimg->operands[0] = Operand(lod);
4639    mimg->operands[1] = Operand(resource);
4640    uint8_t& dmask = mimg->dmask;
4641    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4642    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4643    mimg->da = glsl_sampler_type_is_array(type);
4644    mimg->can_reorder = true;
4645    Definition& def = mimg->definitions[0];
4646    ctx->block->instructions.emplace_back(std::move(mimg));
4647
4648    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4649        glsl_sampler_type_is_array(type)) {
4650
4651       assert(instr->dest.ssa.num_components == 3);
4652       Temp tmp = {ctx->program->allocateId(), v3};
4653       def = Definition(tmp);
4654       emit_split_vector(ctx, tmp, 3);
4655
4656       /* divide 3rd value by 6 by multiplying with magic number */
4657       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4658       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4659
4660       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4661                  emit_extract_vector(ctx, tmp, 0, v1),
4662                  emit_extract_vector(ctx, tmp, 1, v1),
4663                  by_6);
4664
4665    } else if (ctx->options->chip_class == GFX9 &&
4666               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4667               glsl_sampler_type_is_array(type)) {
4668       assert(instr->dest.ssa.num_components == 2);
4669       def = Definition(dst);
4670       dmask = 0x5;
4671    } else {
4672       def = Definition(dst);
4673    }
4674
4675    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4676 }
4677
4678 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4679 {
4680    Builder bld(ctx->program, ctx->block);
4681    unsigned num_components = instr->num_components;
4682
4683    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4684    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4685    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4686
4687    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4688    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4689 }
4690
4691 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4692 {
4693    Builder bld(ctx->program, ctx->block);
4694    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4695    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4696    unsigned writemask = nir_intrinsic_write_mask(instr);
4697    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4698
4699    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4700    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4701
4702    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4703                ctx->options->chip_class >= GFX8;
4704    if (smem)
4705       offset = bld.as_uniform(offset);
4706    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4707
4708    while (writemask) {
4709       int start, count;
4710       u_bit_scan_consecutive_range(&writemask, &start, &count);
4711       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4712          /* GFX6 doesn't support storing vec3, split it. */
4713          writemask |= 1u << (start + 2);
4714          count = 2;
4715       }
4716       int num_bytes = count * elem_size_bytes;
4717
4718       if (num_bytes > 16) {
4719          assert(elem_size_bytes == 8);
4720          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4721          count = 2;
4722          num_bytes = 16;
4723       }
4724
4725       // TODO: check alignment of sub-dword stores
4726       // TODO: split 3 bytes. there is no store instruction for that
4727
4728       Temp write_data;
4729       if (count != instr->num_components) {
4730          emit_split_vector(ctx, data, instr->num_components);
4731          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4732          for (int i = 0; i < count; i++) {
4733             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4734             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4735          }
4736          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4737          vec->definitions[0] = Definition(write_data);
4738          ctx->block->instructions.emplace_back(std::move(vec));
4739       } else if (!smem && data.type() != RegType::vgpr) {
4740          assert(num_bytes % 4 == 0);
4741          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4742       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4743          assert(num_bytes % 4 == 0);
4744          write_data = bld.as_uniform(data);
4745       } else {
4746          write_data = data;
4747       }
4748
4749       aco_opcode vmem_op, smem_op;
4750       switch (num_bytes) {
4751          case 4:
4752             vmem_op = aco_opcode::buffer_store_dword;
4753             smem_op = aco_opcode::s_buffer_store_dword;
4754             break;
4755          case 8:
4756             vmem_op = aco_opcode::buffer_store_dwordx2;
4757             smem_op = aco_opcode::s_buffer_store_dwordx2;
4758             break;
4759          case 12:
4760             vmem_op = aco_opcode::buffer_store_dwordx3;
4761             smem_op = aco_opcode::last_opcode;
4762             assert(!smem && ctx->options->chip_class > GFX6);
4763             break;
4764          case 16:
4765             vmem_op = aco_opcode::buffer_store_dwordx4;
4766             smem_op = aco_opcode::s_buffer_store_dwordx4;
4767             break;
4768          default:
4769             unreachable("Store SSBO not implemented for this size.");
4770       }
4771       if (ctx->stage == fragment_fs)
4772          smem_op = aco_opcode::p_fs_buffer_store_smem;
4773
4774       if (smem) {
4775          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4776          store->operands[0] = Operand(rsrc);
4777          if (start) {
4778             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4779                                 offset, Operand(start * elem_size_bytes));
4780             store->operands[1] = Operand(off);
4781          } else {
4782             store->operands[1] = Operand(offset);
4783          }
4784          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4785             store->operands[1].setFixed(m0);
4786          store->operands[2] = Operand(write_data);
4787          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4788          store->dlc = false;
4789          store->disable_wqm = true;
4790          store->barrier = barrier_buffer;
4791          ctx->block->instructions.emplace_back(std::move(store));
4792          ctx->program->wb_smem_l1_on_end = true;
4793          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4794             ctx->block->kind |= block_kind_needs_lowering;
4795             ctx->program->needs_exact = true;
4796          }
4797       } else {
4798          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4799          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4800          store->operands[1] = Operand(rsrc);
4801          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4802          store->operands[3] = Operand(write_data);
4803          store->offset = start * elem_size_bytes;
4804          store->offen = (offset.type() == RegType::vgpr);
4805          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4806          store->dlc = false;
4807          store->disable_wqm = true;
4808          store->barrier = barrier_buffer;
4809          ctx->program->needs_exact = true;
4810          ctx->block->instructions.emplace_back(std::move(store));
4811       }
4812    }
4813 }
4814
4815 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4816 {
4817    /* return the previous value if dest is ever used */
4818    bool return_previous = false;
4819    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4820       return_previous = true;
4821       break;
4822    }
4823    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4824       return_previous = true;
4825       break;
4826    }
4827
4828    Builder bld(ctx->program, ctx->block);
4829    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4830
4831    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4832       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4833                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4834
4835    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4836    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4837    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4838
4839    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4840
4841    aco_opcode op32, op64;
4842    switch (instr->intrinsic) {
4843       case nir_intrinsic_ssbo_atomic_add:
4844          op32 = aco_opcode::buffer_atomic_add;
4845          op64 = aco_opcode::buffer_atomic_add_x2;
4846          break;
4847       case nir_intrinsic_ssbo_atomic_imin:
4848          op32 = aco_opcode::buffer_atomic_smin;
4849          op64 = aco_opcode::buffer_atomic_smin_x2;
4850          break;
4851       case nir_intrinsic_ssbo_atomic_umin:
4852          op32 = aco_opcode::buffer_atomic_umin;
4853          op64 = aco_opcode::buffer_atomic_umin_x2;
4854          break;
4855       case nir_intrinsic_ssbo_atomic_imax:
4856          op32 = aco_opcode::buffer_atomic_smax;
4857          op64 = aco_opcode::buffer_atomic_smax_x2;
4858          break;
4859       case nir_intrinsic_ssbo_atomic_umax:
4860          op32 = aco_opcode::buffer_atomic_umax;
4861          op64 = aco_opcode::buffer_atomic_umax_x2;
4862          break;
4863       case nir_intrinsic_ssbo_atomic_and:
4864          op32 = aco_opcode::buffer_atomic_and;
4865          op64 = aco_opcode::buffer_atomic_and_x2;
4866          break;
4867       case nir_intrinsic_ssbo_atomic_or:
4868          op32 = aco_opcode::buffer_atomic_or;
4869          op64 = aco_opcode::buffer_atomic_or_x2;
4870          break;
4871       case nir_intrinsic_ssbo_atomic_xor:
4872          op32 = aco_opcode::buffer_atomic_xor;
4873          op64 = aco_opcode::buffer_atomic_xor_x2;
4874          break;
4875       case nir_intrinsic_ssbo_atomic_exchange:
4876          op32 = aco_opcode::buffer_atomic_swap;
4877          op64 = aco_opcode::buffer_atomic_swap_x2;
4878          break;
4879       case nir_intrinsic_ssbo_atomic_comp_swap:
4880          op32 = aco_opcode::buffer_atomic_cmpswap;
4881          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4882          break;
4883       default:
4884          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4885    }
4886    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4887    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4888    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4889    mubuf->operands[1] = Operand(rsrc);
4890    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4891    mubuf->operands[3] = Operand(data);
4892    if (return_previous)
4893       mubuf->definitions[0] = Definition(dst);
4894    mubuf->offset = 0;
4895    mubuf->offen = (offset.type() == RegType::vgpr);
4896    mubuf->glc = return_previous;
4897    mubuf->dlc = false; /* Not needed for atomics */
4898    mubuf->disable_wqm = true;
4899    mubuf->barrier = barrier_buffer;
4900    ctx->program->needs_exact = true;
4901    ctx->block->instructions.emplace_back(std::move(mubuf));
4902 }
4903
4904 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4905
4906    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4907    Builder bld(ctx->program, ctx->block);
4908    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4909    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4910 }
4911
4912 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4913 {
4914    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4915                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4916
4917    if (addr.type() == RegType::vgpr)
4918       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4919    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4920 }
4921
4922 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4923 {
4924    Builder bld(ctx->program, ctx->block);
4925    unsigned num_components = instr->num_components;
4926    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4927
4928    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4929    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4930
4931    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4932    bool dlc = glc && ctx->options->chip_class >= GFX10;
4933    aco_opcode op;
4934    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4935       bool global = ctx->options->chip_class >= GFX9;
4936
4937       if (ctx->options->chip_class >= GFX7) {
4938          aco_opcode op;
4939          switch (num_bytes) {
4940          case 4:
4941             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4942             break;
4943          case 8:
4944             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4945             break;
4946          case 12:
4947             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4948             break;
4949          case 16:
4950             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4951             break;
4952          default:
4953             unreachable("load_global not implemented for this size.");
4954          }
4955
4956          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4957          flat->operands[0] = Operand(addr);
4958          flat->operands[1] = Operand(s1);
4959          flat->glc = glc;
4960          flat->dlc = dlc;
4961          flat->barrier = barrier_buffer;
4962
4963          if (dst.type() == RegType::sgpr) {
4964             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4965             flat->definitions[0] = Definition(vec);
4966             ctx->block->instructions.emplace_back(std::move(flat));
4967             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4968          } else {
4969             flat->definitions[0] = Definition(dst);
4970             ctx->block->instructions.emplace_back(std::move(flat));
4971          }
4972          emit_split_vector(ctx, dst, num_components);
4973       } else {
4974          assert(ctx->options->chip_class == GFX6);
4975
4976          /* GFX6 doesn't support loading vec3, expand to vec4. */
4977          num_bytes = num_bytes == 12 ? 16 : num_bytes;
4978
4979          aco_opcode op;
4980          switch (num_bytes) {
4981          case 4:
4982             op = aco_opcode::buffer_load_dword;
4983             break;
4984          case 8:
4985             op = aco_opcode::buffer_load_dwordx2;
4986             break;
4987          case 16:
4988             op = aco_opcode::buffer_load_dwordx4;
4989             break;
4990          default:
4991             unreachable("load_global not implemented for this size.");
4992          }
4993
4994          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
4995
4996          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4997          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
4998          mubuf->operands[1] = Operand(rsrc);
4999          mubuf->operands[2] = Operand(0u);
5000          mubuf->glc = glc;
5001          mubuf->dlc = false;
5002          mubuf->offset = 0;
5003          mubuf->addr64 = addr.type() == RegType::vgpr;
5004          mubuf->disable_wqm = false;
5005          mubuf->barrier = barrier_buffer;
5006          aco_ptr<Instruction> instr = std::move(mubuf);
5007
5008          /* expand vector */
5009          if (dst.size() == 3) {
5010             Temp vec = bld.tmp(v4);
5011             instr->definitions[0] = Definition(vec);
5012             bld.insert(std::move(instr));
5013             emit_split_vector(ctx, vec, 4);
5014
5015             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5016             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5017             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5018             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5019          }
5020
5021          if (dst.type() == RegType::sgpr) {
5022             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5023             instr->definitions[0] = Definition(vec);
5024             bld.insert(std::move(instr));
5025             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5026             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5027          } else {
5028             instr->definitions[0] = Definition(dst);
5029             bld.insert(std::move(instr));
5030             emit_split_vector(ctx, dst, num_components);
5031          }
5032       }
5033    } else {
5034       switch (num_bytes) {
5035          case 4:
5036             op = aco_opcode::s_load_dword;
5037             break;
5038          case 8:
5039             op = aco_opcode::s_load_dwordx2;
5040             break;
5041          case 12:
5042          case 16:
5043             op = aco_opcode::s_load_dwordx4;
5044             break;
5045          default:
5046             unreachable("load_global not implemented for this size.");
5047       }
5048       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5049       load->operands[0] = Operand(addr);
5050       load->operands[1] = Operand(0u);
5051       load->definitions[0] = Definition(dst);
5052       load->glc = glc;
5053       load->dlc = dlc;
5054       load->barrier = barrier_buffer;
5055       assert(ctx->options->chip_class >= GFX8 || !glc);
5056
5057       if (dst.size() == 3) {
5058          /* trim vector */
5059          Temp vec = bld.tmp(s4);
5060          load->definitions[0] = Definition(vec);
5061          ctx->block->instructions.emplace_back(std::move(load));
5062          emit_split_vector(ctx, vec, 4);
5063
5064          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5065                     emit_extract_vector(ctx, vec, 0, s1),
5066                     emit_extract_vector(ctx, vec, 1, s1),
5067                     emit_extract_vector(ctx, vec, 2, s1));
5068       } else {
5069          ctx->block->instructions.emplace_back(std::move(load));
5070       }
5071    }
5072 }
5073
5074 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5075 {
5076    Builder bld(ctx->program, ctx->block);
5077    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5078
5079    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5080    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5081
5082    if (ctx->options->chip_class >= GFX7)
5083       addr = as_vgpr(ctx, addr);
5084
5085    unsigned writemask = nir_intrinsic_write_mask(instr);
5086    while (writemask) {
5087       int start, count;
5088       u_bit_scan_consecutive_range(&writemask, &start, &count);
5089       if (count == 3 && ctx->options->chip_class == GFX6) {
5090          /* GFX6 doesn't support storing vec3, split it. */
5091          writemask |= 1u << (start + 2);
5092          count = 2;
5093       }
5094       unsigned num_bytes = count * elem_size_bytes;
5095
5096       Temp write_data = data;
5097       if (count != instr->num_components) {
5098          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5099          for (int i = 0; i < count; i++)
5100             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5101          write_data = bld.tmp(RegType::vgpr, count);
5102          vec->definitions[0] = Definition(write_data);
5103          ctx->block->instructions.emplace_back(std::move(vec));
5104       }
5105
5106       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5107       unsigned offset = start * elem_size_bytes;
5108
5109       if (ctx->options->chip_class >= GFX7) {
5110          if (offset > 0 && ctx->options->chip_class < GFX9) {
5111             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5112             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5113             Temp carry = bld.tmp(bld.lm);
5114             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5115
5116             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5117                      Operand(offset), addr0);
5118             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5119                      Operand(0u), addr1,
5120                      carry).def(1).setHint(vcc);
5121
5122             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5123
5124             offset = 0;
5125          }
5126
5127          bool global = ctx->options->chip_class >= GFX9;
5128          aco_opcode op;
5129          switch (num_bytes) {
5130          case 4:
5131             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5132             break;
5133          case 8:
5134             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5135             break;
5136          case 12:
5137             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5138             break;
5139          case 16:
5140             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5141             break;
5142          default:
5143             unreachable("store_global not implemented for this size.");
5144          }
5145
5146          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5147          flat->operands[0] = Operand(addr);
5148          flat->operands[1] = Operand(s1);
5149          flat->operands[2] = Operand(data);
5150          flat->glc = glc;
5151          flat->dlc = false;
5152          flat->offset = offset;
5153          flat->disable_wqm = true;
5154          flat->barrier = barrier_buffer;
5155          ctx->program->needs_exact = true;
5156          ctx->block->instructions.emplace_back(std::move(flat));
5157       } else {
5158          assert(ctx->options->chip_class == GFX6);
5159
5160          aco_opcode op;
5161          switch (num_bytes) {
5162          case 4:
5163             op = aco_opcode::buffer_store_dword;
5164             break;
5165          case 8:
5166             op = aco_opcode::buffer_store_dwordx2;
5167             break;
5168          case 16:
5169             op = aco_opcode::buffer_store_dwordx4;
5170             break;
5171          default:
5172             unreachable("store_global not implemented for this size.");
5173          }
5174
5175          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5176
5177          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5178          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5179          mubuf->operands[1] = Operand(rsrc);
5180          mubuf->operands[2] = Operand(0u);
5181          mubuf->operands[3] = Operand(write_data);
5182          mubuf->glc = glc;
5183          mubuf->dlc = false;
5184          mubuf->offset = offset;
5185          mubuf->addr64 = addr.type() == RegType::vgpr;
5186          mubuf->disable_wqm = true;
5187          mubuf->barrier = barrier_buffer;
5188          ctx->program->needs_exact = true;
5189          ctx->block->instructions.emplace_back(std::move(mubuf));
5190       }
5191    }
5192 }
5193
5194 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5195 {
5196    /* return the previous value if dest is ever used */
5197    bool return_previous = false;
5198    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5199       return_previous = true;
5200       break;
5201    }
5202    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5203       return_previous = true;
5204       break;
5205    }
5206
5207    Builder bld(ctx->program, ctx->block);
5208    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5209    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5210
5211    if (ctx->options->chip_class >= GFX7)
5212       addr = as_vgpr(ctx, addr);
5213
5214    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5215       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5216                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5217
5218    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5219
5220    aco_opcode op32, op64;
5221
5222    if (ctx->options->chip_class >= GFX7) {
5223       bool global = ctx->options->chip_class >= GFX9;
5224       switch (instr->intrinsic) {
5225          case nir_intrinsic_global_atomic_add:
5226             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5227             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5228             break;
5229          case nir_intrinsic_global_atomic_imin:
5230             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5231             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5232             break;
5233          case nir_intrinsic_global_atomic_umin:
5234             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5235             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5236             break;
5237          case nir_intrinsic_global_atomic_imax:
5238             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5239             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5240             break;
5241          case nir_intrinsic_global_atomic_umax:
5242             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5243             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5244             break;
5245          case nir_intrinsic_global_atomic_and:
5246             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5247             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5248             break;
5249          case nir_intrinsic_global_atomic_or:
5250             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5251             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5252             break;
5253          case nir_intrinsic_global_atomic_xor:
5254             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5255             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5256             break;
5257          case nir_intrinsic_global_atomic_exchange:
5258             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5259             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5260             break;
5261          case nir_intrinsic_global_atomic_comp_swap:
5262             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5263             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5264             break;
5265          default:
5266             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5267       }
5268
5269       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5270       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5271       flat->operands[0] = Operand(addr);
5272       flat->operands[1] = Operand(s1);
5273       flat->operands[2] = Operand(data);
5274       if (return_previous)
5275          flat->definitions[0] = Definition(dst);
5276       flat->glc = return_previous;
5277       flat->dlc = false; /* Not needed for atomics */
5278       flat->offset = 0;
5279       flat->disable_wqm = true;
5280       flat->barrier = barrier_buffer;
5281       ctx->program->needs_exact = true;
5282       ctx->block->instructions.emplace_back(std::move(flat));
5283    } else {
5284       assert(ctx->options->chip_class == GFX6);
5285
5286       switch (instr->intrinsic) {
5287          case nir_intrinsic_global_atomic_add:
5288             op32 = aco_opcode::buffer_atomic_add;
5289             op64 = aco_opcode::buffer_atomic_add_x2;
5290             break;
5291          case nir_intrinsic_global_atomic_imin:
5292             op32 = aco_opcode::buffer_atomic_smin;
5293             op64 = aco_opcode::buffer_atomic_smin_x2;
5294             break;
5295          case nir_intrinsic_global_atomic_umin:
5296             op32 = aco_opcode::buffer_atomic_umin;
5297             op64 = aco_opcode::buffer_atomic_umin_x2;
5298             break;
5299          case nir_intrinsic_global_atomic_imax:
5300             op32 = aco_opcode::buffer_atomic_smax;
5301             op64 = aco_opcode::buffer_atomic_smax_x2;
5302             break;
5303          case nir_intrinsic_global_atomic_umax:
5304             op32 = aco_opcode::buffer_atomic_umax;
5305             op64 = aco_opcode::buffer_atomic_umax_x2;
5306             break;
5307          case nir_intrinsic_global_atomic_and:
5308             op32 = aco_opcode::buffer_atomic_and;
5309             op64 = aco_opcode::buffer_atomic_and_x2;
5310             break;
5311          case nir_intrinsic_global_atomic_or:
5312             op32 = aco_opcode::buffer_atomic_or;
5313             op64 = aco_opcode::buffer_atomic_or_x2;
5314             break;
5315          case nir_intrinsic_global_atomic_xor:
5316             op32 = aco_opcode::buffer_atomic_xor;
5317             op64 = aco_opcode::buffer_atomic_xor_x2;
5318             break;
5319          case nir_intrinsic_global_atomic_exchange:
5320             op32 = aco_opcode::buffer_atomic_swap;
5321             op64 = aco_opcode::buffer_atomic_swap_x2;
5322             break;
5323          case nir_intrinsic_global_atomic_comp_swap:
5324             op32 = aco_opcode::buffer_atomic_cmpswap;
5325             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5326             break;
5327          default:
5328             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5329       }
5330
5331       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5332
5333       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5334
5335       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5336       mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5337       mubuf->operands[1] = Operand(rsrc);
5338       mubuf->operands[2] = Operand(0u);
5339       mubuf->operands[3] = Operand(data);
5340       if (return_previous)
5341          mubuf->definitions[0] = Definition(dst);
5342       mubuf->glc = return_previous;
5343       mubuf->dlc = false;
5344       mubuf->offset = 0;
5345       mubuf->addr64 = addr.type() == RegType::vgpr;
5346       mubuf->disable_wqm = true;
5347       mubuf->barrier = barrier_buffer;
5348       ctx->program->needs_exact = true;
5349       ctx->block->instructions.emplace_back(std::move(mubuf));
5350    }
5351 }
5352
5353 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5354    Builder bld(ctx->program, ctx->block);
5355    switch(instr->intrinsic) {
5356       case nir_intrinsic_group_memory_barrier:
5357       case nir_intrinsic_memory_barrier:
5358          bld.barrier(aco_opcode::p_memory_barrier_common);
5359          break;
5360       case nir_intrinsic_memory_barrier_buffer:
5361          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5362          break;
5363       case nir_intrinsic_memory_barrier_image:
5364          bld.barrier(aco_opcode::p_memory_barrier_image);
5365          break;
5366       case nir_intrinsic_memory_barrier_shared:
5367          bld.barrier(aco_opcode::p_memory_barrier_shared);
5368          break;
5369       default:
5370          unreachable("Unimplemented memory barrier intrinsic");
5371          break;
5372    }
5373 }
5374
5375 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5376 {
5377    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5378    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5379    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5380    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5381    Builder bld(ctx->program, ctx->block);
5382
5383    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5384    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5385    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5386 }
5387
5388 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5389 {
5390    unsigned writemask = nir_intrinsic_write_mask(instr);
5391    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5392    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5393    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5394    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5395
5396    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5397    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5398 }
5399
5400 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5401 {
5402    unsigned offset = nir_intrinsic_base(instr);
5403    Operand m = load_lds_size_m0(ctx);
5404    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5405    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5406
5407    unsigned num_operands = 3;
5408    aco_opcode op32, op64, op32_rtn, op64_rtn;
5409    switch(instr->intrinsic) {
5410       case nir_intrinsic_shared_atomic_add:
5411          op32 = aco_opcode::ds_add_u32;
5412          op64 = aco_opcode::ds_add_u64;
5413          op32_rtn = aco_opcode::ds_add_rtn_u32;
5414          op64_rtn = aco_opcode::ds_add_rtn_u64;
5415          break;
5416       case nir_intrinsic_shared_atomic_imin:
5417          op32 = aco_opcode::ds_min_i32;
5418          op64 = aco_opcode::ds_min_i64;
5419          op32_rtn = aco_opcode::ds_min_rtn_i32;
5420          op64_rtn = aco_opcode::ds_min_rtn_i64;
5421          break;
5422       case nir_intrinsic_shared_atomic_umin:
5423          op32 = aco_opcode::ds_min_u32;
5424          op64 = aco_opcode::ds_min_u64;
5425          op32_rtn = aco_opcode::ds_min_rtn_u32;
5426          op64_rtn = aco_opcode::ds_min_rtn_u64;
5427          break;
5428       case nir_intrinsic_shared_atomic_imax:
5429          op32 = aco_opcode::ds_max_i32;
5430          op64 = aco_opcode::ds_max_i64;
5431          op32_rtn = aco_opcode::ds_max_rtn_i32;
5432          op64_rtn = aco_opcode::ds_max_rtn_i64;
5433          break;
5434       case nir_intrinsic_shared_atomic_umax:
5435          op32 = aco_opcode::ds_max_u32;
5436          op64 = aco_opcode::ds_max_u64;
5437          op32_rtn = aco_opcode::ds_max_rtn_u32;
5438          op64_rtn = aco_opcode::ds_max_rtn_u64;
5439          break;
5440       case nir_intrinsic_shared_atomic_and:
5441          op32 = aco_opcode::ds_and_b32;
5442          op64 = aco_opcode::ds_and_b64;
5443          op32_rtn = aco_opcode::ds_and_rtn_b32;
5444          op64_rtn = aco_opcode::ds_and_rtn_b64;
5445          break;
5446       case nir_intrinsic_shared_atomic_or:
5447          op32 = aco_opcode::ds_or_b32;
5448          op64 = aco_opcode::ds_or_b64;
5449          op32_rtn = aco_opcode::ds_or_rtn_b32;
5450          op64_rtn = aco_opcode::ds_or_rtn_b64;
5451          break;
5452       case nir_intrinsic_shared_atomic_xor:
5453          op32 = aco_opcode::ds_xor_b32;
5454          op64 = aco_opcode::ds_xor_b64;
5455          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5456          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5457          break;
5458       case nir_intrinsic_shared_atomic_exchange:
5459          op32 = aco_opcode::ds_write_b32;
5460          op64 = aco_opcode::ds_write_b64;
5461          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5462          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5463          break;
5464       case nir_intrinsic_shared_atomic_comp_swap:
5465          op32 = aco_opcode::ds_cmpst_b32;
5466          op64 = aco_opcode::ds_cmpst_b64;
5467          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5468          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5469          num_operands = 4;
5470          break;
5471       default:
5472          unreachable("Unhandled shared atomic intrinsic");
5473    }
5474
5475    /* return the previous value if dest is ever used */
5476    bool return_previous = false;
5477    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5478       return_previous = true;
5479       break;
5480    }
5481    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5482       return_previous = true;
5483       break;
5484    }
5485
5486    aco_opcode op;
5487    if (data.size() == 1) {
5488       assert(instr->dest.ssa.bit_size == 32);
5489       op = return_previous ? op32_rtn : op32;
5490    } else {
5491       assert(instr->dest.ssa.bit_size == 64);
5492       op = return_previous ? op64_rtn : op64;
5493    }
5494
5495    if (offset > 65535) {
5496       Builder bld(ctx->program, ctx->block);
5497       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5498       offset = 0;
5499    }
5500
5501    aco_ptr<DS_instruction> ds;
5502    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5503    ds->operands[0] = Operand(address);
5504    ds->operands[1] = Operand(data);
5505    if (num_operands == 4)
5506       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5507    ds->operands[num_operands - 1] = m;
5508    ds->offset0 = offset;
5509    if (return_previous)
5510       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5511    ctx->block->instructions.emplace_back(std::move(ds));
5512 }
5513
5514 Temp get_scratch_resource(isel_context *ctx)
5515 {
5516    Builder bld(ctx->program, ctx->block);
5517    Temp scratch_addr = ctx->program->private_segment_buffer;
5518    if (ctx->stage != compute_cs)
5519       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5520
5521    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5522                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5523
5524    if (ctx->program->chip_class >= GFX10) {
5525       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5526                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5527                    S_008F0C_RESOURCE_LEVEL(1);
5528    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5529       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5530                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5531    }
5532
5533    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5534    if (ctx->program->chip_class <= GFX8)
5535       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5536
5537    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5538 }
5539
5540 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5541    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5542    Builder bld(ctx->program, ctx->block);
5543    Temp rsrc = get_scratch_resource(ctx);
5544    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5545    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5546
5547    aco_opcode op;
5548    switch (dst.size()) {
5549       case 1:
5550          op = aco_opcode::buffer_load_dword;
5551          break;
5552       case 2:
5553          op = aco_opcode::buffer_load_dwordx2;
5554          break;
5555       case 3:
5556          op = aco_opcode::buffer_load_dwordx3;
5557          break;
5558       case 4:
5559          op = aco_opcode::buffer_load_dwordx4;
5560          break;
5561       case 6:
5562       case 8: {
5563          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5564          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5565                                 bld.def(v4), offset, rsrc,
5566                                 ctx->program->scratch_offset, 0, true);
5567          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5568                                                   aco_opcode::buffer_load_dwordx4,
5569                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5570                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5571          emit_split_vector(ctx, lower, 2);
5572          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5573          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5574          if (dst.size() == 8) {
5575             emit_split_vector(ctx, upper, 2);
5576             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5577             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5578          } else {
5579             elems[2] = upper;
5580          }
5581
5582          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5583                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5584          for (unsigned i = 0; i < dst.size() / 2; i++)
5585             vec->operands[i] = Operand(elems[i]);
5586          vec->definitions[0] = Definition(dst);
5587          bld.insert(std::move(vec));
5588          ctx->allocated_vec.emplace(dst.id(), elems);
5589          return;
5590       }
5591       default:
5592          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5593    }
5594
5595    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5596    emit_split_vector(ctx, dst, instr->num_components);
5597 }
5598
5599 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5600    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5601    Builder bld(ctx->program, ctx->block);
5602    Temp rsrc = get_scratch_resource(ctx);
5603    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5604    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5605
5606    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5607    unsigned writemask = nir_intrinsic_write_mask(instr);
5608
5609    while (writemask) {
5610       int start, count;
5611       u_bit_scan_consecutive_range(&writemask, &start, &count);
5612       int num_bytes = count * elem_size_bytes;
5613
5614       if (num_bytes > 16) {
5615          assert(elem_size_bytes == 8);
5616          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5617          count = 2;
5618          num_bytes = 16;
5619       }
5620
5621       // TODO: check alignment of sub-dword stores
5622       // TODO: split 3 bytes. there is no store instruction for that
5623
5624       Temp write_data;
5625       if (count != instr->num_components) {
5626          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5627          for (int i = 0; i < count; i++) {
5628             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5629             vec->operands[i] = Operand(elem);
5630          }
5631          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5632          vec->definitions[0] = Definition(write_data);
5633          ctx->block->instructions.emplace_back(std::move(vec));
5634       } else {
5635          write_data = data;
5636       }
5637
5638       aco_opcode op;
5639       switch (num_bytes) {
5640          case 4:
5641             op = aco_opcode::buffer_store_dword;
5642             break;
5643          case 8:
5644             op = aco_opcode::buffer_store_dwordx2;
5645             break;
5646          case 12:
5647             op = aco_opcode::buffer_store_dwordx3;
5648             break;
5649          case 16:
5650             op = aco_opcode::buffer_store_dwordx4;
5651             break;
5652          default:
5653             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5654       }
5655
5656       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5657    }
5658 }
5659
5660 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5661    uint8_t log2_ps_iter_samples;
5662    if (ctx->program->info->ps.force_persample) {
5663       log2_ps_iter_samples =
5664          util_logbase2(ctx->options->key.fs.num_samples);
5665    } else {
5666       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5667    }
5668
5669    /* The bit pattern matches that used by fixed function fragment
5670     * processing. */
5671    static const unsigned ps_iter_masks[] = {
5672       0xffff, /* not used */
5673       0x5555,
5674       0x1111,
5675       0x0101,
5676       0x0001,
5677    };
5678    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5679
5680    Builder bld(ctx->program, ctx->block);
5681
5682    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5683                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5684    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5685    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5686    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5687    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5688 }
5689
5690 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
5691    Builder bld(ctx->program, ctx->block);
5692
5693    unsigned stream = nir_intrinsic_stream_id(instr);
5694    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5695    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
5696    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
5697
5698    /* get GSVS ring */
5699    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
5700
5701    unsigned num_components =
5702       ctx->program->info->gs.num_stream_output_components[stream];
5703    assert(num_components);
5704
5705    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
5706    unsigned stream_offset = 0;
5707    for (unsigned i = 0; i < stream; i++) {
5708       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
5709       stream_offset += prev_stride * ctx->program->wave_size;
5710    }
5711
5712    /* Limit on the stride field for <= GFX7. */
5713    assert(stride < (1 << 14));
5714
5715    Temp gsvs_dwords[4];
5716    for (unsigned i = 0; i < 4; i++)
5717       gsvs_dwords[i] = bld.tmp(s1);
5718    bld.pseudo(aco_opcode::p_split_vector,
5719               Definition(gsvs_dwords[0]),
5720               Definition(gsvs_dwords[1]),
5721               Definition(gsvs_dwords[2]),
5722               Definition(gsvs_dwords[3]),
5723               gsvs_ring);
5724
5725    if (stream_offset) {
5726       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
5727
5728       Temp carry = bld.tmp(s1);
5729       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
5730       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
5731    }
5732
5733    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
5734    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
5735
5736    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5737                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
5738
5739    unsigned offset = 0;
5740    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
5741       if (ctx->program->info->gs.output_streams[i] != stream)
5742          continue;
5743
5744       for (unsigned j = 0; j < 4; j++) {
5745          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
5746             continue;
5747
5748          if (ctx->outputs.mask[i] & (1 << j)) {
5749             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
5750             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
5751             if (const_offset >= 4096u) {
5752                if (vaddr_offset.isUndefined())
5753                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
5754                else
5755                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
5756                const_offset %= 4096u;
5757             }
5758
5759             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
5760             mtbuf->operands[0] = vaddr_offset;
5761             mtbuf->operands[1] = Operand(gsvs_ring);
5762             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
5763             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
5764             mtbuf->offen = !vaddr_offset.isUndefined();
5765             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
5766             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
5767             mtbuf->offset = const_offset;
5768             mtbuf->glc = true;
5769             mtbuf->slc = true;
5770             mtbuf->barrier = barrier_gs_data;
5771             mtbuf->can_reorder = true;
5772             bld.insert(std::move(mtbuf));
5773          }
5774
5775          offset += ctx->shader->info.gs.vertices_out;
5776       }
5777
5778       /* outputs for the next vertex are undefined and keeping them around can
5779        * create invalid IR with control flow */
5780       ctx->outputs.mask[i] = 0;
5781    }
5782
5783    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
5784 }
5785
5786 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5787 {
5788    Builder bld(ctx->program, ctx->block);
5789
5790    if (cluster_size == 1) {
5791       return src;
5792    } if (op == nir_op_iand && cluster_size == 4) {
5793       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5794       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5795       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5796                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5797    } else if (op == nir_op_ior && cluster_size == 4) {
5798       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5799       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5800                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5801    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5802       //subgroupAnd(val) -> (exec & ~val) == 0
5803       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5804       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5805       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5806    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5807       //subgroupOr(val) -> (val & exec) != 0
5808       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5809       return bool_to_vector_condition(ctx, tmp);
5810    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5811       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5812       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5813       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5814       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5815       return bool_to_vector_condition(ctx, tmp);
5816    } else {
5817       //subgroupClustered{And,Or,Xor}(val, n) ->
5818       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5819       //cluster_offset = ~(n - 1) & lane_id
5820       //cluster_mask = ((1 << n) - 1)
5821       //subgroupClusteredAnd():
5822       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5823       //subgroupClusteredOr():
5824       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5825       //subgroupClusteredXor():
5826       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5827       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5828       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5829
5830       Temp tmp;
5831       if (op == nir_op_iand)
5832          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5833       else
5834          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5835
5836       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5837
5838       if (ctx->program->chip_class <= GFX7)
5839          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5840       else if (ctx->program->wave_size == 64)
5841          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5842       else
5843          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5844       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5845       if (cluster_mask != 0xffffffff)
5846          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5847
5848       Definition cmp_def = Definition();
5849       if (op == nir_op_iand) {
5850          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5851       } else if (op == nir_op_ior) {
5852          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5853       } else if (op == nir_op_ixor) {
5854          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5855                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5856          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5857       }
5858       cmp_def.setHint(vcc);
5859       return cmp_def.getTemp();
5860    }
5861 }
5862
5863 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5864 {
5865    Builder bld(ctx->program, ctx->block);
5866
5867    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5868    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5869    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5870    Temp tmp;
5871    if (op == nir_op_iand)
5872       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5873    else
5874       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5875
5876    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5877    Temp lo = lohi.def(0).getTemp();
5878    Temp hi = lohi.def(1).getTemp();
5879    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5880
5881    Definition cmp_def = Definition();
5882    if (op == nir_op_iand)
5883       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5884    else if (op == nir_op_ior)
5885       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5886    else if (op == nir_op_ixor)
5887       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5888                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5889    cmp_def.setHint(vcc);
5890    return cmp_def.getTemp();
5891 }
5892
5893 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5894 {
5895    Builder bld(ctx->program, ctx->block);
5896
5897    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5898    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5899    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5900    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5901    if (op == nir_op_iand)
5902       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5903    else if (op == nir_op_ior)
5904       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5905    else if (op == nir_op_ixor)
5906       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5907
5908    assert(false);
5909    return Temp();
5910 }
5911
5912 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5913 {
5914    Builder bld(ctx->program, ctx->block);
5915    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5916    if (src.regClass().type() == RegType::vgpr) {
5917       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5918    } else if (src.regClass() == s1) {
5919       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5920    } else if (src.regClass() == s2) {
5921       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5922    } else {
5923       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5924       nir_print_instr(&instr->instr, stderr);
5925       fprintf(stderr, "\n");
5926    }
5927 }
5928
5929 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5930 {
5931    Builder bld(ctx->program, ctx->block);
5932    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5933    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5934    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5935
5936    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5937    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5938    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5939    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5940
5941    /* Build DD X/Y */
5942    if (ctx->program->chip_class >= GFX8) {
5943       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5944       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5945       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5946       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5947       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5948       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5949    } else {
5950       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5951       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5952       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5953       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5954       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5955       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5956       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5957       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5958       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5959       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5960    }
5961
5962    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5963    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5964    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5965    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5966    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5967    Temp wqm1 = bld.tmp(v1);
5968    emit_wqm(ctx, tmp1, wqm1, true);
5969    Temp wqm2 = bld.tmp(v1);
5970    emit_wqm(ctx, tmp2, wqm2, true);
5971    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5972    return;
5973 }
5974
5975 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5976 {
5977    Builder bld(ctx->program, ctx->block);
5978    switch(instr->intrinsic) {
5979    case nir_intrinsic_load_barycentric_sample:
5980    case nir_intrinsic_load_barycentric_pixel:
5981    case nir_intrinsic_load_barycentric_centroid: {
5982       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5983       Temp bary = Temp(0, s2);
5984       switch (mode) {
5985       case INTERP_MODE_SMOOTH:
5986       case INTERP_MODE_NONE:
5987          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5988             bary = get_arg(ctx, ctx->args->ac.persp_center);
5989          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5990             bary = ctx->persp_centroid;
5991          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5992             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5993          break;
5994       case INTERP_MODE_NOPERSPECTIVE:
5995          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5996             bary = get_arg(ctx, ctx->args->ac.linear_center);
5997          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5998             bary = ctx->linear_centroid;
5999          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6000             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6001          break;
6002       default:
6003          break;
6004       }
6005       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6006       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6007       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6008       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6009                  Operand(p1), Operand(p2));
6010       emit_split_vector(ctx, dst, 2);
6011       break;
6012    }
6013    case nir_intrinsic_load_barycentric_at_sample: {
6014       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6015       switch (ctx->options->key.fs.num_samples) {
6016          case 2: sample_pos_offset += 1 << 3; break;
6017          case 4: sample_pos_offset += 3 << 3; break;
6018          case 8: sample_pos_offset += 7 << 3; break;
6019          default: break;
6020       }
6021       Temp sample_pos;
6022       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6023       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6024       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6025       if (addr.type() == RegType::sgpr) {
6026          Operand offset;
6027          if (const_addr) {
6028             sample_pos_offset += const_addr->u32 << 3;
6029             offset = Operand(sample_pos_offset);
6030          } else if (ctx->options->chip_class >= GFX9) {
6031             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6032          } else {
6033             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6034             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6035          }
6036
6037          Operand off = bld.copy(bld.def(s1), Operand(offset));
6038          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6039
6040       } else if (ctx->options->chip_class >= GFX9) {
6041          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6042          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6043       } else if (ctx->options->chip_class >= GFX7) {
6044          /* addr += private_segment_buffer + sample_pos_offset */
6045          Temp tmp0 = bld.tmp(s1);
6046          Temp tmp1 = bld.tmp(s1);
6047          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6048          Definition scc_tmp = bld.def(s1, scc);
6049          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6050          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6051          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6052          Temp pck0 = bld.tmp(v1);
6053          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6054          tmp1 = as_vgpr(ctx, tmp1);
6055          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6056          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6057
6058          /* sample_pos = flat_load_dwordx2 addr */
6059          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6060       } else {
6061          assert(ctx->options->chip_class == GFX6);
6062
6063          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6064                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6065          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6066
6067          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6068          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6069
6070          sample_pos = bld.tmp(v2);
6071
6072          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6073          load->definitions[0] = Definition(sample_pos);
6074          load->operands[0] = Operand(addr);
6075          load->operands[1] = Operand(rsrc);
6076          load->operands[2] = Operand(0u);
6077          load->offset = sample_pos_offset;
6078          load->offen = 0;
6079          load->addr64 = true;
6080          load->glc = false;
6081          load->dlc = false;
6082          load->disable_wqm = false;
6083          load->barrier = barrier_none;
6084          load->can_reorder = true;
6085          ctx->block->instructions.emplace_back(std::move(load));
6086       }
6087
6088       /* sample_pos -= 0.5 */
6089       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6090       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6091       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6092       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6093       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6094
6095       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6096       break;
6097    }
6098    case nir_intrinsic_load_barycentric_at_offset: {
6099       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6100       RegClass rc = RegClass(offset.type(), 1);
6101       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6102       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6103       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6104       break;
6105    }
6106    case nir_intrinsic_load_front_face: {
6107       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6108                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6109       break;
6110    }
6111    case nir_intrinsic_load_view_index:
6112    case nir_intrinsic_load_layer_id: {
6113       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & (sw_vs | sw_gs))) {
6114          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6115          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6116          break;
6117       }
6118
6119       unsigned idx = nir_intrinsic_base(instr);
6120       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6121                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6122       break;
6123    }
6124    case nir_intrinsic_load_frag_coord: {
6125       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6126       break;
6127    }
6128    case nir_intrinsic_load_sample_pos: {
6129       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6130       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6131       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6132                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6133                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6134       break;
6135    }
6136    case nir_intrinsic_load_interpolated_input:
6137       visit_load_interpolated_input(ctx, instr);
6138       break;
6139    case nir_intrinsic_store_output:
6140       visit_store_output(ctx, instr);
6141       break;
6142    case nir_intrinsic_load_input:
6143       visit_load_input(ctx, instr);
6144       break;
6145    case nir_intrinsic_load_per_vertex_input:
6146       visit_load_per_vertex_input(ctx, instr);
6147       break;
6148    case nir_intrinsic_load_ubo:
6149       visit_load_ubo(ctx, instr);
6150       break;
6151    case nir_intrinsic_load_push_constant:
6152       visit_load_push_constant(ctx, instr);
6153       break;
6154    case nir_intrinsic_load_constant:
6155       visit_load_constant(ctx, instr);
6156       break;
6157    case nir_intrinsic_vulkan_resource_index:
6158       visit_load_resource(ctx, instr);
6159       break;
6160    case nir_intrinsic_discard:
6161       visit_discard(ctx, instr);
6162       break;
6163    case nir_intrinsic_discard_if:
6164       visit_discard_if(ctx, instr);
6165       break;
6166    case nir_intrinsic_load_shared:
6167       visit_load_shared(ctx, instr);
6168       break;
6169    case nir_intrinsic_store_shared:
6170       visit_store_shared(ctx, instr);
6171       break;
6172    case nir_intrinsic_shared_atomic_add:
6173    case nir_intrinsic_shared_atomic_imin:
6174    case nir_intrinsic_shared_atomic_umin:
6175    case nir_intrinsic_shared_atomic_imax:
6176    case nir_intrinsic_shared_atomic_umax:
6177    case nir_intrinsic_shared_atomic_and:
6178    case nir_intrinsic_shared_atomic_or:
6179    case nir_intrinsic_shared_atomic_xor:
6180    case nir_intrinsic_shared_atomic_exchange:
6181    case nir_intrinsic_shared_atomic_comp_swap:
6182       visit_shared_atomic(ctx, instr);
6183       break;
6184    case nir_intrinsic_image_deref_load:
6185       visit_image_load(ctx, instr);
6186       break;
6187    case nir_intrinsic_image_deref_store:
6188       visit_image_store(ctx, instr);
6189       break;
6190    case nir_intrinsic_image_deref_atomic_add:
6191    case nir_intrinsic_image_deref_atomic_umin:
6192    case nir_intrinsic_image_deref_atomic_imin:
6193    case nir_intrinsic_image_deref_atomic_umax:
6194    case nir_intrinsic_image_deref_atomic_imax:
6195    case nir_intrinsic_image_deref_atomic_and:
6196    case nir_intrinsic_image_deref_atomic_or:
6197    case nir_intrinsic_image_deref_atomic_xor:
6198    case nir_intrinsic_image_deref_atomic_exchange:
6199    case nir_intrinsic_image_deref_atomic_comp_swap:
6200       visit_image_atomic(ctx, instr);
6201       break;
6202    case nir_intrinsic_image_deref_size:
6203       visit_image_size(ctx, instr);
6204       break;
6205    case nir_intrinsic_load_ssbo:
6206       visit_load_ssbo(ctx, instr);
6207       break;
6208    case nir_intrinsic_store_ssbo:
6209       visit_store_ssbo(ctx, instr);
6210       break;
6211    case nir_intrinsic_load_global:
6212       visit_load_global(ctx, instr);
6213       break;
6214    case nir_intrinsic_store_global:
6215       visit_store_global(ctx, instr);
6216       break;
6217    case nir_intrinsic_global_atomic_add:
6218    case nir_intrinsic_global_atomic_imin:
6219    case nir_intrinsic_global_atomic_umin:
6220    case nir_intrinsic_global_atomic_imax:
6221    case nir_intrinsic_global_atomic_umax:
6222    case nir_intrinsic_global_atomic_and:
6223    case nir_intrinsic_global_atomic_or:
6224    case nir_intrinsic_global_atomic_xor:
6225    case nir_intrinsic_global_atomic_exchange:
6226    case nir_intrinsic_global_atomic_comp_swap:
6227       visit_global_atomic(ctx, instr);
6228       break;
6229    case nir_intrinsic_ssbo_atomic_add:
6230    case nir_intrinsic_ssbo_atomic_imin:
6231    case nir_intrinsic_ssbo_atomic_umin:
6232    case nir_intrinsic_ssbo_atomic_imax:
6233    case nir_intrinsic_ssbo_atomic_umax:
6234    case nir_intrinsic_ssbo_atomic_and:
6235    case nir_intrinsic_ssbo_atomic_or:
6236    case nir_intrinsic_ssbo_atomic_xor:
6237    case nir_intrinsic_ssbo_atomic_exchange:
6238    case nir_intrinsic_ssbo_atomic_comp_swap:
6239       visit_atomic_ssbo(ctx, instr);
6240       break;
6241    case nir_intrinsic_load_scratch:
6242       visit_load_scratch(ctx, instr);
6243       break;
6244    case nir_intrinsic_store_scratch:
6245       visit_store_scratch(ctx, instr);
6246       break;
6247    case nir_intrinsic_get_buffer_size:
6248       visit_get_buffer_size(ctx, instr);
6249       break;
6250    case nir_intrinsic_control_barrier: {
6251       unsigned* bsize = ctx->program->info->cs.block_size;
6252       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6253       if (workgroup_size > ctx->program->wave_size)
6254          bld.sopp(aco_opcode::s_barrier);
6255       break;
6256    }
6257    case nir_intrinsic_group_memory_barrier:
6258    case nir_intrinsic_memory_barrier:
6259    case nir_intrinsic_memory_barrier_buffer:
6260    case nir_intrinsic_memory_barrier_image:
6261    case nir_intrinsic_memory_barrier_shared:
6262       emit_memory_barrier(ctx, instr);
6263       break;
6264    case nir_intrinsic_memory_barrier_tcs_patch:
6265       break;
6266    case nir_intrinsic_load_num_work_groups: {
6267       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6268       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6269       emit_split_vector(ctx, dst, 3);
6270       break;
6271    }
6272    case nir_intrinsic_load_local_invocation_id: {
6273       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6274       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6275       emit_split_vector(ctx, dst, 3);
6276       break;
6277    }
6278    case nir_intrinsic_load_work_group_id: {
6279       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6280       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6281       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6282                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6283                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6284                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6285       emit_split_vector(ctx, dst, 3);
6286       break;
6287    }
6288    case nir_intrinsic_load_local_invocation_index: {
6289       Temp id = emit_mbcnt(ctx, bld.def(v1));
6290
6291       /* The tg_size bits [6:11] contain the subgroup id,
6292        * we need this multiplied by the wave size, and then OR the thread id to it.
6293        */
6294       if (ctx->program->wave_size == 64) {
6295          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6296          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6297                                 get_arg(ctx, ctx->args->ac.tg_size));
6298          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6299       } else {
6300          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6301          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6302                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6303          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6304       }
6305       break;
6306    }
6307    case nir_intrinsic_load_subgroup_id: {
6308       if (ctx->stage == compute_cs) {
6309          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6310                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6311       } else {
6312          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6313       }
6314       break;
6315    }
6316    case nir_intrinsic_load_subgroup_invocation: {
6317       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6318       break;
6319    }
6320    case nir_intrinsic_load_num_subgroups: {
6321       if (ctx->stage == compute_cs)
6322          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6323                   get_arg(ctx, ctx->args->ac.tg_size));
6324       else
6325          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6326       break;
6327    }
6328    case nir_intrinsic_ballot: {
6329       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6330       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6331       Definition tmp = bld.def(dst.regClass());
6332       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6333       if (instr->src[0].ssa->bit_size == 1) {
6334          assert(src.regClass() == bld.lm);
6335          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6336       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6337          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6338       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6339          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6340       } else {
6341          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6342          nir_print_instr(&instr->instr, stderr);
6343          fprintf(stderr, "\n");
6344       }
6345       if (dst.size() != bld.lm.size()) {
6346          /* Wave32 with ballot size set to 64 */
6347          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6348       }
6349       emit_wqm(ctx, tmp.getTemp(), dst);
6350       break;
6351    }
6352    case nir_intrinsic_shuffle:
6353    case nir_intrinsic_read_invocation: {
6354       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6355       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6356          emit_uniform_subgroup(ctx, instr, src);
6357       } else {
6358          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6359          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6360             tid = bld.as_uniform(tid);
6361          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6362          if (src.regClass() == v1) {
6363             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6364          } else if (src.regClass() == v2) {
6365             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6366             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6367             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6368             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6369             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6370             emit_split_vector(ctx, dst, 2);
6371          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6372             assert(src.regClass() == bld.lm);
6373             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6374             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6375          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6376             assert(src.regClass() == bld.lm);
6377             Temp tmp;
6378             if (ctx->program->chip_class <= GFX7)
6379                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6380             else if (ctx->program->wave_size == 64)
6381                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6382             else
6383                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6384             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6385             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6386             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6387          } else {
6388             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6389             nir_print_instr(&instr->instr, stderr);
6390             fprintf(stderr, "\n");
6391          }
6392       }
6393       break;
6394    }
6395    case nir_intrinsic_load_sample_id: {
6396       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6397                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6398       break;
6399    }
6400    case nir_intrinsic_load_sample_mask_in: {
6401       visit_load_sample_mask_in(ctx, instr);
6402       break;
6403    }
6404    case nir_intrinsic_read_first_invocation: {
6405       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6406       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6407       if (src.regClass() == v1) {
6408          emit_wqm(ctx,
6409                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6410                   dst);
6411       } else if (src.regClass() == v2) {
6412          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6413          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6414          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6415          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6416          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6417          emit_split_vector(ctx, dst, 2);
6418       } else if (instr->dest.ssa.bit_size == 1) {
6419          assert(src.regClass() == bld.lm);
6420          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6421                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6422          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6423       } else if (src.regClass() == s1) {
6424          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6425       } else if (src.regClass() == s2) {
6426          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6427       } else {
6428          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6429          nir_print_instr(&instr->instr, stderr);
6430          fprintf(stderr, "\n");
6431       }
6432       break;
6433    }
6434    case nir_intrinsic_vote_all: {
6435       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6436       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6437       assert(src.regClass() == bld.lm);
6438       assert(dst.regClass() == bld.lm);
6439
6440       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6441       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6442       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6443       break;
6444    }
6445    case nir_intrinsic_vote_any: {
6446       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6447       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6448       assert(src.regClass() == bld.lm);
6449       assert(dst.regClass() == bld.lm);
6450
6451       Temp tmp = bool_to_scalar_condition(ctx, src);
6452       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6453       break;
6454    }
6455    case nir_intrinsic_reduce:
6456    case nir_intrinsic_inclusive_scan:
6457    case nir_intrinsic_exclusive_scan: {
6458       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6459       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6460       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6461       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6462          nir_intrinsic_cluster_size(instr) : 0;
6463       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6464
6465       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6466          emit_uniform_subgroup(ctx, instr, src);
6467       } else if (instr->dest.ssa.bit_size == 1) {
6468          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6469             op = nir_op_iand;
6470          else if (op == nir_op_iadd)
6471             op = nir_op_ixor;
6472          else if (op == nir_op_umax || op == nir_op_imax)
6473             op = nir_op_ior;
6474          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6475
6476          switch (instr->intrinsic) {
6477          case nir_intrinsic_reduce:
6478             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6479             break;
6480          case nir_intrinsic_exclusive_scan:
6481             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6482             break;
6483          case nir_intrinsic_inclusive_scan:
6484             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6485             break;
6486          default:
6487             assert(false);
6488          }
6489       } else if (cluster_size == 1) {
6490          bld.copy(Definition(dst), src);
6491       } else {
6492          src = as_vgpr(ctx, src);
6493
6494          ReduceOp reduce_op;
6495          switch (op) {
6496          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6497             CASE(iadd)
6498             CASE(imul)
6499             CASE(fadd)
6500             CASE(fmul)
6501             CASE(imin)
6502             CASE(umin)
6503             CASE(fmin)
6504             CASE(imax)
6505             CASE(umax)
6506             CASE(fmax)
6507             CASE(iand)
6508             CASE(ior)
6509             CASE(ixor)
6510             default:
6511                unreachable("unknown reduction op");
6512          #undef CASE
6513          }
6514
6515          aco_opcode aco_op;
6516          switch (instr->intrinsic) {
6517             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6518             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6519             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6520             default:
6521                unreachable("unknown reduce intrinsic");
6522          }
6523
6524          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6525          reduce->operands[0] = Operand(src);
6526          // filled in by aco_reduce_assign.cpp, used internally as part of the
6527          // reduce sequence
6528          assert(dst.size() == 1 || dst.size() == 2);
6529          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6530          reduce->operands[2] = Operand(v1.as_linear());
6531
6532          Temp tmp_dst = bld.tmp(dst.regClass());
6533          reduce->definitions[0] = Definition(tmp_dst);
6534          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6535          reduce->definitions[2] = Definition();
6536          reduce->definitions[3] = Definition(scc, s1);
6537          reduce->definitions[4] = Definition();
6538          reduce->reduce_op = reduce_op;
6539          reduce->cluster_size = cluster_size;
6540          ctx->block->instructions.emplace_back(std::move(reduce));
6541
6542          emit_wqm(ctx, tmp_dst, dst);
6543       }
6544       break;
6545    }
6546    case nir_intrinsic_quad_broadcast: {
6547       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6548       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6549          emit_uniform_subgroup(ctx, instr, src);
6550       } else {
6551          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6552          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6553          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6554
6555          if (instr->dest.ssa.bit_size == 1) {
6556             assert(src.regClass() == bld.lm);
6557             assert(dst.regClass() == bld.lm);
6558             uint32_t half_mask = 0x11111111u << lane;
6559             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6560             Temp tmp = bld.tmp(bld.lm);
6561             bld.sop1(Builder::s_wqm, Definition(tmp),
6562                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6563                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6564             emit_wqm(ctx, tmp, dst);
6565          } else if (instr->dest.ssa.bit_size == 32) {
6566             if (ctx->program->chip_class >= GFX8)
6567                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6568             else
6569                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6570          } else if (instr->dest.ssa.bit_size == 64) {
6571             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6572             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6573             if (ctx->program->chip_class >= GFX8) {
6574                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6575                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6576             } else {
6577                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6578                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6579             }
6580             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6581             emit_split_vector(ctx, dst, 2);
6582          } else {
6583             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6584             nir_print_instr(&instr->instr, stderr);
6585             fprintf(stderr, "\n");
6586          }
6587       }
6588       break;
6589    }
6590    case nir_intrinsic_quad_swap_horizontal:
6591    case nir_intrinsic_quad_swap_vertical:
6592    case nir_intrinsic_quad_swap_diagonal:
6593    case nir_intrinsic_quad_swizzle_amd: {
6594       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6595       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6596          emit_uniform_subgroup(ctx, instr, src);
6597          break;
6598       }
6599       uint16_t dpp_ctrl = 0;
6600       switch (instr->intrinsic) {
6601       case nir_intrinsic_quad_swap_horizontal:
6602          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6603          break;
6604       case nir_intrinsic_quad_swap_vertical:
6605          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6606          break;
6607       case nir_intrinsic_quad_swap_diagonal:
6608          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6609          break;
6610       case nir_intrinsic_quad_swizzle_amd:
6611          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6612          break;
6613       default:
6614          break;
6615       }
6616       if (ctx->program->chip_class < GFX8)
6617          dpp_ctrl |= (1 << 15);
6618
6619       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6620       if (instr->dest.ssa.bit_size == 1) {
6621          assert(src.regClass() == bld.lm);
6622          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6623          if (ctx->program->chip_class >= GFX8)
6624             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6625          else
6626             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6627          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6628          emit_wqm(ctx, tmp, dst);
6629       } else if (instr->dest.ssa.bit_size == 32) {
6630          Temp tmp;
6631          if (ctx->program->chip_class >= GFX8)
6632             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6633          else
6634             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6635          emit_wqm(ctx, tmp, dst);
6636       } else if (instr->dest.ssa.bit_size == 64) {
6637          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6638          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6639          if (ctx->program->chip_class >= GFX8) {
6640             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6641             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6642          } else {
6643             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6644             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6645          }
6646          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6647          emit_split_vector(ctx, dst, 2);
6648       } else {
6649          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6650          nir_print_instr(&instr->instr, stderr);
6651          fprintf(stderr, "\n");
6652       }
6653       break;
6654    }
6655    case nir_intrinsic_masked_swizzle_amd: {
6656       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6657       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6658          emit_uniform_subgroup(ctx, instr, src);
6659          break;
6660       }
6661       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6662       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6663       if (dst.regClass() == v1) {
6664          emit_wqm(ctx,
6665                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6666                   dst);
6667       } else if (dst.regClass() == v2) {
6668          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6669          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6670          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6671          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6672          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6673          emit_split_vector(ctx, dst, 2);
6674       } else {
6675          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6676          nir_print_instr(&instr->instr, stderr);
6677          fprintf(stderr, "\n");
6678       }
6679       break;
6680    }
6681    case nir_intrinsic_write_invocation_amd: {
6682       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6683       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6684       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6685       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6686       if (dst.regClass() == v1) {
6687          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6688          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6689       } else if (dst.regClass() == v2) {
6690          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6691          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6692          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6693          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6694          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6695          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6696          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6697          emit_split_vector(ctx, dst, 2);
6698       } else {
6699          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6700          nir_print_instr(&instr->instr, stderr);
6701          fprintf(stderr, "\n");
6702       }
6703       break;
6704    }
6705    case nir_intrinsic_mbcnt_amd: {
6706       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6707       RegClass rc = RegClass(src.type(), 1);
6708       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6709       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6710       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6711       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6712       emit_wqm(ctx, wqm_tmp, dst);
6713       break;
6714    }
6715    case nir_intrinsic_load_helper_invocation: {
6716       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6717       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6718       ctx->block->kind |= block_kind_needs_lowering;
6719       ctx->program->needs_exact = true;
6720       break;
6721    }
6722    case nir_intrinsic_is_helper_invocation: {
6723       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6724       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6725       ctx->block->kind |= block_kind_needs_lowering;
6726       ctx->program->needs_exact = true;
6727       break;
6728    }
6729    case nir_intrinsic_demote:
6730       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6731
6732       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6733          ctx->cf_info.exec_potentially_empty = true;
6734       ctx->block->kind |= block_kind_uses_demote;
6735       ctx->program->needs_exact = true;
6736       break;
6737    case nir_intrinsic_demote_if: {
6738       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6739       assert(src.regClass() == bld.lm);
6740       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6741       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6742
6743       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6744          ctx->cf_info.exec_potentially_empty = true;
6745       ctx->block->kind |= block_kind_uses_demote;
6746       ctx->program->needs_exact = true;
6747       break;
6748    }
6749    case nir_intrinsic_first_invocation: {
6750       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6751                get_ssa_temp(ctx, &instr->dest.ssa));
6752       break;
6753    }
6754    case nir_intrinsic_shader_clock:
6755       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6756       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6757       break;
6758    case nir_intrinsic_load_vertex_id_zero_base: {
6759       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6760       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6761       break;
6762    }
6763    case nir_intrinsic_load_first_vertex: {
6764       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6765       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6766       break;
6767    }
6768    case nir_intrinsic_load_base_instance: {
6769       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6770       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6771       break;
6772    }
6773    case nir_intrinsic_load_instance_id: {
6774       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6775       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6776       break;
6777    }
6778    case nir_intrinsic_load_draw_id: {
6779       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6780       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6781       break;
6782    }
6783    case nir_intrinsic_load_invocation_id: {
6784       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6785       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6786       if (ctx->options->chip_class >= GFX10)
6787          bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6788       else
6789          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6790       break;
6791    }
6792    case nir_intrinsic_load_primitive_id: {
6793       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6794       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6795       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
6796       break;
6797    }
6798    case nir_intrinsic_emit_vertex_with_counter: {
6799       visit_emit_vertex_with_counter(ctx, instr);
6800       break;
6801    }
6802    case nir_intrinsic_end_primitive_with_counter: {
6803       unsigned stream = nir_intrinsic_stream_id(instr);
6804       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
6805       break;
6806    }
6807    case nir_intrinsic_set_vertex_count: {
6808       /* unused, the HW keeps track of this for us */
6809       break;
6810    }
6811    default:
6812       fprintf(stderr, "Unimplemented intrinsic instr: ");
6813       nir_print_instr(&instr->instr, stderr);
6814       fprintf(stderr, "\n");
6815       abort();
6816
6817       break;
6818    }
6819 }
6820
6821
6822 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6823                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6824                     enum glsl_base_type *stype)
6825 {
6826    nir_deref_instr *texture_deref_instr = NULL;
6827    nir_deref_instr *sampler_deref_instr = NULL;
6828    int plane = -1;
6829
6830    for (unsigned i = 0; i < instr->num_srcs; i++) {
6831       switch (instr->src[i].src_type) {
6832       case nir_tex_src_texture_deref:
6833          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6834          break;
6835       case nir_tex_src_sampler_deref:
6836          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6837          break;
6838       case nir_tex_src_plane:
6839          plane = nir_src_as_int(instr->src[i].src);
6840          break;
6841       default:
6842          break;
6843       }
6844    }
6845
6846    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6847
6848    if (!sampler_deref_instr)
6849       sampler_deref_instr = texture_deref_instr;
6850
6851    if (plane >= 0) {
6852       assert(instr->op != nir_texop_txf_ms &&
6853              instr->op != nir_texop_samples_identical);
6854       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6855       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6856    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6857       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6858    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6859       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6860    } else {
6861       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6862    }
6863    if (samp_ptr) {
6864       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6865
6866       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6867          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6868          Builder bld(ctx->program, ctx->block);
6869
6870          /* to avoid unnecessary moves, we split and recombine sampler and image */
6871          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6872                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6873          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6874          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6875                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6876                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6877          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6878                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6879
6880          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6881          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6882                                img[0], img[1], img[2], img[3],
6883                                img[4], img[5], img[6], img[7]);
6884          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6885                                 samp[0], samp[1], samp[2], samp[3]);
6886       }
6887    }
6888    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6889                      instr->op == nir_texop_samples_identical))
6890       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6891 }
6892
6893 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6894                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6895 {
6896    Builder bld(ctx->program, ctx->block);
6897
6898    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6899    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6900    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6901
6902    Operand neg_one(0xbf800000u);
6903    Operand one(0x3f800000u);
6904    Operand two(0x40000000u);
6905    Operand four(0x40800000u);
6906
6907    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6908    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6909    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6910
6911    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6912    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6913    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6914    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6915
6916    // select sc
6917    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6918    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6919                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6920                        one, is_ma_y);
6921    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6922
6923    // select tc
6924    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6925    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6926    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6927
6928    // select ma
6929    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6930                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6931                   deriv_z, is_ma_z);
6932    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6933    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6934 }
6935
6936 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6937 {
6938    Builder bld(ctx->program, ctx->block);
6939    Temp coord_args[4], ma, tc, sc, id;
6940    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6941       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6942
6943    if (is_array) {
6944       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6945
6946       // see comment in ac_prepare_cube_coords()
6947       if (ctx->options->chip_class <= GFX8)
6948          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6949    }
6950
6951    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6952
6953    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6954    vop3a->operands[0] = Operand(ma);
6955    vop3a->abs[0] = true;
6956    Temp invma = bld.tmp(v1);
6957    vop3a->definitions[0] = Definition(invma);
6958    ctx->block->instructions.emplace_back(std::move(vop3a));
6959
6960    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6961    if (!is_deriv)
6962       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6963
6964    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6965    if (!is_deriv)
6966       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6967
6968    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6969
6970    if (is_deriv) {
6971       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6972       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6973
6974       for (unsigned i = 0; i < 2; i++) {
6975          // see comment in ac_prepare_cube_coords()
6976          Temp deriv_ma;
6977          Temp deriv_sc, deriv_tc;
6978          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6979                            &deriv_ma, &deriv_sc, &deriv_tc);
6980
6981          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6982
6983          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6984                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6985                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6986          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6987                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6988                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6989          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6990       }
6991
6992       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6993       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6994    }
6995
6996    if (is_array)
6997       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6998    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6999
7000 }
7001
7002 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
7003 {
7004    Temp coord_vec[3];
7005    for (unsigned i = 0; i < coords.size(); i++)
7006       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
7007
7008    Builder bld(ctx->program, ctx->block);
7009    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
7010
7011    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7012    for (unsigned i = 0; i < coords.size(); i++)
7013       vec->operands[i] = Operand(coord_vec[i]);
7014    Temp res = bld.tmp(RegType::vgpr, coords.size());
7015    vec->definitions[0] = Definition(res);
7016    ctx->block->instructions.emplace_back(std::move(vec));
7017    return res;
7018 }
7019
7020 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7021 {
7022    if (vec->parent_instr->type != nir_instr_type_alu)
7023       return;
7024    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7025    if (vec_instr->op != nir_op_vec(vec->num_components))
7026       return;
7027
7028    for (unsigned i = 0; i < vec->num_components; i++) {
7029       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7030               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7031    }
7032 }
7033
7034 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7035 {
7036    Builder bld(ctx->program, ctx->block);
7037    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7038         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7039    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
7040         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
7041    nir_const_value *sample_index_cv = NULL;
7042    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7043    enum glsl_base_type stype;
7044    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7045
7046    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7047                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7048    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7049                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7050
7051    for (unsigned i = 0; i < instr->num_srcs; i++) {
7052       switch (instr->src[i].src_type) {
7053       case nir_tex_src_coord:
7054          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
7055          break;
7056       case nir_tex_src_bias:
7057          if (instr->op == nir_texop_txb) {
7058             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7059             has_bias = true;
7060          }
7061          break;
7062       case nir_tex_src_lod: {
7063          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7064
7065          if (val && val->f32 <= 0.0) {
7066             level_zero = true;
7067          } else {
7068             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7069             has_lod = true;
7070          }
7071          break;
7072       }
7073       case nir_tex_src_comparator:
7074          if (instr->is_shadow) {
7075             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7076             has_compare = true;
7077          }
7078          break;
7079       case nir_tex_src_offset:
7080          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7081          get_const_vec(instr->src[i].src.ssa, const_offset);
7082          has_offset = true;
7083          break;
7084       case nir_tex_src_ddx:
7085          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7086          has_ddx = true;
7087          break;
7088       case nir_tex_src_ddy:
7089          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7090          has_ddy = true;
7091          break;
7092       case nir_tex_src_ms_index:
7093          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7094          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7095          has_sample_index = true;
7096          break;
7097       case nir_tex_src_texture_offset:
7098       case nir_tex_src_sampler_offset:
7099       default:
7100          break;
7101       }
7102    }
7103 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
7104    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7105       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7106
7107    if (instr->op == nir_texop_texture_samples) {
7108       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7109
7110       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7111       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7112       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7113       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7114
7115       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7116                samples, Operand(1u), bld.scc(is_msaa));
7117       return;
7118    }
7119
7120    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7121       aco_ptr<Instruction> tmp_instr;
7122       Temp acc, pack = Temp();
7123
7124       uint32_t pack_const = 0;
7125       for (unsigned i = 0; i < offset.size(); i++) {
7126          if (!const_offset[i])
7127             continue;
7128          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7129       }
7130
7131       if (offset.type() == RegType::sgpr) {
7132          for (unsigned i = 0; i < offset.size(); i++) {
7133             if (const_offset[i])
7134                continue;
7135
7136             acc = emit_extract_vector(ctx, offset, i, s1);
7137             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7138
7139             if (i) {
7140                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7141             }
7142
7143             if (pack == Temp()) {
7144                pack = acc;
7145             } else {
7146                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7147             }
7148          }
7149
7150          if (pack_const && pack != Temp())
7151             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7152       } else {
7153          for (unsigned i = 0; i < offset.size(); i++) {
7154             if (const_offset[i])
7155                continue;
7156
7157             acc = emit_extract_vector(ctx, offset, i, v1);
7158             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7159
7160             if (i) {
7161                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7162             }
7163
7164             if (pack == Temp()) {
7165                pack = acc;
7166             } else {
7167                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7168             }
7169          }
7170
7171          if (pack_const && pack != Temp())
7172             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7173       }
7174       if (pack_const && pack == Temp())
7175          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7176       else if (pack == Temp())
7177          has_offset = false;
7178       else
7179          offset = pack;
7180    }
7181
7182    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7183       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7184
7185    /* pack derivatives */
7186    if (has_ddx || has_ddy) {
7187       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7188          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
7189                              ddx, Operand(0u), ddy, Operand(0u));
7190       } else {
7191          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
7192       }
7193       has_derivs = true;
7194    }
7195
7196    if (instr->coord_components > 1 &&
7197        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7198        instr->is_array &&
7199        instr->op != nir_texop_txf)
7200       coords = apply_round_slice(ctx, coords, 1);
7201
7202    if (instr->coord_components > 2 &&
7203       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7204        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7205        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7206        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7207        instr->is_array &&
7208        instr->op != nir_texop_txf &&
7209        instr->op != nir_texop_txf_ms &&
7210        instr->op != nir_texop_fragment_fetch &&
7211        instr->op != nir_texop_fragment_mask_fetch)
7212       coords = apply_round_slice(ctx, coords, 2);
7213
7214    if (ctx->options->chip_class == GFX9 &&
7215        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7216        instr->op != nir_texop_lod && instr->coord_components) {
7217       assert(coords.size() > 0 && coords.size() < 3);
7218
7219       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
7220       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
7221       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
7222       if (coords.size() > 1)
7223          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
7224       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
7225       vec->definitions[0] = Definition(coords);
7226       ctx->block->instructions.emplace_back(std::move(vec));
7227    }
7228
7229    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7230
7231    if (instr->op == nir_texop_samples_identical)
7232       resource = fmask_ptr;
7233
7234    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7235              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7236             instr->op != nir_texop_txs &&
7237             instr->op != nir_texop_fragment_fetch &&
7238             instr->op != nir_texop_fragment_mask_fetch) {
7239       assert(has_sample_index);
7240       Operand op(sample_index);
7241       if (sample_index_cv)
7242          op = Operand(sample_index_cv->u32);
7243       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7244    }
7245
7246    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7247       Temp split_coords[coords.size()];
7248       emit_split_vector(ctx, coords, coords.size());
7249       for (unsigned i = 0; i < coords.size(); i++)
7250          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
7251
7252       unsigned i = 0;
7253       for (; i < std::min(offset.size(), instr->coord_components); i++) {
7254          Temp off = emit_extract_vector(ctx, offset, i, v1);
7255          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
7256       }
7257
7258       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7259       for (unsigned i = 0; i < coords.size(); i++)
7260          vec->operands[i] = Operand(split_coords[i]);
7261       coords = bld.tmp(coords.regClass());
7262       vec->definitions[0] = Definition(coords);
7263       ctx->block->instructions.emplace_back(std::move(vec));
7264
7265       has_offset = false;
7266    }
7267
7268    /* Build tex instruction */
7269    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7270    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7271                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7272                   : 0;
7273    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7274    Temp tmp_dst = dst;
7275
7276    /* gather4 selects the component by dmask and always returns vec4 */
7277    if (instr->op == nir_texop_tg4) {
7278       assert(instr->dest.ssa.num_components == 4);
7279       if (instr->is_shadow)
7280          dmask = 1;
7281       else
7282          dmask = 1 << instr->component;
7283       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7284          tmp_dst = bld.tmp(v4);
7285    } else if (instr->op == nir_texop_samples_identical) {
7286       tmp_dst = bld.tmp(v1);
7287    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7288       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7289    }
7290
7291    aco_ptr<MIMG_instruction> tex;
7292    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7293       if (!has_lod)
7294          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7295
7296       bool div_by_6 = instr->op == nir_texop_txs &&
7297                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7298                       instr->is_array &&
7299                       (dmask & (1 << 2));
7300       if (tmp_dst.id() == dst.id() && div_by_6)
7301          tmp_dst = bld.tmp(tmp_dst.regClass());
7302
7303       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7304       tex->operands[0] = Operand(as_vgpr(ctx,lod));
7305       tex->operands[1] = Operand(resource);
7306       if (ctx->options->chip_class == GFX9 &&
7307           instr->op == nir_texop_txs &&
7308           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7309           instr->is_array) {
7310          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7311       } else if (instr->op == nir_texop_query_levels) {
7312          tex->dmask = 1 << 3;
7313       } else {
7314          tex->dmask = dmask;
7315       }
7316       tex->da = da;
7317       tex->definitions[0] = Definition(tmp_dst);
7318       tex->dim = dim;
7319       tex->can_reorder = true;
7320       ctx->block->instructions.emplace_back(std::move(tex));
7321
7322       if (div_by_6) {
7323          /* divide 3rd value by 6 by multiplying with magic number */
7324          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7325          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7326          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7327          assert(instr->dest.ssa.num_components == 3);
7328          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7329          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7330                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7331                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7332                               by_6);
7333
7334       }
7335
7336       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7337       return;
7338    }
7339
7340    Temp tg4_compare_cube_wa64 = Temp();
7341
7342    if (tg4_integer_workarounds) {
7343       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7344       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7345       tex->operands[1] = Operand(resource);
7346       tex->dim = dim;
7347       tex->dmask = 0x3;
7348       tex->da = da;
7349       Temp size = bld.tmp(v2);
7350       tex->definitions[0] = Definition(size);
7351       tex->can_reorder = true;
7352       ctx->block->instructions.emplace_back(std::move(tex));
7353       emit_split_vector(ctx, size, size.size());
7354
7355       Temp half_texel[2];
7356       for (unsigned i = 0; i < 2; i++) {
7357          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7358          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7359          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7360          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7361       }
7362
7363       Temp orig_coords[2] = {
7364          emit_extract_vector(ctx, coords, 0, v1),
7365          emit_extract_vector(ctx, coords, 1, v1)};
7366       Temp new_coords[2] = {
7367          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
7368          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
7369       };
7370
7371       if (tg4_integer_cube_workaround) {
7372          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7373          Temp desc[resource.size()];
7374          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7375                                                                            Format::PSEUDO, 1, resource.size())};
7376          split->operands[0] = Operand(resource);
7377          for (unsigned i = 0; i < resource.size(); i++) {
7378             desc[i] = bld.tmp(s1);
7379             split->definitions[i] = Definition(desc[i]);
7380          }
7381          ctx->block->instructions.emplace_back(std::move(split));
7382
7383          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7384          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7385                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7386
7387          Temp nfmt;
7388          if (stype == GLSL_TYPE_UINT) {
7389             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7390                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7391                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7392                             bld.scc(compare_cube_wa));
7393          } else {
7394             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7395                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7396                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7397                             bld.scc(compare_cube_wa));
7398          }
7399          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7400          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7401
7402          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7403
7404          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7405                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7406          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7407
7408          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7409                                                                          Format::PSEUDO, resource.size(), 1)};
7410          for (unsigned i = 0; i < resource.size(); i++)
7411             vec->operands[i] = Operand(desc[i]);
7412          resource = bld.tmp(resource.regClass());
7413          vec->definitions[0] = Definition(resource);
7414          ctx->block->instructions.emplace_back(std::move(vec));
7415
7416          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7417                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
7418          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7419                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
7420       }
7421
7422       if (coords.size() == 3) {
7423          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
7424                              new_coords[0], new_coords[1],
7425                              emit_extract_vector(ctx, coords, 2, v1));
7426       } else {
7427          assert(coords.size() == 2);
7428          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
7429                              new_coords[0], new_coords[1]);
7430       }
7431    }
7432
7433    std::vector<Operand> args;
7434    if (has_offset)
7435       args.emplace_back(Operand(offset));
7436    if (has_bias)
7437       args.emplace_back(Operand(bias));
7438    if (has_compare)
7439       args.emplace_back(Operand(compare));
7440    if (has_derivs)
7441       args.emplace_back(Operand(derivs));
7442    args.emplace_back(Operand(coords));
7443    if (has_sample_index)
7444       args.emplace_back(Operand(sample_index));
7445    if (has_lod)
7446       args.emplace_back(lod);
7447
7448    Temp arg;
7449    if (args.size() > 1) {
7450       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7451       unsigned size = 0;
7452       for (unsigned i = 0; i < args.size(); i++) {
7453          size += args[i].size();
7454          vec->operands[i] = args[i];
7455       }
7456       RegClass rc = RegClass(RegType::vgpr, size);
7457       Temp tmp = bld.tmp(rc);
7458       vec->definitions[0] = Definition(tmp);
7459       ctx->block->instructions.emplace_back(std::move(vec));
7460       arg = tmp;
7461    } else {
7462       assert(args[0].isTemp());
7463       arg = as_vgpr(ctx, args[0].getTemp());
7464    }
7465
7466    /* we don't need the bias, sample index, compare value or offset to be
7467     * computed in WQM but if the p_create_vector copies the coordinates, then it
7468     * needs to be in WQM */
7469    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
7470        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7471        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7472       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7473
7474    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7475       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7476
7477       assert(coords.size() == 1);
7478       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7479       aco_opcode op;
7480       switch (last_bit) {
7481       case 1:
7482          op = aco_opcode::buffer_load_format_x; break;
7483       case 2:
7484          op = aco_opcode::buffer_load_format_xy; break;
7485       case 3:
7486          op = aco_opcode::buffer_load_format_xyz; break;
7487       case 4:
7488          op = aco_opcode::buffer_load_format_xyzw; break;
7489       default:
7490          unreachable("Tex instruction loads more than 4 components.");
7491       }
7492
7493       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7494       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7495          tmp_dst = dst;
7496       else
7497          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7498
7499       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7500       mubuf->operands[0] = Operand(coords);
7501       mubuf->operands[1] = Operand(resource);
7502       mubuf->operands[2] = Operand((uint32_t) 0);
7503       mubuf->definitions[0] = Definition(tmp_dst);
7504       mubuf->idxen = true;
7505       mubuf->can_reorder = true;
7506       ctx->block->instructions.emplace_back(std::move(mubuf));
7507
7508       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7509       return;
7510    }
7511
7512
7513    if (instr->op == nir_texop_txf ||
7514        instr->op == nir_texop_txf_ms ||
7515        instr->op == nir_texop_samples_identical ||
7516        instr->op == nir_texop_fragment_fetch ||
7517        instr->op == nir_texop_fragment_mask_fetch) {
7518       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7519       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
7520       tex->operands[0] = Operand(arg);
7521       tex->operands[1] = Operand(resource);
7522       tex->dim = dim;
7523       tex->dmask = dmask;
7524       tex->unrm = true;
7525       tex->da = da;
7526       tex->definitions[0] = Definition(tmp_dst);
7527       tex->can_reorder = true;
7528       ctx->block->instructions.emplace_back(std::move(tex));
7529
7530       if (instr->op == nir_texop_samples_identical) {
7531          assert(dmask == 1 && dst.regClass() == v1);
7532          assert(dst.id() != tmp_dst.id());
7533
7534          Temp tmp = bld.tmp(bld.lm);
7535          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7536          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7537
7538       } else {
7539          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7540       }
7541       return;
7542    }
7543
7544    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7545    aco_opcode opcode = aco_opcode::image_sample;
7546    if (has_offset) { /* image_sample_*_o */
7547       if (has_compare) {
7548          opcode = aco_opcode::image_sample_c_o;
7549          if (has_derivs)
7550             opcode = aco_opcode::image_sample_c_d_o;
7551          if (has_bias)
7552             opcode = aco_opcode::image_sample_c_b_o;
7553          if (level_zero)
7554             opcode = aco_opcode::image_sample_c_lz_o;
7555          if (has_lod)
7556             opcode = aco_opcode::image_sample_c_l_o;
7557       } else {
7558          opcode = aco_opcode::image_sample_o;
7559          if (has_derivs)
7560             opcode = aco_opcode::image_sample_d_o;
7561          if (has_bias)
7562             opcode = aco_opcode::image_sample_b_o;
7563          if (level_zero)
7564             opcode = aco_opcode::image_sample_lz_o;
7565          if (has_lod)
7566             opcode = aco_opcode::image_sample_l_o;
7567       }
7568    } else { /* no offset */
7569       if (has_compare) {
7570          opcode = aco_opcode::image_sample_c;
7571          if (has_derivs)
7572             opcode = aco_opcode::image_sample_c_d;
7573          if (has_bias)
7574             opcode = aco_opcode::image_sample_c_b;
7575          if (level_zero)
7576             opcode = aco_opcode::image_sample_c_lz;
7577          if (has_lod)
7578             opcode = aco_opcode::image_sample_c_l;
7579       } else {
7580          opcode = aco_opcode::image_sample;
7581          if (has_derivs)
7582             opcode = aco_opcode::image_sample_d;
7583          if (has_bias)
7584             opcode = aco_opcode::image_sample_b;
7585          if (level_zero)
7586             opcode = aco_opcode::image_sample_lz;
7587          if (has_lod)
7588             opcode = aco_opcode::image_sample_l;
7589       }
7590    }
7591
7592    if (instr->op == nir_texop_tg4) {
7593       if (has_offset) {
7594          opcode = aco_opcode::image_gather4_lz_o;
7595          if (has_compare)
7596             opcode = aco_opcode::image_gather4_c_lz_o;
7597       } else {
7598          opcode = aco_opcode::image_gather4_lz;
7599          if (has_compare)
7600             opcode = aco_opcode::image_gather4_c_lz;
7601       }
7602    } else if (instr->op == nir_texop_lod) {
7603       opcode = aco_opcode::image_get_lod;
7604    }
7605
7606    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7607    tex->operands[0] = Operand(arg);
7608    tex->operands[1] = Operand(resource);
7609    tex->operands[2] = Operand(sampler);
7610    tex->dim = dim;
7611    tex->dmask = dmask;
7612    tex->da = da;
7613    tex->definitions[0] = Definition(tmp_dst);
7614    tex->can_reorder = true;
7615    ctx->block->instructions.emplace_back(std::move(tex));
7616
7617    if (tg4_integer_cube_workaround) {
7618       assert(tmp_dst.id() != dst.id());
7619       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7620
7621       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7622       Temp val[4];
7623       for (unsigned i = 0; i < dst.size(); i++) {
7624          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7625          Temp cvt_val;
7626          if (stype == GLSL_TYPE_UINT)
7627             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7628          else
7629             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7630          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7631       }
7632       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7633       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7634                            val[0], val[1], val[2], val[3]);
7635    }
7636    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7637    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7638
7639 }
7640
7641
7642 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7643 {
7644    Temp tmp = get_ssa_temp(ctx, ssa);
7645    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7646       return Operand(tmp.regClass());
7647    else
7648       return Operand(tmp);
7649 }
7650
7651 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7652 {
7653    aco_ptr<Pseudo_instruction> phi;
7654    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7655    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7656
7657    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7658    logical |= ctx->block->kind & block_kind_merge;
7659    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7660
7661    /* we want a sorted list of sources, since the predecessor list is also sorted */
7662    std::map<unsigned, nir_ssa_def*> phi_src;
7663    nir_foreach_phi_src(src, instr)
7664       phi_src[src->pred->index] = src->src.ssa;
7665
7666    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7667    unsigned num_operands = 0;
7668    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7669    unsigned num_defined = 0;
7670    unsigned cur_pred_idx = 0;
7671    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7672       if (cur_pred_idx < preds.size()) {
7673          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7674          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7675          unsigned skipped = 0;
7676          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7677             skipped++;
7678          if (cur_pred_idx + skipped < preds.size()) {
7679             for (unsigned i = 0; i < skipped; i++)
7680                operands[num_operands++] = Operand(dst.regClass());
7681             cur_pred_idx += skipped;
7682          } else {
7683             continue;
7684          }
7685       }
7686       cur_pred_idx++;
7687       Operand op = get_phi_operand(ctx, src.second);
7688       operands[num_operands++] = op;
7689       num_defined += !op.isUndefined();
7690    }
7691    /* handle block_kind_continue_or_break at loop exit blocks */
7692    while (cur_pred_idx++ < preds.size())
7693       operands[num_operands++] = Operand(dst.regClass());
7694
7695    if (num_defined == 0) {
7696       Builder bld(ctx->program, ctx->block);
7697       if (dst.regClass() == s1) {
7698          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7699       } else if (dst.regClass() == v1) {
7700          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7701       } else {
7702          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7703          for (unsigned i = 0; i < dst.size(); i++)
7704             vec->operands[i] = Operand(0u);
7705          vec->definitions[0] = Definition(dst);
7706          ctx->block->instructions.emplace_back(std::move(vec));
7707       }
7708       return;
7709    }
7710
7711    /* we can use a linear phi in some cases if one src is undef */
7712    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7713       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7714
7715       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7716       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7717       assert(invert->kind & block_kind_invert);
7718
7719       unsigned then_block = invert->linear_preds[0];
7720
7721       Block* insert_block = NULL;
7722       for (unsigned i = 0; i < num_operands; i++) {
7723          Operand op = operands[i];
7724          if (op.isUndefined())
7725             continue;
7726          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7727          phi->operands[0] = op;
7728          break;
7729       }
7730       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7731       phi->operands[1] = Operand(dst.regClass());
7732       phi->definitions[0] = Definition(dst);
7733       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7734       return;
7735    }
7736
7737    /* try to scalarize vector phis */
7738    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7739       // TODO: scalarize linear phis on divergent ifs
7740       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7741       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7742       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7743          Operand src = operands[i];
7744          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7745             can_scalarize = false;
7746       }
7747       if (can_scalarize) {
7748          unsigned num_components = instr->dest.ssa.num_components;
7749          assert(dst.size() % num_components == 0);
7750          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7751
7752          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7753          for (unsigned k = 0; k < num_components; k++) {
7754             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7755             for (unsigned i = 0; i < num_operands; i++) {
7756                Operand src = operands[i];
7757                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7758             }
7759             Temp phi_dst = {ctx->program->allocateId(), rc};
7760             phi->definitions[0] = Definition(phi_dst);
7761             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7762             new_vec[k] = phi_dst;
7763             vec->operands[k] = Operand(phi_dst);
7764          }
7765          vec->definitions[0] = Definition(dst);
7766          ctx->block->instructions.emplace_back(std::move(vec));
7767          ctx->allocated_vec.emplace(dst.id(), new_vec);
7768          return;
7769       }
7770    }
7771
7772    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7773    for (unsigned i = 0; i < num_operands; i++)
7774       phi->operands[i] = operands[i];
7775    phi->definitions[0] = Definition(dst);
7776    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7777 }
7778
7779
7780 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7781 {
7782    Temp dst = get_ssa_temp(ctx, &instr->def);
7783
7784    assert(dst.type() == RegType::sgpr);
7785
7786    if (dst.size() == 1) {
7787       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7788    } else {
7789       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7790       for (unsigned i = 0; i < dst.size(); i++)
7791          vec->operands[i] = Operand(0u);
7792       vec->definitions[0] = Definition(dst);
7793       ctx->block->instructions.emplace_back(std::move(vec));
7794    }
7795 }
7796
7797 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7798 {
7799    Builder bld(ctx->program, ctx->block);
7800    Block *logical_target;
7801    append_logical_end(ctx->block);
7802    unsigned idx = ctx->block->index;
7803
7804    switch (instr->type) {
7805    case nir_jump_break:
7806       logical_target = ctx->cf_info.parent_loop.exit;
7807       add_logical_edge(idx, logical_target);
7808       ctx->block->kind |= block_kind_break;
7809
7810       if (!ctx->cf_info.parent_if.is_divergent &&
7811           !ctx->cf_info.parent_loop.has_divergent_continue) {
7812          /* uniform break - directly jump out of the loop */
7813          ctx->block->kind |= block_kind_uniform;
7814          ctx->cf_info.has_branch = true;
7815          bld.branch(aco_opcode::p_branch);
7816          add_linear_edge(idx, logical_target);
7817          return;
7818       }
7819       ctx->cf_info.parent_loop.has_divergent_branch = true;
7820       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7821       break;
7822    case nir_jump_continue:
7823       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7824       add_logical_edge(idx, logical_target);
7825       ctx->block->kind |= block_kind_continue;
7826
7827       if (ctx->cf_info.parent_if.is_divergent) {
7828          /* for potential uniform breaks after this continue,
7829             we must ensure that they are handled correctly */
7830          ctx->cf_info.parent_loop.has_divergent_continue = true;
7831          ctx->cf_info.parent_loop.has_divergent_branch = true;
7832          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7833       } else {
7834          /* uniform continue - directly jump to the loop header */
7835          ctx->block->kind |= block_kind_uniform;
7836          ctx->cf_info.has_branch = true;
7837          bld.branch(aco_opcode::p_branch);
7838          add_linear_edge(idx, logical_target);
7839          return;
7840       }
7841       break;
7842    default:
7843       fprintf(stderr, "Unknown NIR jump instr: ");
7844       nir_print_instr(&instr->instr, stderr);
7845       fprintf(stderr, "\n");
7846       abort();
7847    }
7848
7849    /* remove critical edges from linear CFG */
7850    bld.branch(aco_opcode::p_branch);
7851    Block* break_block = ctx->program->create_and_insert_block();
7852    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7853    break_block->kind |= block_kind_uniform;
7854    add_linear_edge(idx, break_block);
7855    /* the loop_header pointer might be invalidated by this point */
7856    if (instr->type == nir_jump_continue)
7857       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7858    add_linear_edge(break_block->index, logical_target);
7859    bld.reset(break_block);
7860    bld.branch(aco_opcode::p_branch);
7861
7862    Block* continue_block = ctx->program->create_and_insert_block();
7863    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7864    add_linear_edge(idx, continue_block);
7865    append_logical_start(continue_block);
7866    ctx->block = continue_block;
7867    return;
7868 }
7869
7870 void visit_block(isel_context *ctx, nir_block *block)
7871 {
7872    nir_foreach_instr(instr, block) {
7873       switch (instr->type) {
7874       case nir_instr_type_alu:
7875          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7876          break;
7877       case nir_instr_type_load_const:
7878          visit_load_const(ctx, nir_instr_as_load_const(instr));
7879          break;
7880       case nir_instr_type_intrinsic:
7881          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7882          break;
7883       case nir_instr_type_tex:
7884          visit_tex(ctx, nir_instr_as_tex(instr));
7885          break;
7886       case nir_instr_type_phi:
7887          visit_phi(ctx, nir_instr_as_phi(instr));
7888          break;
7889       case nir_instr_type_ssa_undef:
7890          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7891          break;
7892       case nir_instr_type_deref:
7893          break;
7894       case nir_instr_type_jump:
7895          visit_jump(ctx, nir_instr_as_jump(instr));
7896          break;
7897       default:
7898          fprintf(stderr, "Unknown NIR instr type: ");
7899          nir_print_instr(instr, stderr);
7900          fprintf(stderr, "\n");
7901          //abort();
7902       }
7903    }
7904
7905    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7906       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7907 }
7908
7909
7910
7911 static void visit_loop(isel_context *ctx, nir_loop *loop)
7912 {
7913    append_logical_end(ctx->block);
7914    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7915    Builder bld(ctx->program, ctx->block);
7916    bld.branch(aco_opcode::p_branch);
7917    unsigned loop_preheader_idx = ctx->block->index;
7918
7919    Block loop_exit = Block();
7920    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7921    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7922
7923    Block* loop_header = ctx->program->create_and_insert_block();
7924    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7925    loop_header->kind |= block_kind_loop_header;
7926    add_edge(loop_preheader_idx, loop_header);
7927    ctx->block = loop_header;
7928
7929    /* emit loop body */
7930    unsigned loop_header_idx = loop_header->index;
7931    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7932    append_logical_start(ctx->block);
7933    visit_cf_list(ctx, &loop->body);
7934
7935    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7936    if (!ctx->cf_info.has_branch) {
7937       append_logical_end(ctx->block);
7938       if (ctx->cf_info.exec_potentially_empty) {
7939          /* Discards can result in code running with an empty exec mask.
7940           * This would result in divergent breaks not ever being taken. As a
7941           * workaround, break the loop when the loop mask is empty instead of
7942           * always continuing. */
7943          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7944          unsigned block_idx = ctx->block->index;
7945
7946          /* create helper blocks to avoid critical edges */
7947          Block *break_block = ctx->program->create_and_insert_block();
7948          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7949          break_block->kind = block_kind_uniform;
7950          bld.reset(break_block);
7951          bld.branch(aco_opcode::p_branch);
7952          add_linear_edge(block_idx, break_block);
7953          add_linear_edge(break_block->index, &loop_exit);
7954
7955          Block *continue_block = ctx->program->create_and_insert_block();
7956          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7957          continue_block->kind = block_kind_uniform;
7958          bld.reset(continue_block);
7959          bld.branch(aco_opcode::p_branch);
7960          add_linear_edge(block_idx, continue_block);
7961          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7962
7963          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7964          ctx->block = &ctx->program->blocks[block_idx];
7965       } else {
7966          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7967          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7968             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7969          else
7970             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7971       }
7972
7973       bld.reset(ctx->block);
7974       bld.branch(aco_opcode::p_branch);
7975    }
7976
7977    /* fixup phis in loop header from unreachable blocks */
7978    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7979       bool linear = ctx->cf_info.has_branch;
7980       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7981       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7982          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7983              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7984             /* the last operand should be the one that needs to be removed */
7985             instr->operands.pop_back();
7986          } else if (!is_phi(instr)) {
7987             break;
7988          }
7989       }
7990    }
7991
7992    ctx->cf_info.has_branch = false;
7993
7994    // TODO: if the loop has not a single exit, we must add one °°
7995    /* emit loop successor block */
7996    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7997    append_logical_start(ctx->block);
7998
7999    #if 0
8000    // TODO: check if it is beneficial to not branch on continues
8001    /* trim linear phis in loop header */
8002    for (auto&& instr : loop_entry->instructions) {
8003       if (instr->opcode == aco_opcode::p_linear_phi) {
8004          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8005          new_phi->definitions[0] = instr->definitions[0];
8006          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8007             new_phi->operands[i] = instr->operands[i];
8008          /* check that the remaining operands are all the same */
8009          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8010             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8011          instr.swap(new_phi);
8012       } else if (instr->opcode == aco_opcode::p_phi) {
8013          continue;
8014       } else {
8015          break;
8016       }
8017    }
8018    #endif
8019 }
8020
8021 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8022 {
8023    ic->cond = cond;
8024
8025    append_logical_end(ctx->block);
8026    ctx->block->kind |= block_kind_branch;
8027
8028    /* branch to linear then block */
8029    assert(cond.regClass() == ctx->program->lane_mask);
8030    aco_ptr<Pseudo_branch_instruction> branch;
8031    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8032    branch->operands[0] = Operand(cond);
8033    ctx->block->instructions.push_back(std::move(branch));
8034
8035    ic->BB_if_idx = ctx->block->index;
8036    ic->BB_invert = Block();
8037    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8038    /* Invert blocks are intentionally not marked as top level because they
8039     * are not part of the logical cfg. */
8040    ic->BB_invert.kind |= block_kind_invert;
8041    ic->BB_endif = Block();
8042    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8043    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8044
8045    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
8046    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8047    ctx->cf_info.parent_if.is_divergent = true;
8048    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8049
8050    /** emit logical then block */
8051    Block* BB_then_logical = ctx->program->create_and_insert_block();
8052    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8053    add_edge(ic->BB_if_idx, BB_then_logical);
8054    ctx->block = BB_then_logical;
8055    append_logical_start(BB_then_logical);
8056 }
8057
8058 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8059 {
8060    Block *BB_then_logical = ctx->block;
8061    append_logical_end(BB_then_logical);
8062     /* branch from logical then block to invert block */
8063    aco_ptr<Pseudo_branch_instruction> branch;
8064    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8065    BB_then_logical->instructions.emplace_back(std::move(branch));
8066    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8067    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8068       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8069    BB_then_logical->kind |= block_kind_uniform;
8070    assert(!ctx->cf_info.has_branch);
8071    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8072    ctx->cf_info.parent_loop.has_divergent_branch = false;
8073
8074    /** emit linear then block */
8075    Block* BB_then_linear = ctx->program->create_and_insert_block();
8076    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8077    BB_then_linear->kind |= block_kind_uniform;
8078    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8079    /* branch from linear then block to invert block */
8080    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8081    BB_then_linear->instructions.emplace_back(std::move(branch));
8082    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8083
8084    /** emit invert merge block */
8085    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8086    ic->invert_idx = ctx->block->index;
8087
8088    /* branch to linear else block (skip else) */
8089    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8090    branch->operands[0] = Operand(ic->cond);
8091    ctx->block->instructions.push_back(std::move(branch));
8092
8093    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
8094    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8095
8096    /** emit logical else block */
8097    Block* BB_else_logical = ctx->program->create_and_insert_block();
8098    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8099    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8100    add_linear_edge(ic->invert_idx, BB_else_logical);
8101    ctx->block = BB_else_logical;
8102    append_logical_start(BB_else_logical);
8103 }
8104
8105 static void end_divergent_if(isel_context *ctx, if_context *ic)
8106 {
8107    Block *BB_else_logical = ctx->block;
8108    append_logical_end(BB_else_logical);
8109
8110    /* branch from logical else block to endif block */
8111    aco_ptr<Pseudo_branch_instruction> branch;
8112    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8113    BB_else_logical->instructions.emplace_back(std::move(branch));
8114    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8115    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8116       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8117    BB_else_logical->kind |= block_kind_uniform;
8118
8119    assert(!ctx->cf_info.has_branch);
8120    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8121
8122
8123    /** emit linear else block */
8124    Block* BB_else_linear = ctx->program->create_and_insert_block();
8125    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8126    BB_else_linear->kind |= block_kind_uniform;
8127    add_linear_edge(ic->invert_idx, BB_else_linear);
8128
8129    /* branch from linear else block to endif block */
8130    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8131    BB_else_linear->instructions.emplace_back(std::move(branch));
8132    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8133
8134
8135    /** emit endif merge block */
8136    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8137    append_logical_start(ctx->block);
8138
8139
8140    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8141    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
8142    /* uniform control flow never has an empty exec-mask */
8143    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
8144       ctx->cf_info.exec_potentially_empty = false;
8145 }
8146
8147 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8148 {
8149    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8150    Builder bld(ctx->program, ctx->block);
8151    aco_ptr<Pseudo_branch_instruction> branch;
8152
8153    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8154       /**
8155        * Uniform conditionals are represented in the following way*) :
8156        *
8157        * The linear and logical CFG:
8158        *                        BB_IF
8159        *                        /    \
8160        *       BB_THEN (logical)      BB_ELSE (logical)
8161        *                        \    /
8162        *                        BB_ENDIF
8163        *
8164        * *) Exceptions may be due to break and continue statements within loops
8165        *    If a break/continue happens within uniform control flow, it branches
8166        *    to the loop exit/entry block. Otherwise, it branches to the next
8167        *    merge block.
8168        **/
8169       append_logical_end(ctx->block);
8170       ctx->block->kind |= block_kind_uniform;
8171
8172       /* emit branch */
8173       assert(cond.regClass() == bld.lm);
8174       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8175       cond = bool_to_scalar_condition(ctx, cond);
8176
8177       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8178       branch->operands[0] = Operand(cond);
8179       branch->operands[0].setFixed(scc);
8180       ctx->block->instructions.emplace_back(std::move(branch));
8181
8182       unsigned BB_if_idx = ctx->block->index;
8183       Block BB_endif = Block();
8184       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8185       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8186
8187       /** emit then block */
8188       Block* BB_then = ctx->program->create_and_insert_block();
8189       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8190       add_edge(BB_if_idx, BB_then);
8191       append_logical_start(BB_then);
8192       ctx->block = BB_then;
8193       visit_cf_list(ctx, &if_stmt->then_list);
8194       BB_then = ctx->block;
8195       bool then_branch = ctx->cf_info.has_branch;
8196       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8197
8198       if (!then_branch) {
8199          append_logical_end(BB_then);
8200          /* branch from then block to endif block */
8201          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8202          BB_then->instructions.emplace_back(std::move(branch));
8203          add_linear_edge(BB_then->index, &BB_endif);
8204          if (!then_branch_divergent)
8205             add_logical_edge(BB_then->index, &BB_endif);
8206          BB_then->kind |= block_kind_uniform;
8207       }
8208
8209       ctx->cf_info.has_branch = false;
8210       ctx->cf_info.parent_loop.has_divergent_branch = false;
8211
8212       /** emit else block */
8213       Block* BB_else = ctx->program->create_and_insert_block();
8214       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8215       add_edge(BB_if_idx, BB_else);
8216       append_logical_start(BB_else);
8217       ctx->block = BB_else;
8218       visit_cf_list(ctx, &if_stmt->else_list);
8219       BB_else = ctx->block;
8220
8221       if (!ctx->cf_info.has_branch) {
8222          append_logical_end(BB_else);
8223          /* branch from then block to endif block */
8224          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8225          BB_else->instructions.emplace_back(std::move(branch));
8226          add_linear_edge(BB_else->index, &BB_endif);
8227          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8228             add_logical_edge(BB_else->index, &BB_endif);
8229          BB_else->kind |= block_kind_uniform;
8230       }
8231
8232       ctx->cf_info.has_branch &= then_branch;
8233       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8234
8235       /** emit endif merge block */
8236       if (!ctx->cf_info.has_branch) {
8237          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8238          append_logical_start(ctx->block);
8239       }
8240    } else { /* non-uniform condition */
8241       /**
8242        * To maintain a logical and linear CFG without critical edges,
8243        * non-uniform conditionals are represented in the following way*) :
8244        *
8245        * The linear CFG:
8246        *                        BB_IF
8247        *                        /    \
8248        *       BB_THEN (logical)      BB_THEN (linear)
8249        *                        \    /
8250        *                        BB_INVERT (linear)
8251        *                        /    \
8252        *       BB_ELSE (logical)      BB_ELSE (linear)
8253        *                        \    /
8254        *                        BB_ENDIF
8255        *
8256        * The logical CFG:
8257        *                        BB_IF
8258        *                        /    \
8259        *       BB_THEN (logical)      BB_ELSE (logical)
8260        *                        \    /
8261        *                        BB_ENDIF
8262        *
8263        * *) Exceptions may be due to break and continue statements within loops
8264        **/
8265
8266       if_context ic;
8267
8268       begin_divergent_if_then(ctx, &ic, cond);
8269       visit_cf_list(ctx, &if_stmt->then_list);
8270
8271       begin_divergent_if_else(ctx, &ic);
8272       visit_cf_list(ctx, &if_stmt->else_list);
8273
8274       end_divergent_if(ctx, &ic);
8275    }
8276 }
8277
8278 static void visit_cf_list(isel_context *ctx,
8279                           struct exec_list *list)
8280 {
8281    foreach_list_typed(nir_cf_node, node, node, list) {
8282       switch (node->type) {
8283       case nir_cf_node_block:
8284          visit_block(ctx, nir_cf_node_as_block(node));
8285          break;
8286       case nir_cf_node_if:
8287          visit_if(ctx, nir_cf_node_as_if(node));
8288          break;
8289       case nir_cf_node_loop:
8290          visit_loop(ctx, nir_cf_node_as_loop(node));
8291          break;
8292       default:
8293          unreachable("unimplemented cf list type");
8294       }
8295    }
8296 }
8297
8298 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8299 {
8300    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8301    uint64_t mask = ctx->outputs.mask[slot];
8302    if (!is_pos && !mask)
8303       return;
8304    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8305       return;
8306    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8307    exp->enabled_mask = mask;
8308    for (unsigned i = 0; i < 4; ++i) {
8309       if (mask & (1 << i))
8310          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8311       else
8312          exp->operands[i] = Operand(v1);
8313    }
8314    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8315     * Setting valid_mask=1 prevents it and has no other effect.
8316     */
8317    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8318    exp->done = false;
8319    exp->compressed = false;
8320    if (is_pos)
8321       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8322    else
8323       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8324    ctx->block->instructions.emplace_back(std::move(exp));
8325 }
8326
8327 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8328 {
8329    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8330    exp->enabled_mask = 0;
8331    for (unsigned i = 0; i < 4; ++i)
8332       exp->operands[i] = Operand(v1);
8333    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8334       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8335       exp->enabled_mask |= 0x1;
8336    }
8337    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8338       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8339       exp->enabled_mask |= 0x4;
8340    }
8341    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8342       if (ctx->options->chip_class < GFX9) {
8343          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8344          exp->enabled_mask |= 0x8;
8345       } else {
8346          Builder bld(ctx->program, ctx->block);
8347
8348          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8349                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8350          if (exp->operands[2].isTemp())
8351             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8352
8353          exp->operands[2] = Operand(out);
8354          exp->enabled_mask |= 0x4;
8355       }
8356    }
8357    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8358    exp->done = false;
8359    exp->compressed = false;
8360    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8361    ctx->block->instructions.emplace_back(std::move(exp));
8362 }
8363
8364 static void create_vs_exports(isel_context *ctx)
8365 {
8366    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8367
8368    if (outinfo->export_prim_id) {
8369       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8370       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8371    }
8372
8373    if (ctx->options->key.has_multiview_view_index) {
8374       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8375       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8376    }
8377
8378    /* the order these position exports are created is important */
8379    int next_pos = 0;
8380    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8381    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8382       export_vs_psiz_layer_viewport(ctx, &next_pos);
8383    }
8384    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8385       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8386    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8387       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8388
8389    if (ctx->export_clip_dists) {
8390       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8391          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8392       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8393          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8394    }
8395
8396    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8397       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8398           i != VARYING_SLOT_PRIMITIVE_ID)
8399          continue;
8400
8401       export_vs_varying(ctx, i, false, NULL);
8402    }
8403 }
8404
8405 static void export_fs_mrt_z(isel_context *ctx)
8406 {
8407    Builder bld(ctx->program, ctx->block);
8408    unsigned enabled_channels = 0;
8409    bool compr = false;
8410    Operand values[4];
8411
8412    for (unsigned i = 0; i < 4; ++i) {
8413       values[i] = Operand(v1);
8414    }
8415
8416    /* Both stencil and sample mask only need 16-bits. */
8417    if (!ctx->program->info->ps.writes_z &&
8418        (ctx->program->info->ps.writes_stencil ||
8419         ctx->program->info->ps.writes_sample_mask)) {
8420       compr = true; /* COMPR flag */
8421
8422       if (ctx->program->info->ps.writes_stencil) {
8423          /* Stencil should be in X[23:16]. */
8424          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8425          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8426          enabled_channels |= 0x3;
8427       }
8428
8429       if (ctx->program->info->ps.writes_sample_mask) {
8430          /* SampleMask should be in Y[15:0]. */
8431          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8432          enabled_channels |= 0xc;
8433      }
8434    } else {
8435       if (ctx->program->info->ps.writes_z) {
8436          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8437          enabled_channels |= 0x1;
8438       }
8439
8440       if (ctx->program->info->ps.writes_stencil) {
8441          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8442          enabled_channels |= 0x2;
8443       }
8444
8445       if (ctx->program->info->ps.writes_sample_mask) {
8446          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8447          enabled_channels |= 0x4;
8448       }
8449    }
8450
8451    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8452     * writemask component.
8453     */
8454    if (ctx->options->chip_class == GFX6 &&
8455        ctx->options->family != CHIP_OLAND &&
8456        ctx->options->family != CHIP_HAINAN) {
8457             enabled_channels |= 0x1;
8458    }
8459
8460    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8461            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8462 }
8463
8464 static void export_fs_mrt_color(isel_context *ctx, int slot)
8465 {
8466    Builder bld(ctx->program, ctx->block);
8467    unsigned write_mask = ctx->outputs.mask[slot];
8468    Operand values[4];
8469
8470    for (unsigned i = 0; i < 4; ++i) {
8471       if (write_mask & (1 << i)) {
8472          values[i] = Operand(ctx->outputs.outputs[slot][i]);
8473       } else {
8474          values[i] = Operand(v1);
8475       }
8476    }
8477
8478    unsigned target, col_format;
8479    unsigned enabled_channels = 0;
8480    aco_opcode compr_op = (aco_opcode)0;
8481
8482    slot -= FRAG_RESULT_DATA0;
8483    target = V_008DFC_SQ_EXP_MRT + slot;
8484    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
8485
8486    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
8487    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
8488
8489    switch (col_format)
8490    {
8491    case V_028714_SPI_SHADER_ZERO:
8492       enabled_channels = 0; /* writemask */
8493       target = V_008DFC_SQ_EXP_NULL;
8494       break;
8495
8496    case V_028714_SPI_SHADER_32_R:
8497       enabled_channels = 1;
8498       break;
8499
8500    case V_028714_SPI_SHADER_32_GR:
8501       enabled_channels = 0x3;
8502       break;
8503
8504    case V_028714_SPI_SHADER_32_AR:
8505       if (ctx->options->chip_class >= GFX10) {
8506          /* Special case: on GFX10, the outputs are different for 32_AR */
8507          enabled_channels = 0x3;
8508          values[1] = values[3];
8509          values[3] = Operand(v1);
8510       } else {
8511          enabled_channels = 0x9;
8512       }
8513       break;
8514
8515    case V_028714_SPI_SHADER_FP16_ABGR:
8516       enabled_channels = 0x5;
8517       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
8518       break;
8519
8520    case V_028714_SPI_SHADER_UNORM16_ABGR:
8521       enabled_channels = 0x5;
8522       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
8523       break;
8524
8525    case V_028714_SPI_SHADER_SNORM16_ABGR:
8526       enabled_channels = 0x5;
8527       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
8528       break;
8529
8530    case V_028714_SPI_SHADER_UINT16_ABGR: {
8531       enabled_channels = 0x5;
8532       compr_op = aco_opcode::v_cvt_pk_u16_u32;
8533       if (is_int8 || is_int10) {
8534          /* clamp */
8535          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
8536          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8537
8538          for (unsigned i = 0; i < 4; i++) {
8539             if ((write_mask >> i) & 1) {
8540                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
8541                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
8542                                     values[i]);
8543             }
8544          }
8545       }
8546       break;
8547    }
8548
8549    case V_028714_SPI_SHADER_SINT16_ABGR:
8550       enabled_channels = 0x5;
8551       compr_op = aco_opcode::v_cvt_pk_i16_i32;
8552       if (is_int8 || is_int10) {
8553          /* clamp */
8554          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
8555          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
8556          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8557          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
8558
8559          for (unsigned i = 0; i < 4; i++) {
8560             if ((write_mask >> i) & 1) {
8561                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
8562                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
8563                                     values[i]);
8564                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
8565                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
8566                                     values[i]);
8567             }
8568          }
8569       }
8570       break;
8571
8572    case V_028714_SPI_SHADER_32_ABGR:
8573       enabled_channels = 0xF;
8574       break;
8575
8576    default:
8577       break;
8578    }
8579
8580    if (target == V_008DFC_SQ_EXP_NULL)
8581       return;
8582
8583    if ((bool) compr_op) {
8584       for (int i = 0; i < 2; i++) {
8585          /* check if at least one of the values to be compressed is enabled */
8586          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
8587          if (enabled) {
8588             enabled_channels |= enabled << (i*2);
8589             values[i] = bld.vop3(compr_op, bld.def(v1),
8590                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
8591                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
8592          } else {
8593             values[i] = Operand(v1);
8594          }
8595       }
8596       values[2] = Operand(v1);
8597       values[3] = Operand(v1);
8598    } else {
8599       for (int i = 0; i < 4; i++)
8600          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
8601    }
8602
8603    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8604            enabled_channels, target, (bool) compr_op);
8605 }
8606
8607 static void create_fs_exports(isel_context *ctx)
8608 {
8609    /* Export depth, stencil and sample mask. */
8610    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
8611        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
8612        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
8613       export_fs_mrt_z(ctx);
8614    }
8615
8616    /* Export all color render targets. */
8617    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
8618       if (ctx->outputs.mask[i])
8619          export_fs_mrt_color(ctx, i);
8620    }
8621 }
8622
8623 static void emit_stream_output(isel_context *ctx,
8624                                Temp const *so_buffers,
8625                                Temp const *so_write_offset,
8626                                const struct radv_stream_output *output)
8627 {
8628    unsigned num_comps = util_bitcount(output->component_mask);
8629    unsigned writemask = (1 << num_comps) - 1;
8630    unsigned loc = output->location;
8631    unsigned buf = output->buffer;
8632
8633    assert(num_comps && num_comps <= 4);
8634    if (!num_comps || num_comps > 4)
8635       return;
8636
8637    unsigned start = ffs(output->component_mask) - 1;
8638
8639    Temp out[4];
8640    bool all_undef = true;
8641    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
8642    for (unsigned i = 0; i < num_comps; i++) {
8643       out[i] = ctx->outputs.outputs[loc][start + i];
8644       all_undef = all_undef && !out[i].id();
8645    }
8646    if (all_undef)
8647       return;
8648
8649    while (writemask) {
8650       int start, count;
8651       u_bit_scan_consecutive_range(&writemask, &start, &count);
8652       if (count == 3 && ctx->options->chip_class == GFX6) {
8653          /* GFX6 doesn't support storing vec3, split it. */
8654          writemask |= 1u << (start + 2);
8655          count = 2;
8656       }
8657
8658       unsigned offset = output->offset + start * 4;
8659
8660       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8661       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8662       for (int i = 0; i < count; ++i)
8663          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8664       vec->definitions[0] = Definition(write_data);
8665       ctx->block->instructions.emplace_back(std::move(vec));
8666
8667       aco_opcode opcode;
8668       switch (count) {
8669       case 1:
8670          opcode = aco_opcode::buffer_store_dword;
8671          break;
8672       case 2:
8673          opcode = aco_opcode::buffer_store_dwordx2;
8674          break;
8675       case 3:
8676          opcode = aco_opcode::buffer_store_dwordx3;
8677          break;
8678       case 4:
8679          opcode = aco_opcode::buffer_store_dwordx4;
8680          break;
8681       default:
8682          unreachable("Unsupported dword count.");
8683       }
8684
8685       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8686       store->operands[0] = Operand(so_write_offset[buf]);
8687       store->operands[1] = Operand(so_buffers[buf]);
8688       store->operands[2] = Operand((uint32_t) 0);
8689       store->operands[3] = Operand(write_data);
8690       if (offset > 4095) {
8691          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8692          Builder bld(ctx->program, ctx->block);
8693          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8694       } else {
8695          store->offset = offset;
8696       }
8697       store->offen = true;
8698       store->glc = true;
8699       store->dlc = false;
8700       store->slc = true;
8701       store->can_reorder = true;
8702       ctx->block->instructions.emplace_back(std::move(store));
8703    }
8704 }
8705
8706 static void emit_streamout(isel_context *ctx, unsigned stream)
8707 {
8708    Builder bld(ctx->program, ctx->block);
8709
8710    Temp so_buffers[4];
8711    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8712    for (unsigned i = 0; i < 4; i++) {
8713       unsigned stride = ctx->program->info->so.strides[i];
8714       if (!stride)
8715          continue;
8716
8717       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
8718       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
8719    }
8720
8721    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8722                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8723
8724    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8725
8726    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8727
8728    if_context ic;
8729    begin_divergent_if_then(ctx, &ic, can_emit);
8730
8731    bld.reset(ctx->block);
8732
8733    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8734
8735    Temp so_write_offset[4];
8736
8737    for (unsigned i = 0; i < 4; i++) {
8738       unsigned stride = ctx->program->info->so.strides[i];
8739       if (!stride)
8740          continue;
8741
8742       if (stride == 1) {
8743          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8744                                 get_arg(ctx, ctx->args->streamout_write_idx),
8745                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8746          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8747
8748          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8749       } else {
8750          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8751          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8752                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8753          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8754       }
8755    }
8756
8757    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8758       struct radv_stream_output *output =
8759          &ctx->program->info->so.outputs[i];
8760       if (stream != output->stream)
8761          continue;
8762
8763       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8764    }
8765
8766    begin_divergent_if_else(ctx, &ic);
8767    end_divergent_if(ctx, &ic);
8768 }
8769
8770 } /* end namespace */
8771
8772 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8773 {
8774    /* Split all arguments except for the first (ring_offsets) and the last
8775     * (exec) so that the dead channels don't stay live throughout the program.
8776     */
8777    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8778       if (startpgm->definitions[i].regClass().size() > 1) {
8779          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8780                            startpgm->definitions[i].regClass().size());
8781       }
8782    }
8783 }
8784
8785 void handle_bc_optimize(isel_context *ctx)
8786 {
8787    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8788    Builder bld(ctx->program, ctx->block);
8789    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8790    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8791    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8792    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8793    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8794    if (uses_center && uses_centroid) {
8795       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8796                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8797
8798       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8799          Temp new_coord[2];
8800          for (unsigned i = 0; i < 2; i++) {
8801             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8802             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8803             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8804                                     persp_centroid, persp_center, sel);
8805          }
8806          ctx->persp_centroid = bld.tmp(v2);
8807          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8808                     Operand(new_coord[0]), Operand(new_coord[1]));
8809          emit_split_vector(ctx, ctx->persp_centroid, 2);
8810       }
8811
8812       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8813          Temp new_coord[2];
8814          for (unsigned i = 0; i < 2; i++) {
8815             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8816             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8817             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8818                                     linear_centroid, linear_center, sel);
8819          }
8820          ctx->linear_centroid = bld.tmp(v2);
8821          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8822                     Operand(new_coord[0]), Operand(new_coord[1]));
8823          emit_split_vector(ctx, ctx->linear_centroid, 2);
8824       }
8825    }
8826 }
8827
8828 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8829 {
8830    Program *program = ctx->program;
8831
8832    unsigned float_controls = shader->info.float_controls_execution_mode;
8833
8834    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8835       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8836    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8837       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8838                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8839
8840    program->next_fp_mode.must_flush_denorms32 =
8841       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8842    program->next_fp_mode.must_flush_denorms16_64 =
8843       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8844                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8845
8846    program->next_fp_mode.care_about_round32 =
8847       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8848
8849    program->next_fp_mode.care_about_round16_64 =
8850       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8851                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8852
8853    /* default to preserving fp16 and fp64 denorms, since it's free */
8854    if (program->next_fp_mode.must_flush_denorms16_64)
8855       program->next_fp_mode.denorm16_64 = 0;
8856    else
8857       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8858
8859    /* preserving fp32 denorms is expensive, so only do it if asked */
8860    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8861       program->next_fp_mode.denorm32 = fp_denorm_keep;
8862    else
8863       program->next_fp_mode.denorm32 = 0;
8864
8865    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8866       program->next_fp_mode.round32 = fp_round_tz;
8867    else
8868       program->next_fp_mode.round32 = fp_round_ne;
8869
8870    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8871       program->next_fp_mode.round16_64 = fp_round_tz;
8872    else
8873       program->next_fp_mode.round16_64 = fp_round_ne;
8874
8875    ctx->block->fp_mode = program->next_fp_mode;
8876 }
8877
8878 void cleanup_cfg(Program *program)
8879 {
8880    /* create linear_succs/logical_succs */
8881    for (Block& BB : program->blocks) {
8882       for (unsigned idx : BB.linear_preds)
8883          program->blocks[idx].linear_succs.emplace_back(BB.index);
8884       for (unsigned idx : BB.logical_preds)
8885          program->blocks[idx].logical_succs.emplace_back(BB.index);
8886    }
8887 }
8888
8889 void select_program(Program *program,
8890                     unsigned shader_count,
8891                     struct nir_shader *const *shaders,
8892                     ac_shader_config* config,
8893                     struct radv_shader_args *args)
8894 {
8895    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
8896
8897    for (unsigned i = 0; i < shader_count; i++) {
8898       nir_shader *nir = shaders[i];
8899       init_context(&ctx, nir);
8900
8901       setup_fp_mode(&ctx, nir);
8902
8903       if (!i) {
8904          /* needs to be after init_context() for FS */
8905          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8906          append_logical_start(ctx.block);
8907          split_arguments(&ctx, startpgm);
8908       }
8909
8910       if_context ic;
8911       if (shader_count >= 2) {
8912          Builder bld(ctx.program, ctx.block);
8913          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
8914          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8915          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8916
8917          begin_divergent_if_then(&ctx, &ic, cond);
8918       }
8919
8920       if (i) {
8921          Builder bld(ctx.program, ctx.block);
8922          assert(ctx.stage == vertex_geometry_gs);
8923          bld.barrier(aco_opcode::p_memory_barrier_shared);
8924          bld.sopp(aco_opcode::s_barrier);
8925
8926          ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
8927       } else if (ctx.stage == geometry_gs)
8928          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
8929
8930       if (ctx.stage == fragment_fs)
8931          handle_bc_optimize(&ctx);
8932
8933       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8934       visit_cf_list(&ctx, &func->body);
8935
8936       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
8937          emit_streamout(&ctx, 0);
8938
8939       if (ctx.stage == vertex_vs) {
8940          create_vs_exports(&ctx);
8941       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
8942          Builder bld(ctx.program, ctx.block);
8943          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
8944          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
8945       }
8946
8947       if (ctx.stage == fragment_fs)
8948          create_fs_exports(&ctx);
8949
8950       if (shader_count >= 2) {
8951          begin_divergent_if_else(&ctx, &ic);
8952          end_divergent_if(&ctx, &ic);
8953       }
8954
8955       ralloc_free(ctx.divergent_vals);
8956    }
8957
8958    program->config->float_mode = program->blocks[0].fp_mode.val;
8959
8960    append_logical_end(ctx.block);
8961    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
8962    Builder bld(ctx.program, ctx.block);
8963    if (ctx.program->wb_smem_l1_on_end)
8964       bld.smem(aco_opcode::s_dcache_wb, false);
8965    bld.sopp(aco_opcode::s_endpgm);
8966
8967    cleanup_cfg(program);
8968 }
8969
8970 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
8971                            ac_shader_config* config,
8972                            struct radv_shader_args *args)
8973 {
8974    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
8975
8976    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
8977    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
8978    program->next_fp_mode.must_flush_denorms32 = false;
8979    program->next_fp_mode.must_flush_denorms16_64 = false;
8980    program->next_fp_mode.care_about_round32 = false;
8981    program->next_fp_mode.care_about_round16_64 = false;
8982    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8983    program->next_fp_mode.denorm32 = 0;
8984    program->next_fp_mode.round32 = fp_round_ne;
8985    program->next_fp_mode.round16_64 = fp_round_ne;
8986    ctx.block->fp_mode = program->next_fp_mode;
8987
8988    add_startpgm(&ctx);
8989    append_logical_start(ctx.block);
8990
8991    Builder bld(ctx.program, ctx.block);
8992
8993    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
8994
8995    Operand stream_id(0u);
8996    if (args->shader_info->so.num_outputs)
8997       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8998                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
8999
9000    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9001
9002    std::stack<Block> endif_blocks;
9003
9004    for (unsigned stream = 0; stream < 4; stream++) {
9005       if (stream_id.isConstant() && stream != stream_id.constantValue())
9006          continue;
9007
9008       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9009       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9010          continue;
9011
9012       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9013
9014       unsigned BB_if_idx = ctx.block->index;
9015       Block BB_endif = Block();
9016       if (!stream_id.isConstant()) {
9017          /* begin IF */
9018          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9019          append_logical_end(ctx.block);
9020          ctx.block->kind |= block_kind_uniform;
9021          bld.branch(aco_opcode::p_cbranch_z, cond);
9022
9023          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9024
9025          ctx.block = ctx.program->create_and_insert_block();
9026          add_edge(BB_if_idx, ctx.block);
9027          bld.reset(ctx.block);
9028          append_logical_start(ctx.block);
9029       }
9030
9031       unsigned offset = 0;
9032       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9033          if (args->shader_info->gs.output_streams[i] != stream)
9034             continue;
9035
9036          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9037          unsigned length = util_last_bit(output_usage_mask);
9038          for (unsigned j = 0; j < length; ++j) {
9039             if (!(output_usage_mask & (1 << j)))
9040                continue;
9041
9042             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9043             Temp voffset = vtx_offset;
9044             if (const_offset >= 4096u) {
9045                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9046                const_offset %= 4096u;
9047             }
9048
9049             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9050             mubuf->definitions[0] = bld.def(v1);
9051             mubuf->operands[0] = Operand(voffset);
9052             mubuf->operands[1] = Operand(gsvs_ring);
9053             mubuf->operands[2] = Operand(0u);
9054             mubuf->offen = true;
9055             mubuf->offset = const_offset;
9056             mubuf->glc = true;
9057             mubuf->slc = true;
9058             mubuf->dlc = args->options->chip_class >= GFX10;
9059             mubuf->barrier = barrier_none;
9060             mubuf->can_reorder = true;
9061
9062             ctx.outputs.mask[i] |= 1 << j;
9063             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
9064
9065             bld.insert(std::move(mubuf));
9066
9067             offset++;
9068          }
9069       }
9070
9071       if (args->shader_info->so.num_outputs) {
9072          emit_streamout(&ctx, stream);
9073          bld.reset(ctx.block);
9074       }
9075
9076       if (stream == 0) {
9077          create_vs_exports(&ctx);
9078          ctx.block->kind |= block_kind_export_end;
9079       }
9080
9081       if (!stream_id.isConstant()) {
9082          append_logical_end(ctx.block);
9083
9084          /* branch from then block to endif block */
9085          bld.branch(aco_opcode::p_branch);
9086          add_edge(ctx.block->index, &BB_endif);
9087          ctx.block->kind |= block_kind_uniform;
9088
9089          /* emit else block */
9090          ctx.block = ctx.program->create_and_insert_block();
9091          add_edge(BB_if_idx, ctx.block);
9092          bld.reset(ctx.block);
9093          append_logical_start(ctx.block);
9094
9095          endif_blocks.push(std::move(BB_endif));
9096       }
9097    }
9098
9099    while (!endif_blocks.empty()) {
9100       Block BB_endif = std::move(endif_blocks.top());
9101       endif_blocks.pop();
9102
9103       Block *BB_else = ctx.block;
9104
9105       append_logical_end(BB_else);
9106       /* branch from else block to endif block */
9107       bld.branch(aco_opcode::p_branch);
9108       add_edge(BB_else->index, &BB_endif);
9109       BB_else->kind |= block_kind_uniform;
9110
9111       /** emit endif merge block */
9112       ctx.block = program->insert_block(std::move(BB_endif));
9113       bld.reset(ctx.block);
9114       append_logical_start(ctx.block);
9115    }
9116
9117    program->config->float_mode = program->blocks[0].fp_mode.val;
9118
9119    append_logical_end(ctx.block);
9120    ctx.block->kind |= block_kind_uniform;
9121    bld.sopp(aco_opcode::s_endpgm);
9122
9123    cleanup_cfg(program);
9124 }
9125 }