src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.readlane(bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 439                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 440 {
 441    Builder bld(ctx->program, ctx->block);
 442    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 443    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 444    if (src1.type() == RegType::sgpr) {
 445       if (commutative && src0.type() == RegType::vgpr) {
 446          Temp t = src0;
 447          src0 = src1;
 448          src1 = t;
 449       } else if (src0.type() == RegType::vgpr &&
 450                  op != aco_opcode::v_madmk_f32 &&
 451                  op != aco_opcode::v_madak_f32 &&
 452                  op != aco_opcode::v_madmk_f16 &&
 453                  op != aco_opcode::v_madak_f16) {
 454          /* If the instruction is not commutative, we emit a VOP3A instruction */
 455          bld.vop2_e64(op, Definition(dst), src0, src1);
 456          return;
 457       } else {
 458          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 459       }
 460    }
 461
 462    if (flush_denorms && ctx->program->chip_class < GFX9) {
 463       assert(dst.size() == 1);
 464       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 465       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 466    } else {
 467       bld.vop2(op, Definition(dst), src0, src1);
 468    }
 469 }
 470
 471 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 472                             bool flush_denorms = false)
 473 {
 474    Temp src0 = get_alu_src(ctx, instr->src[0]);
 475    Temp src1 = get_alu_src(ctx, instr->src[1]);
 476    Temp src2 = get_alu_src(ctx, instr->src[2]);
 477
 478    /* ensure that the instruction has at most 1 sgpr operand
 479     * The optimizer will inline constants for us */
 480    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 481       src0 = as_vgpr(ctx, src0);
 482    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 483       src1 = as_vgpr(ctx, src1);
 484    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 485       src2 = as_vgpr(ctx, src2);
 486
 487    Builder bld(ctx->program, ctx->block);
 488    if (flush_denorms && ctx->program->chip_class < GFX9) {
 489       assert(dst.size() == 1);
 490       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 491       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 492    } else {
 493       bld.vop3(op, Definition(dst), src0, src1, src2);
 494    }
 495 }
 496
 497 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 498 {
 499    Builder bld(ctx->program, ctx->block);
 500    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 501 }
 502
 503 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 504 {
 505    Temp src0 = get_alu_src(ctx, instr->src[0]);
 506    Temp src1 = get_alu_src(ctx, instr->src[1]);
 507    assert(src0.size() == src1.size());
 508
 509    aco_ptr<Instruction> vopc;
 510    if (src1.type() == RegType::sgpr) {
 511       if (src0.type() == RegType::vgpr) {
 512          /* to swap the operands, we might also have to change the opcode */
 513          switch (op) {
 514             case aco_opcode::v_cmp_lt_f32:
 515                op = aco_opcode::v_cmp_gt_f32;
 516                break;
 517             case aco_opcode::v_cmp_ge_f32:
 518                op = aco_opcode::v_cmp_le_f32;
 519                break;
 520             case aco_opcode::v_cmp_lt_i32:
 521                op = aco_opcode::v_cmp_gt_i32;
 522                break;
 523             case aco_opcode::v_cmp_ge_i32:
 524                op = aco_opcode::v_cmp_le_i32;
 525                break;
 526             case aco_opcode::v_cmp_lt_u32:
 527                op = aco_opcode::v_cmp_gt_u32;
 528                break;
 529             case aco_opcode::v_cmp_ge_u32:
 530                op = aco_opcode::v_cmp_le_u32;
 531                break;
 532             case aco_opcode::v_cmp_lt_f64:
 533                op = aco_opcode::v_cmp_gt_f64;
 534                break;
 535             case aco_opcode::v_cmp_ge_f64:
 536                op = aco_opcode::v_cmp_le_f64;
 537                break;
 538             case aco_opcode::v_cmp_lt_i64:
 539                op = aco_opcode::v_cmp_gt_i64;
 540                break;
 541             case aco_opcode::v_cmp_ge_i64:
 542                op = aco_opcode::v_cmp_le_i64;
 543                break;
 544             case aco_opcode::v_cmp_lt_u64:
 545                op = aco_opcode::v_cmp_gt_u64;
 546                break;
 547             case aco_opcode::v_cmp_ge_u64:
 548                op = aco_opcode::v_cmp_le_u64;
 549                break;
 550             default: /* eq and ne are commutative */
 551                break;
 552          }
 553          Temp t = src0;
 554          src0 = src1;
 555          src1 = t;
 556       } else {
 557          src1 = as_vgpr(ctx, src1);
 558       }
 559    }
 560
 561    Builder bld(ctx->program, ctx->block);
 562    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 563 }
 564
 565 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 566 {
 567    Temp src0 = get_alu_src(ctx, instr->src[0]);
 568    Temp src1 = get_alu_src(ctx, instr->src[1]);
 569    Builder bld(ctx->program, ctx->block);
 570
 571    assert(dst.regClass() == bld.lm);
 572    assert(src0.type() == RegType::sgpr);
 573    assert(src1.type() == RegType::sgpr);
 574    assert(src0.regClass() == src1.regClass());
 575
 576    /* Emit the SALU comparison instruction */
 577    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 578    /* Turn the result into a per-lane bool */
 579    bool_to_vector_condition(ctx, cmp, dst);
 580 }
 581
 582 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 583                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 584 {
 585    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 586    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 587    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 588    bool use_valu = s_op == aco_opcode::num_opcodes ||
 589                    divergent_vals ||
 590                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 591                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 592    aco_opcode op = use_valu ? v_op : s_op;
 593    assert(op != aco_opcode::num_opcodes);
 594    assert(dst.regClass() == ctx->program->lane_mask);
 595
 596    if (use_valu)
 597       emit_vopc_instruction(ctx, instr, op, dst);
 598    else
 599       emit_sopc_instruction(ctx, instr, op, dst);
 600 }
 601
 602 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 603 {
 604    Builder bld(ctx->program, ctx->block);
 605    Temp src0 = get_alu_src(ctx, instr->src[0]);
 606    Temp src1 = get_alu_src(ctx, instr->src[1]);
 607
 608    assert(dst.regClass() == bld.lm);
 609    assert(src0.regClass() == bld.lm);
 610    assert(src1.regClass() == bld.lm);
 611
 612    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 613 }
 614
 615 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 616 {
 617    Builder bld(ctx->program, ctx->block);
 618    Temp cond = get_alu_src(ctx, instr->src[0]);
 619    Temp then = get_alu_src(ctx, instr->src[1]);
 620    Temp els = get_alu_src(ctx, instr->src[2]);
 621
 622    assert(cond.regClass() == bld.lm);
 623
 624    if (dst.type() == RegType::vgpr) {
 625       aco_ptr<Instruction> bcsel;
 626       if (dst.size() == 1) {
 627          then = as_vgpr(ctx, then);
 628          els = as_vgpr(ctx, els);
 629
 630          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 631       } else if (dst.size() == 2) {
 632          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 633          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 634          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 635          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 636
 637          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 638          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 639
 640          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 641       } else {
 642          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 643          nir_print_instr(&instr->instr, stderr);
 644          fprintf(stderr, "\n");
 645       }
 646       return;
 647    }
 648
 649    if (instr->dest.dest.ssa.bit_size == 1) {
 650       assert(dst.regClass() == bld.lm);
 651       assert(then.regClass() == bld.lm);
 652       assert(els.regClass() == bld.lm);
 653    }
 654
 655    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 656       if (dst.regClass() == s1 || dst.regClass() == s2) {
 657          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 658          assert(dst.size() == then.size());
 659          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 660          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 661       } else {
 662          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 663          nir_print_instr(&instr->instr, stderr);
 664          fprintf(stderr, "\n");
 665       }
 666       return;
 667    }
 668
 669    /* divergent boolean bcsel
 670     * this implements bcsel on bools: dst = s0 ? s1 : s2
 671     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 672    assert(instr->dest.dest.ssa.bit_size == 1);
 673
 674    if (cond.id() != then.id())
 675       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 676
 677    if (cond.id() == els.id())
 678       bld.sop1(Builder::s_mov, Definition(dst), then);
 679    else
 680       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 681                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 682 }
 683
 684 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 685                     aco_opcode op, uint32_t undo)
 686 {
 687    /* multiply by 16777216 to handle denormals */
 688    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 689                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 690    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 691    scaled = bld.vop1(op, bld.def(v1), scaled);
 692    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 693
 694    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 695
 696    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 697 }
 698
 699 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 700 {
 701    if (ctx->block->fp_mode.denorm32 == 0) {
 702       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 703       return;
 704    }
 705
 706    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 707 }
 708
 709 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 710 {
 711    if (ctx->block->fp_mode.denorm32 == 0) {
 712       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 713       return;
 714    }
 715
 716    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 717 }
 718
 719 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 720 {
 721    if (ctx->block->fp_mode.denorm32 == 0) {
 722       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 723       return;
 724    }
 725
 726    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 727 }
 728
 729 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 730 {
 731    if (ctx->block->fp_mode.denorm32 == 0) {
 732       bld.vop1(aco_opcode::v_log_f32, dst, val);
 733       return;
 734    }
 735
 736    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 737 }
 738
 739 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 740 {
 741    if (!instr->dest.dest.is_ssa) {
 742       fprintf(stderr, "nir alu dst not in ssa: ");
 743       nir_print_instr(&instr->instr, stderr);
 744       fprintf(stderr, "\n");
 745       abort();
 746    }
 747    Builder bld(ctx->program, ctx->block);
 748    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 749    switch(instr->op) {
 750    case nir_op_vec2:
 751    case nir_op_vec3:
 752    case nir_op_vec4: {
 753       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 754       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 755       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 756          elems[i] = get_alu_src(ctx, instr->src[i]);
 757          vec->operands[i] = Operand{elems[i]};
 758       }
 759       vec->definitions[0] = Definition(dst);
 760       ctx->block->instructions.emplace_back(std::move(vec));
 761       ctx->allocated_vec.emplace(dst.id(), elems);
 762       break;
 763    }
 764    case nir_op_mov: {
 765       Temp src = get_alu_src(ctx, instr->src[0]);
 766       aco_ptr<Instruction> mov;
 767       if (dst.type() == RegType::sgpr) {
 768          if (src.type() == RegType::vgpr)
 769             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 770          else if (src.regClass() == s1)
 771             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 772          else if (src.regClass() == s2)
 773             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 774          else
 775             unreachable("wrong src register class for nir_op_imov");
 776       } else if (dst.regClass() == v1) {
 777          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 778       } else if (dst.regClass() == v2) {
 779          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 780       } else {
 781          nir_print_instr(&instr->instr, stderr);
 782          unreachable("Should have been lowered to scalar.");
 783       }
 784       break;
 785    }
 786    case nir_op_inot: {
 787       Temp src = get_alu_src(ctx, instr->src[0]);
 788       if (instr->dest.dest.ssa.bit_size == 1) {
 789          assert(src.regClass() == bld.lm);
 790          assert(dst.regClass() == bld.lm);
 791          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 792          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 793          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 794       } else if (dst.regClass() == v1) {
 795          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 796       } else if (dst.type() == RegType::sgpr) {
 797          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 798          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 799       } else {
 800          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 801          nir_print_instr(&instr->instr, stderr);
 802          fprintf(stderr, "\n");
 803       }
 804       break;
 805    }
 806    case nir_op_ineg: {
 807       Temp src = get_alu_src(ctx, instr->src[0]);
 808       if (dst.regClass() == v1) {
 809          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 810       } else if (dst.regClass() == s1) {
 811          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 812       } else if (dst.size() == 2) {
 813          Temp src0 = bld.tmp(dst.type(), 1);
 814          Temp src1 = bld.tmp(dst.type(), 1);
 815          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 816
 817          if (dst.regClass() == s2) {
 818             Temp carry = bld.tmp(s1);
 819             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 820             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 821             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 822          } else {
 823             Temp lower = bld.tmp(v1);
 824             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 825             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 826             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 827          }
 828       } else {
 829          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 830          nir_print_instr(&instr->instr, stderr);
 831          fprintf(stderr, "\n");
 832       }
 833       break;
 834    }
 835    case nir_op_iabs: {
 836       if (dst.regClass() == s1) {
 837          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 838       } else if (dst.regClass() == v1) {
 839          Temp src = get_alu_src(ctx, instr->src[0]);
 840          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 841       } else {
 842          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 843          nir_print_instr(&instr->instr, stderr);
 844          fprintf(stderr, "\n");
 845       }
 846       break;
 847    }
 848    case nir_op_isign: {
 849       Temp src = get_alu_src(ctx, instr->src[0]);
 850       if (dst.regClass() == s1) {
 851          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 852          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 853          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 854       } else if (dst.regClass() == s2) {
 855          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 856          Temp neqz;
 857          if (ctx->program->chip_class >= GFX8)
 858             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 859          else
 860             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 861          /* SCC gets zero-extended to 64 bit */
 862          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 863       } else if (dst.regClass() == v1) {
 864          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 865          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 866          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 867       } else if (dst.regClass() == v2) {
 868          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 869          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 870          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 871          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 872          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 873          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 874       } else {
 875          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 876          nir_print_instr(&instr->instr, stderr);
 877          fprintf(stderr, "\n");
 878       }
 879       break;
 880    }
 881    case nir_op_imax: {
 882       if (dst.regClass() == v1) {
 883          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 884       } else if (dst.regClass() == s1) {
 885          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 886       } else {
 887          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 888          nir_print_instr(&instr->instr, stderr);
 889          fprintf(stderr, "\n");
 890       }
 891       break;
 892    }
 893    case nir_op_umax: {
 894       if (dst.regClass() == v1) {
 895          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 896       } else if (dst.regClass() == s1) {
 897          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 898       } else {
 899          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 900          nir_print_instr(&instr->instr, stderr);
 901          fprintf(stderr, "\n");
 902       }
 903       break;
 904    }
 905    case nir_op_imin: {
 906       if (dst.regClass() == v1) {
 907          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 908       } else if (dst.regClass() == s1) {
 909          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 910       } else {
 911          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 912          nir_print_instr(&instr->instr, stderr);
 913          fprintf(stderr, "\n");
 914       }
 915       break;
 916    }
 917    case nir_op_umin: {
 918       if (dst.regClass() == v1) {
 919          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 920       } else if (dst.regClass() == s1) {
 921          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 922       } else {
 923          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 924          nir_print_instr(&instr->instr, stderr);
 925          fprintf(stderr, "\n");
 926       }
 927       break;
 928    }
 929    case nir_op_ior: {
 930       if (instr->dest.dest.ssa.bit_size == 1) {
 931          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 932       } else if (dst.regClass() == v1) {
 933          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 934       } else if (dst.regClass() == s1) {
 935          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 936       } else if (dst.regClass() == s2) {
 937          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 938       } else {
 939          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 940          nir_print_instr(&instr->instr, stderr);
 941          fprintf(stderr, "\n");
 942       }
 943       break;
 944    }
 945    case nir_op_iand: {
 946       if (instr->dest.dest.ssa.bit_size == 1) {
 947          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 948       } else if (dst.regClass() == v1) {
 949          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 950       } else if (dst.regClass() == s1) {
 951          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 952       } else if (dst.regClass() == s2) {
 953          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 954       } else {
 955          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 956          nir_print_instr(&instr->instr, stderr);
 957          fprintf(stderr, "\n");
 958       }
 959       break;
 960    }
 961    case nir_op_ixor: {
 962       if (instr->dest.dest.ssa.bit_size == 1) {
 963          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 964       } else if (dst.regClass() == v1) {
 965          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 966       } else if (dst.regClass() == s1) {
 967          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 968       } else if (dst.regClass() == s2) {
 969          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 970       } else {
 971          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 972          nir_print_instr(&instr->instr, stderr);
 973          fprintf(stderr, "\n");
 974       }
 975       break;
 976    }
 977    case nir_op_ushr: {
 978       if (dst.regClass() == v1) {
 979          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 980       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 981          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 982                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 983       } else if (dst.regClass() == v2) {
 984          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
 985                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
 986       } else if (dst.regClass() == s2) {
 987          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 988       } else if (dst.regClass() == s1) {
 989          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 990       } else {
 991          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 992          nir_print_instr(&instr->instr, stderr);
 993          fprintf(stderr, "\n");
 994       }
 995       break;
 996    }
 997    case nir_op_ishl: {
 998       if (dst.regClass() == v1) {
 999          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1000       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1001          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1002                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1003       } else if (dst.regClass() == v2) {
1004          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1005                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1006       } else if (dst.regClass() == s1) {
1007          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1008       } else if (dst.regClass() == s2) {
1009          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1010       } else {
1011          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1012          nir_print_instr(&instr->instr, stderr);
1013          fprintf(stderr, "\n");
1014       }
1015       break;
1016    }
1017    case nir_op_ishr: {
1018       if (dst.regClass() == v1) {
1019          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1020       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1021          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1022                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1023       } else if (dst.regClass() == v2) {
1024          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1025                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1026       } else if (dst.regClass() == s1) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1028       } else if (dst.regClass() == s2) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1030       } else {
1031          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1032          nir_print_instr(&instr->instr, stderr);
1033          fprintf(stderr, "\n");
1034       }
1035       break;
1036    }
1037    case nir_op_find_lsb: {
1038       Temp src = get_alu_src(ctx, instr->src[0]);
1039       if (src.regClass() == s1) {
1040          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1041       } else if (src.regClass() == v1) {
1042          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1043       } else if (src.regClass() == s2) {
1044          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1045       } else {
1046          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1047          nir_print_instr(&instr->instr, stderr);
1048          fprintf(stderr, "\n");
1049       }
1050       break;
1051    }
1052    case nir_op_ufind_msb:
1053    case nir_op_ifind_msb: {
1054       Temp src = get_alu_src(ctx, instr->src[0]);
1055       if (src.regClass() == s1 || src.regClass() == s2) {
1056          aco_opcode op = src.regClass() == s2 ?
1057                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1058                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1059          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1060
1061          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1062                                         Operand(src.size() * 32u - 1u), msb_rev);
1063          Temp msb = sub.def(0).getTemp();
1064          Temp carry = sub.def(1).getTemp();
1065
1066          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1067       } else if (src.regClass() == v1) {
1068          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1069          Temp msb_rev = bld.tmp(v1);
1070          emit_vop1_instruction(ctx, instr, op, msb_rev);
1071          Temp msb = bld.tmp(v1);
1072          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1073          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1074       } else {
1075          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1076          nir_print_instr(&instr->instr, stderr);
1077          fprintf(stderr, "\n");
1078       }
1079       break;
1080    }
1081    case nir_op_bitfield_reverse: {
1082       if (dst.regClass() == s1) {
1083          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1084       } else if (dst.regClass() == v1) {
1085          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_iadd: {
1094       if (dst.regClass() == s1) {
1095          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1096          break;
1097       }
1098
1099       Temp src0 = get_alu_src(ctx, instr->src[0]);
1100       Temp src1 = get_alu_src(ctx, instr->src[1]);
1101       if (dst.regClass() == v1) {
1102          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1103          break;
1104       }
1105
1106       assert(src0.size() == 2 && src1.size() == 2);
1107       Temp src00 = bld.tmp(src0.type(), 1);
1108       Temp src01 = bld.tmp(dst.type(), 1);
1109       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1110       Temp src10 = bld.tmp(src1.type(), 1);
1111       Temp src11 = bld.tmp(dst.type(), 1);
1112       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1113
1114       if (dst.regClass() == s2) {
1115          Temp carry = bld.tmp(s1);
1116          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1117          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1118          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1119       } else if (dst.regClass() == v2) {
1120          Temp dst0 = bld.tmp(v1);
1121          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1122          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1123          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1124       } else {
1125          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1126          nir_print_instr(&instr->instr, stderr);
1127          fprintf(stderr, "\n");
1128       }
1129       break;
1130    }
1131    case nir_op_uadd_sat: {
1132       Temp src0 = get_alu_src(ctx, instr->src[0]);
1133       Temp src1 = get_alu_src(ctx, instr->src[1]);
1134       if (dst.regClass() == s1) {
1135          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1136          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1137                   src0, src1);
1138          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1139       } else if (dst.regClass() == v1) {
1140          if (ctx->options->chip_class >= GFX9) {
1141             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1142             add->operands[0] = Operand(src0);
1143             add->operands[1] = Operand(src1);
1144             add->definitions[0] = Definition(dst);
1145             add->clamp = 1;
1146             ctx->block->instructions.emplace_back(std::move(add));
1147          } else {
1148             if (src1.regClass() != v1)
1149                std::swap(src0, src1);
1150             assert(src1.regClass() == v1);
1151             Temp tmp = bld.tmp(v1);
1152             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1153             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1154          }
1155       } else {
1156          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1157          nir_print_instr(&instr->instr, stderr);
1158          fprintf(stderr, "\n");
1159       }
1160       break;
1161    }
1162    case nir_op_uadd_carry: {
1163       Temp src0 = get_alu_src(ctx, instr->src[0]);
1164       Temp src1 = get_alu_src(ctx, instr->src[1]);
1165       if (dst.regClass() == s1) {
1166          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1167          break;
1168       }
1169       if (dst.regClass() == v1) {
1170          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1171          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1172          break;
1173       }
1174
1175       Temp src00 = bld.tmp(src0.type(), 1);
1176       Temp src01 = bld.tmp(dst.type(), 1);
1177       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1178       Temp src10 = bld.tmp(src1.type(), 1);
1179       Temp src11 = bld.tmp(dst.type(), 1);
1180       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1181       if (dst.regClass() == s2) {
1182          Temp carry = bld.tmp(s1);
1183          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1184          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1185          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1186       } else if (dst.regClass() == v2) {
1187          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1188          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1189          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1190          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1191       } else {
1192          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1193          nir_print_instr(&instr->instr, stderr);
1194          fprintf(stderr, "\n");
1195       }
1196       break;
1197    }
1198    case nir_op_isub: {
1199       if (dst.regClass() == s1) {
1200          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1201          break;
1202       }
1203
1204       Temp src0 = get_alu_src(ctx, instr->src[0]);
1205       Temp src1 = get_alu_src(ctx, instr->src[1]);
1206       if (dst.regClass() == v1) {
1207          bld.vsub32(Definition(dst), src0, src1);
1208          break;
1209       }
1210
1211       Temp src00 = bld.tmp(src0.type(), 1);
1212       Temp src01 = bld.tmp(dst.type(), 1);
1213       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1214       Temp src10 = bld.tmp(src1.type(), 1);
1215       Temp src11 = bld.tmp(dst.type(), 1);
1216       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1217       if (dst.regClass() == s2) {
1218          Temp carry = bld.tmp(s1);
1219          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1220          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1221          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1222       } else if (dst.regClass() == v2) {
1223          Temp lower = bld.tmp(v1);
1224          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1225          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1226          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1227       } else {
1228          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1229          nir_print_instr(&instr->instr, stderr);
1230          fprintf(stderr, "\n");
1231       }
1232       break;
1233    }
1234    case nir_op_usub_borrow: {
1235       Temp src0 = get_alu_src(ctx, instr->src[0]);
1236       Temp src1 = get_alu_src(ctx, instr->src[1]);
1237       if (dst.regClass() == s1) {
1238          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1239          break;
1240       } else if (dst.regClass() == v1) {
1241          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1242          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1243          break;
1244       }
1245
1246       Temp src00 = bld.tmp(src0.type(), 1);
1247       Temp src01 = bld.tmp(dst.type(), 1);
1248       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1249       Temp src10 = bld.tmp(src1.type(), 1);
1250       Temp src11 = bld.tmp(dst.type(), 1);
1251       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1252       if (dst.regClass() == s2) {
1253          Temp borrow = bld.tmp(s1);
1254          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1255          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1256          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1257       } else if (dst.regClass() == v2) {
1258          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1259          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1260          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1261          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1262       } else {
1263          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1264          nir_print_instr(&instr->instr, stderr);
1265          fprintf(stderr, "\n");
1266       }
1267       break;
1268    }
1269    case nir_op_imul: {
1270       if (dst.regClass() == v1) {
1271          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1272                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1273       } else if (dst.regClass() == s1) {
1274          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1275       } else {
1276          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1277          nir_print_instr(&instr->instr, stderr);
1278          fprintf(stderr, "\n");
1279       }
1280       break;
1281    }
1282    case nir_op_umul_high: {
1283       if (dst.regClass() == v1) {
1284          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1285       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1286          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1287       } else if (dst.regClass() == s1) {
1288          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1289                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1290          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1291       } else {
1292          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1293          nir_print_instr(&instr->instr, stderr);
1294          fprintf(stderr, "\n");
1295       }
1296       break;
1297    }
1298    case nir_op_imul_high: {
1299       if (dst.regClass() == v1) {
1300          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1301       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1302          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1303       } else if (dst.regClass() == s1) {
1304          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1305                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1306          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1307       } else {
1308          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1309          nir_print_instr(&instr->instr, stderr);
1310          fprintf(stderr, "\n");
1311       }
1312       break;
1313    }
1314    case nir_op_fmul: {
1315       if (dst.size() == 1) {
1316          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1317       } else if (dst.size() == 2) {
1318          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1319                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1320       } else {
1321          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1322          nir_print_instr(&instr->instr, stderr);
1323          fprintf(stderr, "\n");
1324       }
1325       break;
1326    }
1327    case nir_op_fadd: {
1328       if (dst.size() == 1) {
1329          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1330       } else if (dst.size() == 2) {
1331          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1332                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1333       } else {
1334          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1335          nir_print_instr(&instr->instr, stderr);
1336          fprintf(stderr, "\n");
1337       }
1338       break;
1339    }
1340    case nir_op_fsub: {
1341       Temp src0 = get_alu_src(ctx, instr->src[0]);
1342       Temp src1 = get_alu_src(ctx, instr->src[1]);
1343       if (dst.size() == 1) {
1344          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1345             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1346          else
1347             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1348       } else if (dst.size() == 2) {
1349          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1350                                      get_alu_src(ctx, instr->src[0]),
1351                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1352          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1353          sub->neg[1] = true;
1354       } else {
1355          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1356          nir_print_instr(&instr->instr, stderr);
1357          fprintf(stderr, "\n");
1358       }
1359       break;
1360    }
1361    case nir_op_fmax: {
1362       if (dst.size() == 1) {
1363          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1364       } else if (dst.size() == 2) {
1365          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1366             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1367                                 get_alu_src(ctx, instr->src[0]),
1368                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1369             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1370          } else {
1371             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1372                      get_alu_src(ctx, instr->src[0]),
1373                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1374          }
1375       } else {
1376          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1377          nir_print_instr(&instr->instr, stderr);
1378          fprintf(stderr, "\n");
1379       }
1380       break;
1381    }
1382    case nir_op_fmin: {
1383       if (dst.size() == 1) {
1384          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1385       } else if (dst.size() == 2) {
1386          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1387             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1388                                 get_alu_src(ctx, instr->src[0]),
1389                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1390             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1391          } else {
1392             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1393                      get_alu_src(ctx, instr->src[0]),
1394                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1395          }
1396       } else {
1397          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1398          nir_print_instr(&instr->instr, stderr);
1399          fprintf(stderr, "\n");
1400       }
1401       break;
1402    }
1403    case nir_op_fmax3: {
1404       if (dst.size() == 1) {
1405          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1406       } else {
1407          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1408          nir_print_instr(&instr->instr, stderr);
1409          fprintf(stderr, "\n");
1410       }
1411       break;
1412    }
1413    case nir_op_fmin3: {
1414       if (dst.size() == 1) {
1415          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1416       } else {
1417          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1418          nir_print_instr(&instr->instr, stderr);
1419          fprintf(stderr, "\n");
1420       }
1421       break;
1422    }
1423    case nir_op_fmed3: {
1424       if (dst.size() == 1) {
1425          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1426       } else {
1427          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1428          nir_print_instr(&instr->instr, stderr);
1429          fprintf(stderr, "\n");
1430       }
1431       break;
1432    }
1433    case nir_op_umax3: {
1434       if (dst.size() == 1) {
1435          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1436       } else {
1437          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1438          nir_print_instr(&instr->instr, stderr);
1439          fprintf(stderr, "\n");
1440       }
1441       break;
1442    }
1443    case nir_op_umin3: {
1444       if (dst.size() == 1) {
1445          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1446       } else {
1447          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1448          nir_print_instr(&instr->instr, stderr);
1449          fprintf(stderr, "\n");
1450       }
1451       break;
1452    }
1453    case nir_op_umed3: {
1454       if (dst.size() == 1) {
1455          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1456       } else {
1457          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1458          nir_print_instr(&instr->instr, stderr);
1459          fprintf(stderr, "\n");
1460       }
1461       break;
1462    }
1463    case nir_op_imax3: {
1464       if (dst.size() == 1) {
1465          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1466       } else {
1467          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1468          nir_print_instr(&instr->instr, stderr);
1469          fprintf(stderr, "\n");
1470       }
1471       break;
1472    }
1473    case nir_op_imin3: {
1474       if (dst.size() == 1) {
1475          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1476       } else {
1477          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1478          nir_print_instr(&instr->instr, stderr);
1479          fprintf(stderr, "\n");
1480       }
1481       break;
1482    }
1483    case nir_op_imed3: {
1484       if (dst.size() == 1) {
1485          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1486       } else {
1487          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1488          nir_print_instr(&instr->instr, stderr);
1489          fprintf(stderr, "\n");
1490       }
1491       break;
1492    }
1493    case nir_op_cube_face_coord: {
1494       Temp in = get_alu_src(ctx, instr->src[0], 3);
1495       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1496                       emit_extract_vector(ctx, in, 1, v1),
1497                       emit_extract_vector(ctx, in, 2, v1) };
1498       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1499       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1500       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1501       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1502       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1503       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1504       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1505       break;
1506    }
1507    case nir_op_cube_face_index: {
1508       Temp in = get_alu_src(ctx, instr->src[0], 3);
1509       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1510                       emit_extract_vector(ctx, in, 1, v1),
1511                       emit_extract_vector(ctx, in, 2, v1) };
1512       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1513       break;
1514    }
1515    case nir_op_bcsel: {
1516       emit_bcsel(ctx, instr, dst);
1517       break;
1518    }
1519    case nir_op_frsq: {
1520       if (dst.size() == 1) {
1521          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1522       } else if (dst.size() == 2) {
1523          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1524       } else {
1525          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1526          nir_print_instr(&instr->instr, stderr);
1527          fprintf(stderr, "\n");
1528       }
1529       break;
1530    }
1531    case nir_op_fneg: {
1532       Temp src = get_alu_src(ctx, instr->src[0]);
1533       if (dst.size() == 1) {
1534          if (ctx->block->fp_mode.must_flush_denorms32)
1535             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1536          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1537       } else if (dst.size() == 2) {
1538          if (ctx->block->fp_mode.must_flush_denorms16_64)
1539             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1540          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1541          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1542          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1543          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_fabs: {
1552       Temp src = get_alu_src(ctx, instr->src[0]);
1553       if (dst.size() == 1) {
1554          if (ctx->block->fp_mode.must_flush_denorms32)
1555             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1556          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1557       } else if (dst.size() == 2) {
1558          if (ctx->block->fp_mode.must_flush_denorms16_64)
1559             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1560          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1561          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1562          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1563          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1564       } else {
1565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1566          nir_print_instr(&instr->instr, stderr);
1567          fprintf(stderr, "\n");
1568       }
1569       break;
1570    }
1571    case nir_op_fsat: {
1572       Temp src = get_alu_src(ctx, instr->src[0]);
1573       if (dst.size() == 1) {
1574          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1575          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1576          // TODO: confirm that this holds under any circumstances
1577       } else if (dst.size() == 2) {
1578          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1579          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1580          vop3->clamp = true;
1581       } else {
1582          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1583          nir_print_instr(&instr->instr, stderr);
1584          fprintf(stderr, "\n");
1585       }
1586       break;
1587    }
1588    case nir_op_flog2: {
1589       if (dst.size() == 1) {
1590          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1591       } else {
1592          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1593          nir_print_instr(&instr->instr, stderr);
1594          fprintf(stderr, "\n");
1595       }
1596       break;
1597    }
1598    case nir_op_frcp: {
1599       if (dst.size() == 1) {
1600          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1601       } else if (dst.size() == 2) {
1602          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1603       } else {
1604          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1605          nir_print_instr(&instr->instr, stderr);
1606          fprintf(stderr, "\n");
1607       }
1608       break;
1609    }
1610    case nir_op_fexp2: {
1611       if (dst.size() == 1) {
1612          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1613       } else {
1614          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1615          nir_print_instr(&instr->instr, stderr);
1616          fprintf(stderr, "\n");
1617       }
1618       break;
1619    }
1620    case nir_op_fsqrt: {
1621       if (dst.size() == 1) {
1622          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1623       } else if (dst.size() == 2) {
1624          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_ffract: {
1633       if (dst.size() == 1) {
1634          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1635       } else if (dst.size() == 2) {
1636          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1637       } else {
1638          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1639          nir_print_instr(&instr->instr, stderr);
1640          fprintf(stderr, "\n");
1641       }
1642       break;
1643    }
1644    case nir_op_ffloor: {
1645       if (dst.size() == 1) {
1646          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1647       } else if (dst.size() == 2) {
1648          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1649       } else {
1650          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1651          nir_print_instr(&instr->instr, stderr);
1652          fprintf(stderr, "\n");
1653       }
1654       break;
1655    }
1656    case nir_op_fceil: {
1657       if (dst.size() == 1) {
1658          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1659       } else if (dst.size() == 2) {
1660          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1661       } else {
1662          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1663          nir_print_instr(&instr->instr, stderr);
1664          fprintf(stderr, "\n");
1665       }
1666       break;
1667    }
1668    case nir_op_ftrunc: {
1669       if (dst.size() == 1) {
1670          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1671       } else if (dst.size() == 2) {
1672          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1673       } else {
1674          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1675          nir_print_instr(&instr->instr, stderr);
1676          fprintf(stderr, "\n");
1677       }
1678       break;
1679    }
1680    case nir_op_fround_even: {
1681       if (dst.size() == 1) {
1682          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1683       } else if (dst.size() == 2) {
1684          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1685       } else {
1686          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1687          nir_print_instr(&instr->instr, stderr);
1688          fprintf(stderr, "\n");
1689       }
1690       break;
1691    }
1692    case nir_op_fsin:
1693    case nir_op_fcos: {
1694       Temp src = get_alu_src(ctx, instr->src[0]);
1695       aco_ptr<Instruction> norm;
1696       if (dst.size() == 1) {
1697          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1698          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1699
1700          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1701          if (ctx->options->chip_class < GFX9)
1702             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1703
1704          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1705          bld.vop1(opcode, Definition(dst), tmp);
1706       } else {
1707          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1708          nir_print_instr(&instr->instr, stderr);
1709          fprintf(stderr, "\n");
1710       }
1711       break;
1712    }
1713    case nir_op_ldexp: {
1714       if (dst.size() == 1) {
1715          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1716                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1717                   get_alu_src(ctx, instr->src[1]));
1718       } else if (dst.size() == 2) {
1719          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1720                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1721                   get_alu_src(ctx, instr->src[1]));
1722       } else {
1723          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1724          nir_print_instr(&instr->instr, stderr);
1725          fprintf(stderr, "\n");
1726       }
1727       break;
1728    }
1729    case nir_op_frexp_sig: {
1730       if (dst.size() == 1) {
1731          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1732                   get_alu_src(ctx, instr->src[0]));
1733       } else if (dst.size() == 2) {
1734          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1735                   get_alu_src(ctx, instr->src[0]));
1736       } else {
1737          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1738          nir_print_instr(&instr->instr, stderr);
1739          fprintf(stderr, "\n");
1740       }
1741       break;
1742    }
1743    case nir_op_frexp_exp: {
1744       if (instr->src[0].src.ssa->bit_size == 32) {
1745          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1746                   get_alu_src(ctx, instr->src[0]));
1747       } else if (instr->src[0].src.ssa->bit_size == 64) {
1748          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1749                   get_alu_src(ctx, instr->src[0]));
1750       } else {
1751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1752          nir_print_instr(&instr->instr, stderr);
1753          fprintf(stderr, "\n");
1754       }
1755       break;
1756    }
1757    case nir_op_fsign: {
1758       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1759       if (dst.size() == 1) {
1760          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1761          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1762          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1763          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1764       } else if (dst.size() == 2) {
1765          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1766          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1767          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1768
1769          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1770          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1771          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1772
1773          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1774       } else {
1775          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1776          nir_print_instr(&instr->instr, stderr);
1777          fprintf(stderr, "\n");
1778       }
1779       break;
1780    }
1781    case nir_op_f2f32: {
1782       if (instr->src[0].src.ssa->bit_size == 64) {
1783          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1784       } else {
1785          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1786          nir_print_instr(&instr->instr, stderr);
1787          fprintf(stderr, "\n");
1788       }
1789       break;
1790    }
1791    case nir_op_f2f64: {
1792       if (instr->src[0].src.ssa->bit_size == 32) {
1793          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1794       } else {
1795          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1796          nir_print_instr(&instr->instr, stderr);
1797          fprintf(stderr, "\n");
1798       }
1799       break;
1800    }
1801    case nir_op_i2f32: {
1802       assert(dst.size() == 1);
1803       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1804       break;
1805    }
1806    case nir_op_i2f64: {
1807       if (instr->src[0].src.ssa->bit_size == 32) {
1808          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1809       } else if (instr->src[0].src.ssa->bit_size == 64) {
1810          Temp src = get_alu_src(ctx, instr->src[0]);
1811          RegClass rc = RegClass(src.type(), 1);
1812          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1813          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1814          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1815          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1816          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1817          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1818
1819       } else {
1820          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1821          nir_print_instr(&instr->instr, stderr);
1822          fprintf(stderr, "\n");
1823       }
1824       break;
1825    }
1826    case nir_op_u2f32: {
1827       assert(dst.size() == 1);
1828       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1829       break;
1830    }
1831    case nir_op_u2f64: {
1832       if (instr->src[0].src.ssa->bit_size == 32) {
1833          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1834       } else if (instr->src[0].src.ssa->bit_size == 64) {
1835          Temp src = get_alu_src(ctx, instr->src[0]);
1836          RegClass rc = RegClass(src.type(), 1);
1837          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1838          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1839          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1840          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1841          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1842          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1843       } else {
1844          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1845          nir_print_instr(&instr->instr, stderr);
1846          fprintf(stderr, "\n");
1847       }
1848       break;
1849    }
1850    case nir_op_f2i32: {
1851       Temp src = get_alu_src(ctx, instr->src[0]);
1852       if (instr->src[0].src.ssa->bit_size == 32) {
1853          if (dst.type() == RegType::vgpr)
1854             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1855          else
1856             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1857                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1858
1859       } else if (instr->src[0].src.ssa->bit_size == 64) {
1860          if (dst.type() == RegType::vgpr)
1861             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1862          else
1863             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1864                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1865
1866       } else {
1867          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1868          nir_print_instr(&instr->instr, stderr);
1869          fprintf(stderr, "\n");
1870       }
1871       break;
1872    }
1873    case nir_op_f2u32: {
1874       Temp src = get_alu_src(ctx, instr->src[0]);
1875       if (instr->src[0].src.ssa->bit_size == 32) {
1876          if (dst.type() == RegType::vgpr)
1877             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1878          else
1879             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1880                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1881
1882       } else if (instr->src[0].src.ssa->bit_size == 64) {
1883          if (dst.type() == RegType::vgpr)
1884             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1885          else
1886             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1887                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1888
1889       } else {
1890          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1891          nir_print_instr(&instr->instr, stderr);
1892          fprintf(stderr, "\n");
1893       }
1894       break;
1895    }
1896    case nir_op_f2i64: {
1897       Temp src = get_alu_src(ctx, instr->src[0]);
1898       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1899          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1900          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1901          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1902          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1903          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1904          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1905          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1906          Temp new_exponent = bld.tmp(v1);
1907          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1908          if (ctx->program->chip_class >= GFX8)
1909             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1910          else
1911             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
1912          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1913          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1914          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1915          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1916          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1917          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1918          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1919          Temp new_lower = bld.tmp(v1);
1920          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1921          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1922          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1923
1924       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1925          if (src.type() == RegType::vgpr)
1926             src = bld.as_uniform(src);
1927          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1928          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1929          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1930          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1931          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1932          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1933          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1934          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1935          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1936          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1937          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1938          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1939          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1940          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1941          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1942          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1943          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1944          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1945          Temp borrow = bld.tmp(s1);
1946          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1947          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1948          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1949
1950       } else if (instr->src[0].src.ssa->bit_size == 64) {
1951          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1952          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1953          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1954          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1955          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1956          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1957          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1958          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1959          if (dst.type() == RegType::sgpr) {
1960             lower = bld.as_uniform(lower);
1961             upper = bld.as_uniform(upper);
1962          }
1963          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1964
1965       } else {
1966          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1967          nir_print_instr(&instr->instr, stderr);
1968          fprintf(stderr, "\n");
1969       }
1970       break;
1971    }
1972    case nir_op_f2u64: {
1973       Temp src = get_alu_src(ctx, instr->src[0]);
1974       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1975          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1976          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1977          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1978          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1979          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1980          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1981          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1982          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1983          Temp new_exponent = bld.tmp(v1);
1984          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1985          if (ctx->program->chip_class >= GFX8)
1986             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1987          else
1988             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
1989          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1990          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1991          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1992          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1993          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1994          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1995          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1996
1997       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1998          if (src.type() == RegType::vgpr)
1999             src = bld.as_uniform(src);
2000          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2001          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2002          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2003          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2004          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2005          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2006          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2007          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2008          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2009          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2010          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2011          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2012          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2013          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2014          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2015          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2016          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2017          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2018
2019       } else if (instr->src[0].src.ssa->bit_size == 64) {
2020          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2021          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
2022          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2023          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2024          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
2025          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2026          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2027          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2028          if (dst.type() == RegType::sgpr) {
2029             lower = bld.as_uniform(lower);
2030             upper = bld.as_uniform(upper);
2031          }
2032          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2033
2034       } else {
2035          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2036          nir_print_instr(&instr->instr, stderr);
2037          fprintf(stderr, "\n");
2038       }
2039       break;
2040    }
2041    case nir_op_b2f32: {
2042       Temp src = get_alu_src(ctx, instr->src[0]);
2043       assert(src.regClass() == bld.lm);
2044
2045       if (dst.regClass() == s1) {
2046          src = bool_to_scalar_condition(ctx, src);
2047          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2048       } else if (dst.regClass() == v1) {
2049          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2050       } else {
2051          unreachable("Wrong destination register class for nir_op_b2f32.");
2052       }
2053       break;
2054    }
2055    case nir_op_b2f64: {
2056       Temp src = get_alu_src(ctx, instr->src[0]);
2057       assert(src.regClass() == bld.lm);
2058
2059       if (dst.regClass() == s2) {
2060          src = bool_to_scalar_condition(ctx, src);
2061          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2062       } else if (dst.regClass() == v2) {
2063          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2064          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2065          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2066       } else {
2067          unreachable("Wrong destination register class for nir_op_b2f64.");
2068       }
2069       break;
2070    }
2071    case nir_op_i2i32: {
2072       Temp src = get_alu_src(ctx, instr->src[0]);
2073       if (instr->src[0].src.ssa->bit_size == 64) {
2074          /* we can actually just say dst = src, as it would map the lower register */
2075          emit_extract_vector(ctx, src, 0, dst);
2076       } else {
2077          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2078          nir_print_instr(&instr->instr, stderr);
2079          fprintf(stderr, "\n");
2080       }
2081       break;
2082    }
2083    case nir_op_u2u32: {
2084       Temp src = get_alu_src(ctx, instr->src[0]);
2085       if (instr->src[0].src.ssa->bit_size == 16) {
2086          if (dst.regClass() == s1) {
2087             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2088          } else {
2089             // TODO: do better with SDWA
2090             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2091          }
2092       } else if (instr->src[0].src.ssa->bit_size == 64) {
2093          /* we can actually just say dst = src, as it would map the lower register */
2094          emit_extract_vector(ctx, src, 0, dst);
2095       } else {
2096          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2097          nir_print_instr(&instr->instr, stderr);
2098          fprintf(stderr, "\n");
2099       }
2100       break;
2101    }
2102    case nir_op_i2i64: {
2103       Temp src = get_alu_src(ctx, instr->src[0]);
2104       if (src.regClass() == s1) {
2105          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2106          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2107       } else if (src.regClass() == v1) {
2108          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2109          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2110       } else {
2111          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2112          nir_print_instr(&instr->instr, stderr);
2113          fprintf(stderr, "\n");
2114       }
2115       break;
2116    }
2117    case nir_op_u2u64: {
2118       Temp src = get_alu_src(ctx, instr->src[0]);
2119       if (instr->src[0].src.ssa->bit_size == 32) {
2120          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2121       } else {
2122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2123          nir_print_instr(&instr->instr, stderr);
2124          fprintf(stderr, "\n");
2125       }
2126       break;
2127    }
2128    case nir_op_b2i32: {
2129       Temp src = get_alu_src(ctx, instr->src[0]);
2130       assert(src.regClass() == bld.lm);
2131
2132       if (dst.regClass() == s1) {
2133          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2134          bool_to_scalar_condition(ctx, src, dst);
2135       } else if (dst.regClass() == v1) {
2136          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2137       } else {
2138          unreachable("Invalid register class for b2i32");
2139       }
2140       break;
2141    }
2142    case nir_op_i2b1: {
2143       Temp src = get_alu_src(ctx, instr->src[0]);
2144       assert(dst.regClass() == bld.lm);
2145
2146       if (src.type() == RegType::vgpr) {
2147          assert(src.regClass() == v1 || src.regClass() == v2);
2148          assert(dst.regClass() == bld.lm);
2149          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2150                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2151       } else {
2152          assert(src.regClass() == s1 || src.regClass() == s2);
2153          Temp tmp;
2154          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2155             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2156          } else {
2157             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2158                            bld.scc(bld.def(s1)), Operand(0u), src);
2159          }
2160          bool_to_vector_condition(ctx, tmp, dst);
2161       }
2162       break;
2163    }
2164    case nir_op_pack_64_2x32_split: {
2165       Temp src0 = get_alu_src(ctx, instr->src[0]);
2166       Temp src1 = get_alu_src(ctx, instr->src[1]);
2167
2168       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2169       break;
2170    }
2171    case nir_op_unpack_64_2x32_split_x:
2172       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2173       break;
2174    case nir_op_unpack_64_2x32_split_y:
2175       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2176       break;
2177    case nir_op_pack_half_2x16: {
2178       Temp src = get_alu_src(ctx, instr->src[0], 2);
2179
2180       if (dst.regClass() == v1) {
2181          Temp src0 = bld.tmp(v1);
2182          Temp src1 = bld.tmp(v1);
2183          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2184          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2185             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2186          else
2187             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2188                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2189                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2190       } else {
2191          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2192          nir_print_instr(&instr->instr, stderr);
2193          fprintf(stderr, "\n");
2194       }
2195       break;
2196    }
2197    case nir_op_unpack_half_2x16_split_x: {
2198       if (dst.regClass() == v1) {
2199          Builder bld(ctx->program, ctx->block);
2200          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2201       } else {
2202          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2203          nir_print_instr(&instr->instr, stderr);
2204          fprintf(stderr, "\n");
2205       }
2206       break;
2207    }
2208    case nir_op_unpack_half_2x16_split_y: {
2209       if (dst.regClass() == v1) {
2210          Builder bld(ctx->program, ctx->block);
2211          /* TODO: use SDWA here */
2212          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2213                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2214       } else {
2215          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2216          nir_print_instr(&instr->instr, stderr);
2217          fprintf(stderr, "\n");
2218       }
2219       break;
2220    }
2221    case nir_op_fquantize2f16: {
2222       Temp src = get_alu_src(ctx, instr->src[0]);
2223       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2224       Temp f32, cmp_res;
2225
2226       if (ctx->program->chip_class >= GFX8) {
2227          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2228          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2229          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2230       } else {
2231          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2232           * so compare the result and flush to 0 if it's smaller.
2233           */
2234          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2235          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2236          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2237          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2238          cmp_res = vop3->definitions[0].getTemp();
2239       }
2240
2241       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2242          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2243          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2244       } else {
2245          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2246       }
2247       break;
2248    }
2249    case nir_op_bfm: {
2250       Temp bits = get_alu_src(ctx, instr->src[0]);
2251       Temp offset = get_alu_src(ctx, instr->src[1]);
2252
2253       if (dst.regClass() == s1) {
2254          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2255       } else if (dst.regClass() == v1) {
2256          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2257       } else {
2258          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2259          nir_print_instr(&instr->instr, stderr);
2260          fprintf(stderr, "\n");
2261       }
2262       break;
2263    }
2264    case nir_op_bitfield_select: {
2265       /* (mask & insert) | (~mask & base) */
2266       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2267       Temp insert = get_alu_src(ctx, instr->src[1]);
2268       Temp base = get_alu_src(ctx, instr->src[2]);
2269
2270       /* dst = (insert & bitmask) | (base & ~bitmask) */
2271       if (dst.regClass() == s1) {
2272          aco_ptr<Instruction> sop2;
2273          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2274          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2275          Operand lhs;
2276          if (const_insert && const_bitmask) {
2277             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2278          } else {
2279             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2280             lhs = Operand(insert);
2281          }
2282
2283          Operand rhs;
2284          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2285          if (const_base && const_bitmask) {
2286             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2287          } else {
2288             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2289             rhs = Operand(base);
2290          }
2291
2292          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2293
2294       } else if (dst.regClass() == v1) {
2295          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2296             base = as_vgpr(ctx, base);
2297          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2298             insert = as_vgpr(ctx, insert);
2299
2300          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2301
2302       } else {
2303          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2304          nir_print_instr(&instr->instr, stderr);
2305          fprintf(stderr, "\n");
2306       }
2307       break;
2308    }
2309    case nir_op_ubfe:
2310    case nir_op_ibfe: {
2311       Temp base = get_alu_src(ctx, instr->src[0]);
2312       Temp offset = get_alu_src(ctx, instr->src[1]);
2313       Temp bits = get_alu_src(ctx, instr->src[2]);
2314
2315       if (dst.type() == RegType::sgpr) {
2316          Operand extract;
2317          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2318          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2319          if (const_offset && const_bits) {
2320             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2321             extract = Operand(const_extract);
2322          } else {
2323             Operand width;
2324             if (const_bits) {
2325                width = Operand(const_bits->u32 << 16);
2326             } else {
2327                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2328             }
2329             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2330          }
2331
2332          aco_opcode opcode;
2333          if (dst.regClass() == s1) {
2334             if (instr->op == nir_op_ubfe)
2335                opcode = aco_opcode::s_bfe_u32;
2336             else
2337                opcode = aco_opcode::s_bfe_i32;
2338          } else if (dst.regClass() == s2) {
2339             if (instr->op == nir_op_ubfe)
2340                opcode = aco_opcode::s_bfe_u64;
2341             else
2342                opcode = aco_opcode::s_bfe_i64;
2343          } else {
2344             unreachable("Unsupported BFE bit size");
2345          }
2346
2347          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2348
2349       } else {
2350          aco_opcode opcode;
2351          if (dst.regClass() == v1) {
2352             if (instr->op == nir_op_ubfe)
2353                opcode = aco_opcode::v_bfe_u32;
2354             else
2355                opcode = aco_opcode::v_bfe_i32;
2356          } else {
2357             unreachable("Unsupported BFE bit size");
2358          }
2359
2360          emit_vop3a_instruction(ctx, instr, opcode, dst);
2361       }
2362       break;
2363    }
2364    case nir_op_bit_count: {
2365       Temp src = get_alu_src(ctx, instr->src[0]);
2366       if (src.regClass() == s1) {
2367          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2368       } else if (src.regClass() == v1) {
2369          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2370       } else if (src.regClass() == v2) {
2371          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2372                   emit_extract_vector(ctx, src, 1, v1),
2373                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2374                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2375       } else if (src.regClass() == s2) {
2376          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2377       } else {
2378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2379          nir_print_instr(&instr->instr, stderr);
2380          fprintf(stderr, "\n");
2381       }
2382       break;
2383    }
2384    case nir_op_flt: {
2385       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2386       break;
2387    }
2388    case nir_op_fge: {
2389       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2390       break;
2391    }
2392    case nir_op_feq: {
2393       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2394       break;
2395    }
2396    case nir_op_fne: {
2397       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2398       break;
2399    }
2400    case nir_op_ilt: {
2401       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2402       break;
2403    }
2404    case nir_op_ige: {
2405       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2406       break;
2407    }
2408    case nir_op_ieq: {
2409       if (instr->src[0].src.ssa->bit_size == 1)
2410          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2411       else
2412          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2413                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2414       break;
2415    }
2416    case nir_op_ine: {
2417       if (instr->src[0].src.ssa->bit_size == 1)
2418          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2419       else
2420          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2421                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2422       break;
2423    }
2424    case nir_op_ult: {
2425       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2426       break;
2427    }
2428    case nir_op_uge: {
2429       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2430       break;
2431    }
2432    case nir_op_fddx:
2433    case nir_op_fddy:
2434    case nir_op_fddx_fine:
2435    case nir_op_fddy_fine:
2436    case nir_op_fddx_coarse:
2437    case nir_op_fddy_coarse: {
2438       Temp src = get_alu_src(ctx, instr->src[0]);
2439       uint16_t dpp_ctrl1, dpp_ctrl2;
2440       if (instr->op == nir_op_fddx_fine) {
2441          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2442          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2443       } else if (instr->op == nir_op_fddy_fine) {
2444          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2445          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2446       } else {
2447          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2448          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2449             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2450          else
2451             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2452       }
2453
2454       Temp tmp;
2455       if (ctx->program->chip_class >= GFX8) {
2456          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2457          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2458       } else {
2459          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2460          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2461          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2462       }
2463       emit_wqm(ctx, tmp, dst, true);
2464       break;
2465    }
2466    default:
2467       fprintf(stderr, "Unknown NIR ALU instr: ");
2468       nir_print_instr(&instr->instr, stderr);
2469       fprintf(stderr, "\n");
2470    }
2471 }
2472
2473 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2474 {
2475    Temp dst = get_ssa_temp(ctx, &instr->def);
2476
2477    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2478    // which get truncated the lsb if double and msb if int
2479    // for now, we only use s_mov_b64 with 64bit inline constants
2480    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2481    assert(dst.type() == RegType::sgpr);
2482
2483    Builder bld(ctx->program, ctx->block);
2484
2485    if (instr->def.bit_size == 1) {
2486       assert(dst.regClass() == bld.lm);
2487       int val = instr->value[0].b ? -1 : 0;
2488       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2489       bld.sop1(Builder::s_mov, Definition(dst), op);
2490    } else if (dst.size() == 1) {
2491       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2492    } else {
2493       assert(dst.size() != 1);
2494       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2495       if (instr->def.bit_size == 64)
2496          for (unsigned i = 0; i < dst.size(); i++)
2497             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2498       else {
2499          for (unsigned i = 0; i < dst.size(); i++)
2500             vec->operands[i] = Operand{instr->value[i].u32};
2501       }
2502       vec->definitions[0] = Definition(dst);
2503       ctx->block->instructions.emplace_back(std::move(vec));
2504    }
2505 }
2506
2507 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2508 {
2509    uint32_t new_mask = 0;
2510    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2511       if (mask & (1u << i))
2512          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2513    return new_mask;
2514 }
2515
2516 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2517 {
2518    /* This wouldn't work inside control flow or with indirect offsets but
2519     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2520
2521    unsigned write_mask = nir_intrinsic_write_mask(instr);
2522    unsigned component = nir_intrinsic_component(instr);
2523    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2524    unsigned idx = nir_intrinsic_base(instr) + component;
2525
2526    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2527    if (off_instr->type != nir_instr_type_load_const) {
2528       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2529       nir_print_instr(off_instr, stderr);
2530       fprintf(stderr, "\n");
2531    }
2532    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2533
2534    if (instr->src[0].ssa->bit_size == 64)
2535       write_mask = widen_mask(write_mask, 2);
2536
2537    for (unsigned i = 0; i < 8; ++i) {
2538       if (write_mask & (1 << i)) {
2539          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2540          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2541       }
2542       idx++;
2543    }
2544 }
2545
2546 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2547 {
2548    Builder bld(ctx->program, ctx->block);
2549    unsigned write_mask = nir_intrinsic_write_mask(instr);
2550    Operand values[4];
2551    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2552    for (unsigned i = 0; i < 4; ++i) {
2553       if (write_mask & (1 << i)) {
2554          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2555          values[i] = Operand(tmp);
2556       } else {
2557          values[i] = Operand(v1);
2558       }
2559    }
2560
2561    unsigned index = nir_intrinsic_base(instr) / 4;
2562    unsigned target, col_format;
2563    unsigned enabled_channels = 0xF;
2564    aco_opcode compr_op = (aco_opcode)0;
2565
2566    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2567    assert(offset && "Non-const offsets on exports not yet supported");
2568    index += offset->u32;
2569
2570    assert(index != FRAG_RESULT_COLOR);
2571
2572    /* Unlike vertex shader exports, it's fine to use multiple exports to
2573     * export separate channels of one target. So shaders which export both
2574     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2575     * TODO: combine the exports in those cases and create better code
2576     */
2577
2578    if (index == FRAG_RESULT_SAMPLE_MASK) {
2579
2580       if (ctx->program->info->ps.writes_z) {
2581          target = V_008DFC_SQ_EXP_MRTZ;
2582          enabled_channels = 0x4;
2583          col_format = (unsigned) -1;
2584
2585          values[2] = values[0];
2586          values[0] = Operand(v1);
2587       } else {
2588          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2589                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2590          return;
2591       }
2592
2593    } else if (index == FRAG_RESULT_DEPTH) {
2594
2595       target = V_008DFC_SQ_EXP_MRTZ;
2596       enabled_channels = 0x1;
2597       col_format = (unsigned) -1;
2598
2599    } else if (index == FRAG_RESULT_STENCIL) {
2600
2601       if (ctx->program->info->ps.writes_z) {
2602          target = V_008DFC_SQ_EXP_MRTZ;
2603          enabled_channels = 0x2;
2604          col_format = (unsigned) -1;
2605
2606          values[1] = values[0];
2607          values[0] = Operand(v1);
2608       } else {
2609          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2610          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2611                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2612          return;
2613       }
2614
2615    } else {
2616       index -= FRAG_RESULT_DATA0;
2617       target = V_008DFC_SQ_EXP_MRT + index;
2618       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2619    }
2620    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2621    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2622
2623    switch (col_format)
2624    {
2625    case V_028714_SPI_SHADER_ZERO:
2626       enabled_channels = 0; /* writemask */
2627       target = V_008DFC_SQ_EXP_NULL;
2628       break;
2629
2630    case V_028714_SPI_SHADER_32_R:
2631       enabled_channels = 1;
2632       break;
2633
2634    case V_028714_SPI_SHADER_32_GR:
2635       enabled_channels = 0x3;
2636       break;
2637
2638    case V_028714_SPI_SHADER_32_AR:
2639       if (ctx->options->chip_class >= GFX10) {
2640          /* Special case: on GFX10, the outputs are different for 32_AR */
2641          enabled_channels = 0x3;
2642          values[1] = values[3];
2643          values[3] = Operand(v1);
2644       } else {
2645          enabled_channels = 0x9;
2646       }
2647       break;
2648
2649    case V_028714_SPI_SHADER_FP16_ABGR:
2650       enabled_channels = 0x5;
2651       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2652       break;
2653
2654    case V_028714_SPI_SHADER_UNORM16_ABGR:
2655       enabled_channels = 0x5;
2656       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2657       break;
2658
2659    case V_028714_SPI_SHADER_SNORM16_ABGR:
2660       enabled_channels = 0x5;
2661       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2662       break;
2663
2664    case V_028714_SPI_SHADER_UINT16_ABGR: {
2665       enabled_channels = 0x5;
2666       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2667       if (is_int8 || is_int10) {
2668          /* clamp */
2669          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2670          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2671
2672          for (unsigned i = 0; i < 4; i++) {
2673             if ((write_mask >> i) & 1) {
2674                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2675                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2676                                     values[i]);
2677             }
2678          }
2679       }
2680       break;
2681    }
2682
2683    case V_028714_SPI_SHADER_SINT16_ABGR:
2684       enabled_channels = 0x5;
2685       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2686       if (is_int8 || is_int10) {
2687          /* clamp */
2688          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2689          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2690          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2691          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2692
2693          for (unsigned i = 0; i < 4; i++) {
2694             if ((write_mask >> i) & 1) {
2695                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2696                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2697                                     values[i]);
2698                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2699                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2700                                     values[i]);
2701             }
2702          }
2703       }
2704       break;
2705
2706    case V_028714_SPI_SHADER_32_ABGR:
2707       enabled_channels = 0xF;
2708       break;
2709
2710    default:
2711       break;
2712    }
2713
2714    if (target == V_008DFC_SQ_EXP_NULL)
2715       return;
2716
2717    if ((bool) compr_op) {
2718       for (int i = 0; i < 2; i++) {
2719          /* check if at least one of the values to be compressed is enabled */
2720          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2721          if (enabled) {
2722             enabled_channels |= enabled << (i*2);
2723             values[i] = bld.vop3(compr_op, bld.def(v1),
2724                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2725                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2726          } else {
2727             values[i] = Operand(v1);
2728          }
2729       }
2730       values[2] = Operand(v1);
2731       values[3] = Operand(v1);
2732    } else {
2733       for (int i = 0; i < 4; i++)
2734          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2735    }
2736
2737    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2738            enabled_channels, target, (bool) compr_op);
2739 }
2740
2741 Operand load_lds_size_m0(isel_context *ctx)
2742 {
2743    /* TODO: m0 does not need to be initialized on GFX9+ */
2744    Builder bld(ctx->program, ctx->block);
2745    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2746 }
2747
2748 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2749               Temp address, unsigned base_offset, unsigned align)
2750 {
2751    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2752
2753    Builder bld(ctx->program, ctx->block);
2754
2755    Operand m = load_lds_size_m0(ctx);
2756
2757    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2758    unsigned bytes_read = 0;
2759    unsigned result_size = 0;
2760    unsigned total_bytes = num_components * elem_size_bytes;
2761    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2762    bool large_ds_read = ctx->options->chip_class >= GFX7;
2763
2764    while (bytes_read < total_bytes) {
2765       unsigned todo = total_bytes - bytes_read;
2766       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2767       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2768
2769       aco_opcode op = aco_opcode::last_opcode;
2770       bool read2 = false;
2771       if (todo >= 16 && aligned16 && large_ds_read) {
2772          op = aco_opcode::ds_read_b128;
2773          todo = 16;
2774       } else if (todo >= 16 && aligned8) {
2775          op = aco_opcode::ds_read2_b64;
2776          read2 = true;
2777          todo = 16;
2778       } else if (todo >= 12 && aligned16 && large_ds_read) {
2779          op = aco_opcode::ds_read_b96;
2780          todo = 12;
2781       } else if (todo >= 8 && aligned8) {
2782          op = aco_opcode::ds_read_b64;
2783          todo = 8;
2784       } else if (todo >= 8) {
2785          op = aco_opcode::ds_read2_b32;
2786          read2 = true;
2787          todo = 8;
2788       } else if (todo >= 4) {
2789          op = aco_opcode::ds_read_b32;
2790          todo = 4;
2791       } else {
2792          assert(false);
2793       }
2794       assert(todo % elem_size_bytes == 0);
2795       unsigned num_elements = todo / elem_size_bytes;
2796       unsigned offset = base_offset + bytes_read;
2797       unsigned max_offset = read2 ? 1019 : 65535;
2798
2799       Temp address_offset = address;
2800       if (offset > max_offset) {
2801          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2802          offset = bytes_read;
2803       }
2804       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2805
2806       Temp res;
2807       if (num_components == 1 && dst.type() == RegType::vgpr)
2808          res = dst;
2809       else
2810          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2811
2812       if (read2)
2813          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2814       else
2815          res = bld.ds(op, Definition(res), address_offset, m, offset);
2816
2817       if (num_components == 1) {
2818          assert(todo == total_bytes);
2819          if (dst.type() == RegType::sgpr)
2820             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2821          return;
2822       }
2823
2824       if (dst.type() == RegType::sgpr) {
2825          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2826          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2827          res = new_res;
2828       }
2829
2830       if (num_elements == 1) {
2831          result[result_size++] = res;
2832       } else {
2833          assert(res != dst && res.size() % num_elements == 0);
2834          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2835          split->operands[0] = Operand(res);
2836          for (unsigned i = 0; i < num_elements; i++)
2837             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2838          ctx->block->instructions.emplace_back(std::move(split));
2839       }
2840
2841       bytes_read += todo;
2842    }
2843
2844    assert(result_size == num_components && result_size > 1);
2845    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2846    for (unsigned i = 0; i < result_size; i++)
2847       vec->operands[i] = Operand(result[i]);
2848    vec->definitions[0] = Definition(dst);
2849    ctx->block->instructions.emplace_back(std::move(vec));
2850    ctx->allocated_vec.emplace(dst.id(), result);
2851 }
2852
2853 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2854 {
2855    if (start == 0 && size == data.size())
2856       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2857
2858    unsigned size_hint = 1;
2859    auto it = ctx->allocated_vec.find(data.id());
2860    if (it != ctx->allocated_vec.end())
2861       size_hint = it->second[0].size();
2862    if (size % size_hint || start % size_hint)
2863       size_hint = 1;
2864
2865    start /= size_hint;
2866    size /= size_hint;
2867
2868    Temp elems[size];
2869    for (unsigned i = 0; i < size; i++)
2870       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2871
2872    if (size == 1)
2873       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2874
2875    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2876    for (unsigned i = 0; i < size; i++)
2877       vec->operands[i] = Operand(elems[i]);
2878    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2879    vec->definitions[0] = Definition(res);
2880    ctx->block->instructions.emplace_back(std::move(vec));
2881    return res;
2882 }
2883
2884 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2885 {
2886    Builder bld(ctx->program, ctx->block);
2887    unsigned bytes_written = 0;
2888    bool large_ds_write = ctx->options->chip_class >= GFX7;
2889
2890    while (bytes_written < total_size * 4) {
2891       unsigned todo = total_size * 4 - bytes_written;
2892       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2893       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2894
2895       aco_opcode op = aco_opcode::last_opcode;
2896       bool write2 = false;
2897       unsigned size = 0;
2898       if (todo >= 16 && aligned16 && large_ds_write) {
2899          op = aco_opcode::ds_write_b128;
2900          size = 4;
2901       } else if (todo >= 16 && aligned8) {
2902          op = aco_opcode::ds_write2_b64;
2903          write2 = true;
2904          size = 4;
2905       } else if (todo >= 12 && aligned16 && large_ds_write) {
2906          op = aco_opcode::ds_write_b96;
2907          size = 3;
2908       } else if (todo >= 8 && aligned8) {
2909          op = aco_opcode::ds_write_b64;
2910          size = 2;
2911       } else if (todo >= 8) {
2912          op = aco_opcode::ds_write2_b32;
2913          write2 = true;
2914          size = 2;
2915       } else if (todo >= 4) {
2916          op = aco_opcode::ds_write_b32;
2917          size = 1;
2918       } else {
2919          assert(false);
2920       }
2921
2922       unsigned offset = offset0 + offset1 + bytes_written;
2923       unsigned max_offset = write2 ? 1020 : 65535;
2924       Temp address_offset = address;
2925       if (offset > max_offset) {
2926          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2927          offset = offset1 + bytes_written;
2928       }
2929       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2930
2931       if (write2) {
2932          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2933          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2934          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2935       } else {
2936          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2937          bld.ds(op, address_offset, val, m, offset);
2938       }
2939
2940       bytes_written += size * 4;
2941    }
2942 }
2943
2944 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2945                Temp address, unsigned base_offset, unsigned align)
2946 {
2947    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2948
2949    Operand m = load_lds_size_m0(ctx);
2950
2951    /* we need at most two stores for 32bit variables */
2952    int start[2], count[2];
2953    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2954    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2955    assert(wrmask == 0);
2956
2957    /* one combined store is sufficient */
2958    if (count[0] == count[1]) {
2959       Builder bld(ctx->program, ctx->block);
2960
2961       Temp address_offset = address;
2962       if ((base_offset >> 2) + start[1] > 255) {
2963          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2964          base_offset = 0;
2965       }
2966
2967       assert(count[0] == 1);
2968       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2969       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2970       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2971       base_offset = base_offset / elem_size_bytes;
2972       bld.ds(op, address_offset, val0, val1, m,
2973              base_offset + start[0], base_offset + start[1]);
2974       return;
2975    }
2976
2977    for (unsigned i = 0; i < 2; i++) {
2978       if (count[i] == 0)
2979          continue;
2980
2981       unsigned elem_size_words = elem_size_bytes / 4;
2982       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2983                       base_offset, start[i] * elem_size_bytes, align);
2984    }
2985    return;
2986 }
2987
2988 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2989 {
2990    if (ctx->stage == vertex_vs) {
2991       visit_store_vs_output(ctx, instr);
2992    } else if (ctx->stage == fragment_fs) {
2993       visit_store_fs_output(ctx, instr);
2994    } else {
2995       unreachable("Shader stage not implemented");
2996    }
2997 }
2998
2999 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3000 {
3001    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3002    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3003
3004    Builder bld(ctx->program, ctx->block);
3005    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3006    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3007 }
3008
3009 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3010 {
3011    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3012    for (unsigned i = 0; i < num_components; i++)
3013       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3014    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3015       assert(num_components == 4);
3016       Builder bld(ctx->program, ctx->block);
3017       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3018    }
3019
3020    for (Operand& op : vec->operands)
3021       op = op.isUndefined() ? Operand(0u) : op;
3022
3023    vec->definitions[0] = Definition(dst);
3024    ctx->block->instructions.emplace_back(std::move(vec));
3025    emit_split_vector(ctx, dst, num_components);
3026    return;
3027 }
3028
3029 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3030 {
3031    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3032    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3033    unsigned idx = nir_intrinsic_base(instr);
3034    unsigned component = nir_intrinsic_component(instr);
3035    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3036
3037    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3038    if (offset) {
3039       assert(offset->u32 == 0);
3040    } else {
3041       /* the lower 15bit of the prim_mask contain the offset into LDS
3042        * while the upper bits contain the number of prims */
3043       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3044       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3045       Builder bld(ctx->program, ctx->block);
3046       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3047       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3048       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3049       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3050       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3051    }
3052
3053    if (instr->dest.ssa.num_components == 1) {
3054       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3055    } else {
3056       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3057       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3058       {
3059          Temp tmp = {ctx->program->allocateId(), v1};
3060          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3061          vec->operands[i] = Operand(tmp);
3062       }
3063       vec->definitions[0] = Definition(dst);
3064       ctx->block->instructions.emplace_back(std::move(vec));
3065    }
3066 }
3067
3068 unsigned get_num_channels_from_data_format(unsigned data_format)
3069 {
3070    switch (data_format) {
3071    case V_008F0C_BUF_DATA_FORMAT_8:
3072    case V_008F0C_BUF_DATA_FORMAT_16:
3073    case V_008F0C_BUF_DATA_FORMAT_32:
3074       return 1;
3075    case V_008F0C_BUF_DATA_FORMAT_8_8:
3076    case V_008F0C_BUF_DATA_FORMAT_16_16:
3077    case V_008F0C_BUF_DATA_FORMAT_32_32:
3078       return 2;
3079    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3080    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3081    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3082       return 3;
3083    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3084    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3085    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3086    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3087    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3088       return 4;
3089    default:
3090       break;
3091    }
3092
3093    return 4;
3094 }
3095
3096 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3097  * so we may need to fix it up. */
3098 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3099 {
3100    Builder bld(ctx->program, ctx->block);
3101
3102    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3103       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3104
3105    /* For the integer-like cases, do a natural sign extension.
3106     *
3107     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3108     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3109     * exponent.
3110     */
3111    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3112    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3113
3114    /* Convert back to the right type. */
3115    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3116       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3117       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3118       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3119    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3120       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3121    }
3122
3123    return alpha;
3124 }
3125
3126 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3127 {
3128    Builder bld(ctx->program, ctx->block);
3129    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3130    if (ctx->stage & sw_vs) {
3131
3132       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3133       if (off_instr->type != nir_instr_type_load_const) {
3134          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3135          nir_print_instr(off_instr, stderr);
3136          fprintf(stderr, "\n");
3137       }
3138       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3139
3140       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3141
3142       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3143       unsigned component = nir_intrinsic_component(instr);
3144       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3145       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3146       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3147       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3148
3149       unsigned dfmt = attrib_format & 0xf;
3150
3151       unsigned nfmt = (attrib_format >> 4) & 0x7;
3152       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3153       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3154       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3155       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3156       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3157       if (post_shuffle)
3158          num_channels = MAX2(num_channels, 3);
3159
3160       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3161
3162       Temp index;
3163       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3164          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3165          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3166          if (divisor) {
3167             ctx->needs_instance_id = true;
3168             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3169             if (divisor != 1) {
3170                Temp divided = bld.tmp(v1);
3171                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3172                index = bld.vadd32(bld.def(v1), start_instance, divided);
3173             } else {
3174                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3175             }
3176          } else {
3177             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3178          }
3179       } else {
3180          index = bld.vadd32(bld.def(v1),
3181                             get_arg(ctx, ctx->args->ac.base_vertex),
3182                             get_arg(ctx, ctx->args->ac.vertex_id));
3183       }
3184
3185       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3186          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3187          attrib_offset = attrib_offset % attrib_stride;
3188       }
3189
3190       Operand soffset(0u);
3191       if (attrib_offset >= 4096) {
3192          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3193          attrib_offset = 0;
3194       }
3195
3196       aco_opcode opcode;
3197       switch (num_channels) {
3198       case 1:
3199          opcode = aco_opcode::tbuffer_load_format_x;
3200          break;
3201       case 2:
3202          opcode = aco_opcode::tbuffer_load_format_xy;
3203          break;
3204       case 3:
3205          opcode = aco_opcode::tbuffer_load_format_xyz;
3206          break;
3207       case 4:
3208          opcode = aco_opcode::tbuffer_load_format_xyzw;
3209          break;
3210       default:
3211          unreachable("Unimplemented load_input vector size");
3212       }
3213
3214       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3215
3216       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3217       mubuf->operands[0] = Operand(index);
3218       mubuf->operands[1] = Operand(list);
3219       mubuf->operands[2] = soffset;
3220       mubuf->definitions[0] = Definition(tmp);
3221       mubuf->idxen = true;
3222       mubuf->can_reorder = true;
3223       mubuf->dfmt = dfmt;
3224       mubuf->nfmt = nfmt;
3225       assert(attrib_offset < 4096);
3226       mubuf->offset = attrib_offset;
3227       ctx->block->instructions.emplace_back(std::move(mubuf));
3228
3229       emit_split_vector(ctx, tmp, tmp.size());
3230
3231       if (tmp.id() != dst.id()) {
3232          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3233                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3234
3235          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3236          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3237          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3238
3239          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3240          for (unsigned i = 0; i < dst.size(); i++) {
3241             unsigned idx = i + component;
3242             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3243                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3244                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3245             } else if (idx < num_channels) {
3246                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3247             } else if (is_float && idx == 3) {
3248                vec->operands[i] = Operand(0x3f800000u);
3249             } else if (!is_float && idx == 3) {
3250                vec->operands[i] = Operand(1u);
3251             } else {
3252                vec->operands[i] = Operand(0u);
3253             }
3254          }
3255          vec->definitions[0] = Definition(dst);
3256          ctx->block->instructions.emplace_back(std::move(vec));
3257          emit_split_vector(ctx, dst, dst.size());
3258       }
3259
3260    } else if (ctx->stage == fragment_fs) {
3261       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3262       if (off_instr->type != nir_instr_type_load_const ||
3263           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3264          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3265          nir_print_instr(off_instr, stderr);
3266          fprintf(stderr, "\n");
3267       }
3268
3269       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3270       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3271       if (offset) {
3272          assert(offset->u32 == 0);
3273       } else {
3274          /* the lower 15bit of the prim_mask contain the offset into LDS
3275           * while the upper bits contain the number of prims */
3276          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3277          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3278          Builder bld(ctx->program, ctx->block);
3279          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3280          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3281          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3282          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3283          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3284       }
3285
3286       unsigned idx = nir_intrinsic_base(instr);
3287       unsigned component = nir_intrinsic_component(instr);
3288
3289       if (dst.size() == 1) {
3290          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3291       } else {
3292          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3293          for (unsigned i = 0; i < dst.size(); i++)
3294             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3295          vec->definitions[0] = Definition(dst);
3296          bld.insert(std::move(vec));
3297       }
3298
3299    } else {
3300       unreachable("Shader stage not implemented");
3301    }
3302 }
3303
3304 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3305 {
3306    if (ctx->program->info->need_indirect_descriptor_sets) {
3307       Builder bld(ctx->program, ctx->block);
3308       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3309       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3310    }
3311
3312    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3313 }
3314
3315
3316 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3317 {
3318    Builder bld(ctx->program, ctx->block);
3319    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3320    if (!ctx->divergent_vals[instr->dest.ssa.index])
3321       index = bld.as_uniform(index);
3322    unsigned desc_set = nir_intrinsic_desc_set(instr);
3323    unsigned binding = nir_intrinsic_binding(instr);
3324
3325    Temp desc_ptr;
3326    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3327    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3328    unsigned offset = layout->binding[binding].offset;
3329    unsigned stride;
3330    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3331        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3332       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3333       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3334       offset = pipeline_layout->push_constant_size + 16 * idx;
3335       stride = 16;
3336    } else {
3337       desc_ptr = load_desc_ptr(ctx, desc_set);
3338       stride = layout->binding[binding].size;
3339    }
3340
3341    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3342    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3343    if (stride != 1) {
3344       if (nir_const_index) {
3345          const_index = const_index * stride;
3346       } else if (index.type() == RegType::vgpr) {
3347          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3348          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3349       } else {
3350          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3351       }
3352    }
3353    if (offset) {
3354       if (nir_const_index) {
3355          const_index = const_index + offset;
3356       } else if (index.type() == RegType::vgpr) {
3357          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3358       } else {
3359          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3360       }
3361    }
3362
3363    if (nir_const_index && const_index == 0) {
3364       index = desc_ptr;
3365    } else if (index.type() == RegType::vgpr) {
3366       index = bld.vadd32(bld.def(v1),
3367                          nir_const_index ? Operand(const_index) : Operand(index),
3368                          Operand(desc_ptr));
3369    } else {
3370       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3371                        nir_const_index ? Operand(const_index) : Operand(index),
3372                        Operand(desc_ptr));
3373    }
3374
3375    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3376 }
3377
3378 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3379                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3380 {
3381    Builder bld(ctx->program, ctx->block);
3382
3383    unsigned num_bytes = dst.size() * 4;
3384    bool dlc = glc && ctx->options->chip_class >= GFX10;
3385
3386    aco_opcode op;
3387    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3388       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3389       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3390       unsigned const_offset = 0;
3391
3392       Temp lower = Temp();
3393       if (num_bytes > 16) {
3394          assert(num_components == 3 || num_components == 4);
3395          op = aco_opcode::buffer_load_dwordx4;
3396          lower = bld.tmp(v4);
3397          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3398          mubuf->definitions[0] = Definition(lower);
3399          mubuf->operands[0] = vaddr;
3400          mubuf->operands[1] = Operand(rsrc);
3401          mubuf->operands[2] = soffset;
3402          mubuf->offen = (offset.type() == RegType::vgpr);
3403          mubuf->glc = glc;
3404          mubuf->dlc = dlc;
3405          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3406          mubuf->can_reorder = readonly;
3407          bld.insert(std::move(mubuf));
3408          emit_split_vector(ctx, lower, 2);
3409          num_bytes -= 16;
3410          const_offset = 16;
3411       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3412          /* GFX6 doesn't support loading vec3, expand to vec4. */
3413          num_bytes = 16;
3414       }
3415
3416       switch (num_bytes) {
3417          case 4:
3418             op = aco_opcode::buffer_load_dword;
3419             break;
3420          case 8:
3421             op = aco_opcode::buffer_load_dwordx2;
3422             break;
3423          case 12:
3424             assert(ctx->options->chip_class > GFX6);
3425             op = aco_opcode::buffer_load_dwordx3;
3426             break;
3427          case 16:
3428             op = aco_opcode::buffer_load_dwordx4;
3429             break;
3430          default:
3431             unreachable("Load SSBO not implemented for this size.");
3432       }
3433       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3434       mubuf->operands[0] = vaddr;
3435       mubuf->operands[1] = Operand(rsrc);
3436       mubuf->operands[2] = soffset;
3437       mubuf->offen = (offset.type() == RegType::vgpr);
3438       mubuf->glc = glc;
3439       mubuf->dlc = dlc;
3440       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3441       mubuf->can_reorder = readonly;
3442       mubuf->offset = const_offset;
3443       aco_ptr<Instruction> instr = std::move(mubuf);
3444
3445       if (dst.size() > 4) {
3446          assert(lower != Temp());
3447          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3448          instr->definitions[0] = Definition(upper);
3449          bld.insert(std::move(instr));
3450          if (dst.size() == 8)
3451             emit_split_vector(ctx, upper, 2);
3452          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3453          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3454          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3455          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3456          if (dst.size() == 8)
3457             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3458       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3459          Temp vec = bld.tmp(v4);
3460          instr->definitions[0] = Definition(vec);
3461          bld.insert(std::move(instr));
3462          emit_split_vector(ctx, vec, 4);
3463
3464          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3465          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3466          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3467          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3468       }
3469
3470       if (dst.type() == RegType::sgpr) {
3471          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3472          instr->definitions[0] = Definition(vec);
3473          bld.insert(std::move(instr));
3474          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3475       } else {
3476          instr->definitions[0] = Definition(dst);
3477          bld.insert(std::move(instr));
3478          emit_split_vector(ctx, dst, num_components);
3479       }
3480    } else {
3481       switch (num_bytes) {
3482          case 4:
3483             op = aco_opcode::s_buffer_load_dword;
3484             break;
3485          case 8:
3486             op = aco_opcode::s_buffer_load_dwordx2;
3487             break;
3488          case 12:
3489          case 16:
3490             op = aco_opcode::s_buffer_load_dwordx4;
3491             break;
3492          case 24:
3493          case 32:
3494             op = aco_opcode::s_buffer_load_dwordx8;
3495             break;
3496          default:
3497             unreachable("Load SSBO not implemented for this size.");
3498       }
3499       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3500       load->operands[0] = Operand(rsrc);
3501       load->operands[1] = Operand(bld.as_uniform(offset));
3502       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3503       load->definitions[0] = Definition(dst);
3504       load->glc = glc;
3505       load->dlc = dlc;
3506       load->barrier = readonly ? barrier_none : barrier_buffer;
3507       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3508       assert(ctx->options->chip_class >= GFX8 || !glc);
3509
3510       /* trim vector */
3511       if (dst.size() == 3) {
3512          Temp vec = bld.tmp(s4);
3513          load->definitions[0] = Definition(vec);
3514          bld.insert(std::move(load));
3515          emit_split_vector(ctx, vec, 4);
3516
3517          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3518                     emit_extract_vector(ctx, vec, 0, s1),
3519                     emit_extract_vector(ctx, vec, 1, s1),
3520                     emit_extract_vector(ctx, vec, 2, s1));
3521       } else if (dst.size() == 6) {
3522          Temp vec = bld.tmp(s8);
3523          load->definitions[0] = Definition(vec);
3524          bld.insert(std::move(load));
3525          emit_split_vector(ctx, vec, 4);
3526
3527          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3528                     emit_extract_vector(ctx, vec, 0, s2),
3529                     emit_extract_vector(ctx, vec, 1, s2),
3530                     emit_extract_vector(ctx, vec, 2, s2));
3531       } else {
3532          bld.insert(std::move(load));
3533       }
3534       emit_split_vector(ctx, dst, num_components);
3535    }
3536 }
3537
3538 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3539 {
3540    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3541    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3542
3543    Builder bld(ctx->program, ctx->block);
3544
3545    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3546    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3547    unsigned binding = nir_intrinsic_binding(idx_instr);
3548    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3549
3550    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3551       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3552                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3553                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3554                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3555       if (ctx->options->chip_class >= GFX10) {
3556          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3557                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3558                       S_008F0C_RESOURCE_LEVEL(1);
3559       } else {
3560          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3561                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3562       }
3563       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3564                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3565                                      Operand(0xFFFFFFFFu),
3566                                      Operand(desc_type));
3567       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3568                         rsrc, upper_dwords);
3569    } else {
3570       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3571       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3572    }
3573
3574    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3575 }
3576
3577 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3578 {
3579    Builder bld(ctx->program, ctx->block);
3580    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3581
3582    unsigned offset = nir_intrinsic_base(instr);
3583    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3584    if (index_cv && instr->dest.ssa.bit_size == 32) {
3585
3586       unsigned count = instr->dest.ssa.num_components;
3587       unsigned start = (offset + index_cv->u32) / 4u;
3588       start -= ctx->args->ac.base_inline_push_consts;
3589       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3590          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3591          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3592          for (unsigned i = 0; i < count; ++i) {
3593             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3594             vec->operands[i] = Operand{elems[i]};
3595          }
3596          vec->definitions[0] = Definition(dst);
3597          ctx->block->instructions.emplace_back(std::move(vec));
3598          ctx->allocated_vec.emplace(dst.id(), elems);
3599          return;
3600       }
3601    }
3602
3603    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3604    if (offset != 0) // TODO check if index != 0 as well
3605       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3606    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3607    Temp vec = dst;
3608    bool trim = false;
3609    aco_opcode op;
3610
3611    switch (dst.size()) {
3612    case 1:
3613       op = aco_opcode::s_load_dword;
3614       break;
3615    case 2:
3616       op = aco_opcode::s_load_dwordx2;
3617       break;
3618    case 3:
3619       vec = bld.tmp(s4);
3620       trim = true;
3621    case 4:
3622       op = aco_opcode::s_load_dwordx4;
3623       break;
3624    case 6:
3625       vec = bld.tmp(s8);
3626       trim = true;
3627    case 8:
3628       op = aco_opcode::s_load_dwordx8;
3629       break;
3630    default:
3631       unreachable("unimplemented or forbidden load_push_constant.");
3632    }
3633
3634    bld.smem(op, Definition(vec), ptr, index);
3635
3636    if (trim) {
3637       emit_split_vector(ctx, vec, 4);
3638       RegClass rc = dst.size() == 3 ? s1 : s2;
3639       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3640                  emit_extract_vector(ctx, vec, 0, rc),
3641                  emit_extract_vector(ctx, vec, 1, rc),
3642                  emit_extract_vector(ctx, vec, 2, rc));
3643
3644    }
3645    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3646 }
3647
3648 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3649 {
3650    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3651
3652    Builder bld(ctx->program, ctx->block);
3653
3654    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3655                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3656                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3657                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3658    if (ctx->options->chip_class >= GFX10) {
3659       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3660                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3661                    S_008F0C_RESOURCE_LEVEL(1);
3662    } else {
3663       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3664                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3665    }
3666
3667    unsigned base = nir_intrinsic_base(instr);
3668    unsigned range = nir_intrinsic_range(instr);
3669
3670    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3671    if (base && offset.type() == RegType::sgpr)
3672       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3673    else if (base && offset.type() == RegType::vgpr)
3674       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3675
3676    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3677                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3678                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3679                           Operand(desc_type));
3680
3681    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3682 }
3683
3684 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3685 {
3686    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3687       ctx->cf_info.exec_potentially_empty = true;
3688
3689    ctx->program->needs_exact = true;
3690
3691    // TODO: optimize uniform conditions
3692    Builder bld(ctx->program, ctx->block);
3693    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3694    assert(src.regClass() == bld.lm);
3695    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3696    bld.pseudo(aco_opcode::p_discard_if, src);
3697    ctx->block->kind |= block_kind_uses_discard_if;
3698    return;
3699 }
3700
3701 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3702 {
3703    Builder bld(ctx->program, ctx->block);
3704
3705    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3706       ctx->cf_info.exec_potentially_empty = true;
3707
3708    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3709                     ctx->cf_info.parent_loop.has_divergent_continue;
3710
3711    if (ctx->block->loop_nest_depth &&
3712        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3713       /* we handle discards the same way as jump instructions */
3714       append_logical_end(ctx->block);
3715
3716       /* in loops, discard behaves like break */
3717       Block *linear_target = ctx->cf_info.parent_loop.exit;
3718       ctx->block->kind |= block_kind_discard;
3719
3720       if (!divergent) {
3721          /* uniform discard - loop ends here */
3722          assert(nir_instr_is_last(&instr->instr));
3723          ctx->block->kind |= block_kind_uniform;
3724          ctx->cf_info.has_branch = true;
3725          bld.branch(aco_opcode::p_branch);
3726          add_linear_edge(ctx->block->index, linear_target);
3727          return;
3728       }
3729
3730       /* we add a break right behind the discard() instructions */
3731       ctx->block->kind |= block_kind_break;
3732       unsigned idx = ctx->block->index;
3733
3734       /* remove critical edges from linear CFG */
3735       bld.branch(aco_opcode::p_branch);
3736       Block* break_block = ctx->program->create_and_insert_block();
3737       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3738       break_block->kind |= block_kind_uniform;
3739       add_linear_edge(idx, break_block);
3740       add_linear_edge(break_block->index, linear_target);
3741       bld.reset(break_block);
3742       bld.branch(aco_opcode::p_branch);
3743
3744       Block* continue_block = ctx->program->create_and_insert_block();
3745       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3746       add_linear_edge(idx, continue_block);
3747       append_logical_start(continue_block);
3748       ctx->block = continue_block;
3749
3750       return;
3751    }
3752
3753    /* it can currently happen that NIR doesn't remove the unreachable code */
3754    if (!nir_instr_is_last(&instr->instr)) {
3755       ctx->program->needs_exact = true;
3756       /* save exec somewhere temporarily so that it doesn't get
3757        * overwritten before the discard from outer exec masks */
3758       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3759       bld.pseudo(aco_opcode::p_discard_if, cond);
3760       ctx->block->kind |= block_kind_uses_discard_if;
3761       return;
3762    }
3763
3764    /* This condition is incorrect for uniformly branched discards in a loop
3765     * predicated by a divergent condition, but the above code catches that case
3766     * and the discard would end up turning into a discard_if.
3767     * For example:
3768     * if (divergent) {
3769     *    while (...) {
3770     *       if (uniform) {
3771     *          discard;
3772     *       }
3773     *    }
3774     * }
3775     */
3776    if (!ctx->cf_info.parent_if.is_divergent) {
3777       /* program just ends here */
3778       ctx->block->kind |= block_kind_uniform;
3779       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3780               0 /* enabled mask */, 9 /* dest */,
3781               false /* compressed */, true/* done */, true /* valid mask */);
3782       bld.sopp(aco_opcode::s_endpgm);
3783       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3784    } else {
3785       ctx->block->kind |= block_kind_discard;
3786       /* branch and linear edge is added by visit_if() */
3787    }
3788 }
3789
3790 enum aco_descriptor_type {
3791    ACO_DESC_IMAGE,
3792    ACO_DESC_FMASK,
3793    ACO_DESC_SAMPLER,
3794    ACO_DESC_BUFFER,
3795    ACO_DESC_PLANE_0,
3796    ACO_DESC_PLANE_1,
3797    ACO_DESC_PLANE_2,
3798 };
3799
3800 static bool
3801 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3802    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3803       return false;
3804    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3805    return dim == ac_image_cube ||
3806           dim == ac_image_1darray ||
3807           dim == ac_image_2darray ||
3808           dim == ac_image_2darraymsaa;
3809 }
3810
3811 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3812                       enum aco_descriptor_type desc_type,
3813                       const nir_tex_instr *tex_instr, bool image, bool write)
3814 {
3815 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3816    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3817    if (it != ctx->tex_desc.end())
3818       return it->second;
3819 */
3820    Temp index = Temp();
3821    bool index_set = false;
3822    unsigned constant_index = 0;
3823    unsigned descriptor_set;
3824    unsigned base_index;
3825    Builder bld(ctx->program, ctx->block);
3826
3827    if (!deref_instr) {
3828       assert(tex_instr && !image);
3829       descriptor_set = 0;
3830       base_index = tex_instr->sampler_index;
3831    } else {
3832       while(deref_instr->deref_type != nir_deref_type_var) {
3833          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3834          if (!array_size)
3835             array_size = 1;
3836
3837          assert(deref_instr->deref_type == nir_deref_type_array);
3838          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3839          if (const_value) {
3840             constant_index += array_size * const_value->u32;
3841          } else {
3842             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3843             if (indirect.type() == RegType::vgpr)
3844                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3845
3846             if (array_size != 1)
3847                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3848
3849             if (!index_set) {
3850                index = indirect;
3851                index_set = true;
3852             } else {
3853                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3854             }
3855          }
3856
3857          deref_instr = nir_src_as_deref(deref_instr->parent);
3858       }
3859       descriptor_set = deref_instr->var->data.descriptor_set;
3860       base_index = deref_instr->var->data.binding;
3861    }
3862
3863    Temp list = load_desc_ptr(ctx, descriptor_set);
3864    list = convert_pointer_to_64_bit(ctx, list);
3865
3866    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3867    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3868    unsigned offset = binding->offset;
3869    unsigned stride = binding->size;
3870    aco_opcode opcode;
3871    RegClass type;
3872
3873    assert(base_index < layout->binding_count);
3874
3875    switch (desc_type) {
3876    case ACO_DESC_IMAGE:
3877       type = s8;
3878       opcode = aco_opcode::s_load_dwordx8;
3879       break;
3880    case ACO_DESC_FMASK:
3881       type = s8;
3882       opcode = aco_opcode::s_load_dwordx8;
3883       offset += 32;
3884       break;
3885    case ACO_DESC_SAMPLER:
3886       type = s4;
3887       opcode = aco_opcode::s_load_dwordx4;
3888       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3889          offset += radv_combined_image_descriptor_sampler_offset(binding);
3890       break;
3891    case ACO_DESC_BUFFER:
3892       type = s4;
3893       opcode = aco_opcode::s_load_dwordx4;
3894       break;
3895    case ACO_DESC_PLANE_0:
3896    case ACO_DESC_PLANE_1:
3897       type = s8;
3898       opcode = aco_opcode::s_load_dwordx8;
3899       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3900       break;
3901    case ACO_DESC_PLANE_2:
3902       type = s4;
3903       opcode = aco_opcode::s_load_dwordx4;
3904       offset += 64;
3905       break;
3906    default:
3907       unreachable("invalid desc_type\n");
3908    }
3909
3910    offset += constant_index * stride;
3911
3912    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3913       (!index_set || binding->immutable_samplers_equal)) {
3914       if (binding->immutable_samplers_equal)
3915          constant_index = 0;
3916
3917       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3918       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3919                         Operand(samplers[constant_index * 4 + 0]),
3920                         Operand(samplers[constant_index * 4 + 1]),
3921                         Operand(samplers[constant_index * 4 + 2]),
3922                         Operand(samplers[constant_index * 4 + 3]));
3923    }
3924
3925    Operand off;
3926    if (!index_set) {
3927       off = Operand(offset);
3928    } else {
3929       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3930                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3931    }
3932
3933    Temp res = bld.smem(opcode, bld.def(type), list, off);
3934
3935    if (desc_type == ACO_DESC_PLANE_2) {
3936       Temp components[8];
3937       for (unsigned i = 0; i < 8; i++)
3938          components[i] = bld.tmp(s1);
3939       bld.pseudo(aco_opcode::p_split_vector,
3940                  Definition(components[0]),
3941                  Definition(components[1]),
3942                  Definition(components[2]),
3943                  Definition(components[3]),
3944                  res);
3945
3946       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3947       bld.pseudo(aco_opcode::p_split_vector,
3948                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3949                  Definition(components[4]),
3950                  Definition(components[5]),
3951                  Definition(components[6]),
3952                  Definition(components[7]),
3953                  desc2);
3954
3955       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3956                        components[0], components[1], components[2], components[3],
3957                        components[4], components[5], components[6], components[7]);
3958    }
3959
3960    return res;
3961 }
3962
3963 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3964 {
3965    switch (dim) {
3966    case GLSL_SAMPLER_DIM_BUF:
3967       return 1;
3968    case GLSL_SAMPLER_DIM_1D:
3969       return array ? 2 : 1;
3970    case GLSL_SAMPLER_DIM_2D:
3971       return array ? 3 : 2;
3972    case GLSL_SAMPLER_DIM_MS:
3973       return array ? 4 : 3;
3974    case GLSL_SAMPLER_DIM_3D:
3975    case GLSL_SAMPLER_DIM_CUBE:
3976       return 3;
3977    case GLSL_SAMPLER_DIM_RECT:
3978    case GLSL_SAMPLER_DIM_SUBPASS:
3979       return 2;
3980    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3981       return 3;
3982    default:
3983       break;
3984    }
3985    return 0;
3986 }
3987
3988
3989 /* Adjust the sample index according to FMASK.
3990  *
3991  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3992  * which is the identity mapping. Each nibble says which physical sample
3993  * should be fetched to get that sample.
3994  *
3995  * For example, 0x11111100 means there are only 2 samples stored and
3996  * the second sample covers 3/4 of the pixel. When reading samples 0
3997  * and 1, return physical sample 0 (determined by the first two 0s
3998  * in FMASK), otherwise return physical sample 1.
3999  *
4000  * The sample index should be adjusted as follows:
4001  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4002  */
4003 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4004 {
4005    Builder bld(ctx->program, ctx->block);
4006    Temp fmask = bld.tmp(v1);
4007    unsigned dim = ctx->options->chip_class >= GFX10
4008                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4009                   : 0;
4010
4011    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4012    load->operands[0] = Operand(coords);
4013    load->operands[1] = Operand(fmask_desc_ptr);
4014    load->definitions[0] = Definition(fmask);
4015    load->glc = false;
4016    load->dlc = false;
4017    load->dmask = 0x1;
4018    load->unrm = true;
4019    load->da = da;
4020    load->dim = dim;
4021    load->can_reorder = true; /* fmask images shouldn't be modified */
4022    ctx->block->instructions.emplace_back(std::move(load));
4023
4024    Operand sample_index4;
4025    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4026       sample_index4 = Operand(sample_index.constantValue() << 2);
4027    } else if (sample_index.regClass() == s1) {
4028       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4029    } else {
4030       assert(sample_index.regClass() == v1);
4031       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4032    }
4033
4034    Temp final_sample;
4035    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4036       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4037    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4038       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4039    else
4040       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4041
4042    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4043     * resource descriptor is 0 (invalid),
4044     */
4045    Temp compare = bld.tmp(bld.lm);
4046    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4047                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4048
4049    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4050
4051    /* Replace the MSAA sample index. */
4052    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4053 }
4054
4055 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4056 {
4057
4058    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4059    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4060    bool is_array = glsl_sampler_type_is_array(type);
4061    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4062    assert(!add_frag_pos && "Input attachments should be lowered.");
4063    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4064    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4065    int count = image_type_to_components_count(dim, is_array);
4066    std::vector<Operand> coords(count);
4067
4068    if (is_ms) {
4069       Operand sample_index;
4070       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4071       if (sample_cv)
4072          sample_index = Operand(sample_cv->u32);
4073       else
4074          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4075
4076       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4077          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4078          for (unsigned i = 0; i < vec->operands.size(); i++)
4079             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4080          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4081          vec->definitions[0] = Definition(fmask_load_address);
4082          ctx->block->instructions.emplace_back(std::move(vec));
4083
4084          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4085          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4086       }
4087       count--;
4088       coords[count] = sample_index;
4089    }
4090
4091    if (count == 1 && !gfx9_1d)
4092       return emit_extract_vector(ctx, src0, 0, v1);
4093
4094    if (gfx9_1d) {
4095       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4096       coords.resize(coords.size() + 1);
4097       coords[1] = Operand((uint32_t) 0);
4098       if (is_array)
4099          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4100    } else {
4101       for (int i = 0; i < count; i++)
4102          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4103    }
4104
4105    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4106        instr->intrinsic == nir_intrinsic_image_deref_store) {
4107       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4108       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4109
4110       if (!level_zero)
4111          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4112    }
4113
4114    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4115    for (unsigned i = 0; i < coords.size(); i++)
4116       vec->operands[i] = coords[i];
4117    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4118    vec->definitions[0] = Definition(res);
4119    ctx->block->instructions.emplace_back(std::move(vec));
4120    return res;
4121 }
4122
4123
4124 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4125 {
4126    Builder bld(ctx->program, ctx->block);
4127    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4128    const struct glsl_type *type = glsl_without_array(var->type);
4129    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4130    bool is_array = glsl_sampler_type_is_array(type);
4131    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4132
4133    if (dim == GLSL_SAMPLER_DIM_BUF) {
4134       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4135       unsigned num_channels = util_last_bit(mask);
4136       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4137       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4138
4139       aco_opcode opcode;
4140       switch (num_channels) {
4141       case 1:
4142          opcode = aco_opcode::buffer_load_format_x;
4143          break;
4144       case 2:
4145          opcode = aco_opcode::buffer_load_format_xy;
4146          break;
4147       case 3:
4148          opcode = aco_opcode::buffer_load_format_xyz;
4149          break;
4150       case 4:
4151          opcode = aco_opcode::buffer_load_format_xyzw;
4152          break;
4153       default:
4154          unreachable(">4 channel buffer image load");
4155       }
4156       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4157       load->operands[0] = Operand(vindex);
4158       load->operands[1] = Operand(rsrc);
4159       load->operands[2] = Operand((uint32_t) 0);
4160       Temp tmp;
4161       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4162          tmp = dst;
4163       else
4164          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4165       load->definitions[0] = Definition(tmp);
4166       load->idxen = true;
4167       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4168       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4169       load->barrier = barrier_image;
4170       ctx->block->instructions.emplace_back(std::move(load));
4171
4172       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4173       return;
4174    }
4175
4176    Temp coords = get_image_coords(ctx, instr, type);
4177    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4178
4179    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4180    unsigned num_components = util_bitcount(dmask);
4181    Temp tmp;
4182    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4183       tmp = dst;
4184    else
4185       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4186
4187    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4188    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4189
4190    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4191    load->operands[0] = Operand(coords);
4192    load->operands[1] = Operand(resource);
4193    load->definitions[0] = Definition(tmp);
4194    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4195    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4196    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4197    load->dmask = dmask;
4198    load->unrm = true;
4199    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4200    load->barrier = barrier_image;
4201    ctx->block->instructions.emplace_back(std::move(load));
4202
4203    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4204    return;
4205 }
4206
4207 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4208 {
4209    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4210    const struct glsl_type *type = glsl_without_array(var->type);
4211    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4212    bool is_array = glsl_sampler_type_is_array(type);
4213    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4214
4215    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4216
4217    if (dim == GLSL_SAMPLER_DIM_BUF) {
4218       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4219       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4220       aco_opcode opcode;
4221       switch (data.size()) {
4222       case 1:
4223          opcode = aco_opcode::buffer_store_format_x;
4224          break;
4225       case 2:
4226          opcode = aco_opcode::buffer_store_format_xy;
4227          break;
4228       case 3:
4229          opcode = aco_opcode::buffer_store_format_xyz;
4230          break;
4231       case 4:
4232          opcode = aco_opcode::buffer_store_format_xyzw;
4233          break;
4234       default:
4235          unreachable(">4 channel buffer image store");
4236       }
4237       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4238       store->operands[0] = Operand(vindex);
4239       store->operands[1] = Operand(rsrc);
4240       store->operands[2] = Operand((uint32_t) 0);
4241       store->operands[3] = Operand(data);
4242       store->idxen = true;
4243       store->glc = glc;
4244       store->dlc = false;
4245       store->disable_wqm = true;
4246       store->barrier = barrier_image;
4247       ctx->program->needs_exact = true;
4248       ctx->block->instructions.emplace_back(std::move(store));
4249       return;
4250    }
4251
4252    assert(data.type() == RegType::vgpr);
4253    Temp coords = get_image_coords(ctx, instr, type);
4254    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4255
4256    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4257    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4258
4259    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4260    store->operands[0] = Operand(coords);
4261    store->operands[1] = Operand(resource);
4262    store->operands[2] = Operand(s4);
4263    store->operands[3] = Operand(data);
4264    store->glc = glc;
4265    store->dlc = false;
4266    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4267    store->dmask = (1 << data.size()) - 1;
4268    store->unrm = true;
4269    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4270    store->disable_wqm = true;
4271    store->barrier = barrier_image;
4272    ctx->program->needs_exact = true;
4273    ctx->block->instructions.emplace_back(std::move(store));
4274    return;
4275 }
4276
4277 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4278 {
4279    /* return the previous value if dest is ever used */
4280    bool return_previous = false;
4281    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4282       return_previous = true;
4283       break;
4284    }
4285    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4286       return_previous = true;
4287       break;
4288    }
4289
4290    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4291    const struct glsl_type *type = glsl_without_array(var->type);
4292    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4293    bool is_array = glsl_sampler_type_is_array(type);
4294    Builder bld(ctx->program, ctx->block);
4295
4296    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4297    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4298
4299    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4300       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4301
4302    aco_opcode buf_op, image_op;
4303    switch (instr->intrinsic) {
4304       case nir_intrinsic_image_deref_atomic_add:
4305          buf_op = aco_opcode::buffer_atomic_add;
4306          image_op = aco_opcode::image_atomic_add;
4307          break;
4308       case nir_intrinsic_image_deref_atomic_umin:
4309          buf_op = aco_opcode::buffer_atomic_umin;
4310          image_op = aco_opcode::image_atomic_umin;
4311          break;
4312       case nir_intrinsic_image_deref_atomic_imin:
4313          buf_op = aco_opcode::buffer_atomic_smin;
4314          image_op = aco_opcode::image_atomic_smin;
4315          break;
4316       case nir_intrinsic_image_deref_atomic_umax:
4317          buf_op = aco_opcode::buffer_atomic_umax;
4318          image_op = aco_opcode::image_atomic_umax;
4319          break;
4320       case nir_intrinsic_image_deref_atomic_imax:
4321          buf_op = aco_opcode::buffer_atomic_smax;
4322          image_op = aco_opcode::image_atomic_smax;
4323          break;
4324       case nir_intrinsic_image_deref_atomic_and:
4325          buf_op = aco_opcode::buffer_atomic_and;
4326          image_op = aco_opcode::image_atomic_and;
4327          break;
4328       case nir_intrinsic_image_deref_atomic_or:
4329          buf_op = aco_opcode::buffer_atomic_or;
4330          image_op = aco_opcode::image_atomic_or;
4331          break;
4332       case nir_intrinsic_image_deref_atomic_xor:
4333          buf_op = aco_opcode::buffer_atomic_xor;
4334          image_op = aco_opcode::image_atomic_xor;
4335          break;
4336       case nir_intrinsic_image_deref_atomic_exchange:
4337          buf_op = aco_opcode::buffer_atomic_swap;
4338          image_op = aco_opcode::image_atomic_swap;
4339          break;
4340       case nir_intrinsic_image_deref_atomic_comp_swap:
4341          buf_op = aco_opcode::buffer_atomic_cmpswap;
4342          image_op = aco_opcode::image_atomic_cmpswap;
4343          break;
4344       default:
4345          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4346    }
4347
4348    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4349
4350    if (dim == GLSL_SAMPLER_DIM_BUF) {
4351       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4352       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4353       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4354       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4355       mubuf->operands[0] = Operand(vindex);
4356       mubuf->operands[1] = Operand(resource);
4357       mubuf->operands[2] = Operand((uint32_t)0);
4358       mubuf->operands[3] = Operand(data);
4359       if (return_previous)
4360          mubuf->definitions[0] = Definition(dst);
4361       mubuf->offset = 0;
4362       mubuf->idxen = true;
4363       mubuf->glc = return_previous;
4364       mubuf->dlc = false; /* Not needed for atomics */
4365       mubuf->disable_wqm = true;
4366       mubuf->barrier = barrier_image;
4367       ctx->program->needs_exact = true;
4368       ctx->block->instructions.emplace_back(std::move(mubuf));
4369       return;
4370    }
4371
4372    Temp coords = get_image_coords(ctx, instr, type);
4373    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4374    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4375    mimg->operands[0] = Operand(coords);
4376    mimg->operands[1] = Operand(resource);
4377    mimg->operands[2] = Operand(s4); /* no sampler */
4378    mimg->operands[3] = Operand(data);
4379    if (return_previous)
4380       mimg->definitions[0] = Definition(dst);
4381    mimg->glc = return_previous;
4382    mimg->dlc = false; /* Not needed for atomics */
4383    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4384    mimg->dmask = (1 << data.size()) - 1;
4385    mimg->unrm = true;
4386    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4387    mimg->disable_wqm = true;
4388    mimg->barrier = barrier_image;
4389    ctx->program->needs_exact = true;
4390    ctx->block->instructions.emplace_back(std::move(mimg));
4391    return;
4392 }
4393
4394 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4395 {
4396    if (in_elements && ctx->options->chip_class == GFX8) {
4397       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4398       Builder bld(ctx->program, ctx->block);
4399
4400       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4401
4402       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4403       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4404
4405       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4406       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4407
4408       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4409       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4410
4411       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4412       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4413                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4414       if (dst.type() == RegType::vgpr)
4415          bld.copy(Definition(dst), shr_dst);
4416
4417       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4418    } else {
4419       emit_extract_vector(ctx, desc, 2, dst);
4420    }
4421 }
4422
4423 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4424 {
4425    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4426    const struct glsl_type *type = glsl_without_array(var->type);
4427    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4428    bool is_array = glsl_sampler_type_is_array(type);
4429    Builder bld(ctx->program, ctx->block);
4430
4431    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4432       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4433       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4434    }
4435
4436    /* LOD */
4437    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4438
4439    /* Resource */
4440    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4441
4442    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4443
4444    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4445    mimg->operands[0] = Operand(lod);
4446    mimg->operands[1] = Operand(resource);
4447    uint8_t& dmask = mimg->dmask;
4448    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4449    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4450    mimg->da = glsl_sampler_type_is_array(type);
4451    mimg->can_reorder = true;
4452    Definition& def = mimg->definitions[0];
4453    ctx->block->instructions.emplace_back(std::move(mimg));
4454
4455    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4456        glsl_sampler_type_is_array(type)) {
4457
4458       assert(instr->dest.ssa.num_components == 3);
4459       Temp tmp = {ctx->program->allocateId(), v3};
4460       def = Definition(tmp);
4461       emit_split_vector(ctx, tmp, 3);
4462
4463       /* divide 3rd value by 6 by multiplying with magic number */
4464       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4465       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4466
4467       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4468                  emit_extract_vector(ctx, tmp, 0, v1),
4469                  emit_extract_vector(ctx, tmp, 1, v1),
4470                  by_6);
4471
4472    } else if (ctx->options->chip_class == GFX9 &&
4473               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4474               glsl_sampler_type_is_array(type)) {
4475       assert(instr->dest.ssa.num_components == 2);
4476       def = Definition(dst);
4477       dmask = 0x5;
4478    } else {
4479       def = Definition(dst);
4480    }
4481
4482    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4483 }
4484
4485 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4486 {
4487    Builder bld(ctx->program, ctx->block);
4488    unsigned num_components = instr->num_components;
4489
4490    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4491    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4492    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4493
4494    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4495    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4496 }
4497
4498 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4499 {
4500    Builder bld(ctx->program, ctx->block);
4501    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4502    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4503    unsigned writemask = nir_intrinsic_write_mask(instr);
4504    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4505
4506    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4507    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4508
4509    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4510                ctx->options->chip_class >= GFX8;
4511    if (smem)
4512       offset = bld.as_uniform(offset);
4513    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4514
4515    while (writemask) {
4516       int start, count;
4517       u_bit_scan_consecutive_range(&writemask, &start, &count);
4518       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4519          /* GFX6 doesn't support storing vec3, split it. */
4520          writemask |= 1u << (start + 2);
4521          count = 2;
4522       }
4523       int num_bytes = count * elem_size_bytes;
4524
4525       if (num_bytes > 16) {
4526          assert(elem_size_bytes == 8);
4527          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4528          count = 2;
4529          num_bytes = 16;
4530       }
4531
4532       // TODO: check alignment of sub-dword stores
4533       // TODO: split 3 bytes. there is no store instruction for that
4534
4535       Temp write_data;
4536       if (count != instr->num_components) {
4537          emit_split_vector(ctx, data, instr->num_components);
4538          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4539          for (int i = 0; i < count; i++) {
4540             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4541             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4542          }
4543          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4544          vec->definitions[0] = Definition(write_data);
4545          ctx->block->instructions.emplace_back(std::move(vec));
4546       } else if (!smem && data.type() != RegType::vgpr) {
4547          assert(num_bytes % 4 == 0);
4548          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4549       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4550          assert(num_bytes % 4 == 0);
4551          write_data = bld.as_uniform(data);
4552       } else {
4553          write_data = data;
4554       }
4555
4556       aco_opcode vmem_op, smem_op;
4557       switch (num_bytes) {
4558          case 4:
4559             vmem_op = aco_opcode::buffer_store_dword;
4560             smem_op = aco_opcode::s_buffer_store_dword;
4561             break;
4562          case 8:
4563             vmem_op = aco_opcode::buffer_store_dwordx2;
4564             smem_op = aco_opcode::s_buffer_store_dwordx2;
4565             break;
4566          case 12:
4567             vmem_op = aco_opcode::buffer_store_dwordx3;
4568             smem_op = aco_opcode::last_opcode;
4569             assert(!smem && ctx->options->chip_class > GFX6);
4570             break;
4571          case 16:
4572             vmem_op = aco_opcode::buffer_store_dwordx4;
4573             smem_op = aco_opcode::s_buffer_store_dwordx4;
4574             break;
4575          default:
4576             unreachable("Store SSBO not implemented for this size.");
4577       }
4578       if (ctx->stage == fragment_fs)
4579          smem_op = aco_opcode::p_fs_buffer_store_smem;
4580
4581       if (smem) {
4582          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4583          store->operands[0] = Operand(rsrc);
4584          if (start) {
4585             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4586                                 offset, Operand(start * elem_size_bytes));
4587             store->operands[1] = Operand(off);
4588          } else {
4589             store->operands[1] = Operand(offset);
4590          }
4591          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4592             store->operands[1].setFixed(m0);
4593          store->operands[2] = Operand(write_data);
4594          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4595          store->dlc = false;
4596          store->disable_wqm = true;
4597          store->barrier = barrier_buffer;
4598          ctx->block->instructions.emplace_back(std::move(store));
4599          ctx->program->wb_smem_l1_on_end = true;
4600          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4601             ctx->block->kind |= block_kind_needs_lowering;
4602             ctx->program->needs_exact = true;
4603          }
4604       } else {
4605          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4606          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4607          store->operands[1] = Operand(rsrc);
4608          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4609          store->operands[3] = Operand(write_data);
4610          store->offset = start * elem_size_bytes;
4611          store->offen = (offset.type() == RegType::vgpr);
4612          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4613          store->dlc = false;
4614          store->disable_wqm = true;
4615          store->barrier = barrier_buffer;
4616          ctx->program->needs_exact = true;
4617          ctx->block->instructions.emplace_back(std::move(store));
4618       }
4619    }
4620 }
4621
4622 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4623 {
4624    /* return the previous value if dest is ever used */
4625    bool return_previous = false;
4626    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4627       return_previous = true;
4628       break;
4629    }
4630    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4631       return_previous = true;
4632       break;
4633    }
4634
4635    Builder bld(ctx->program, ctx->block);
4636    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4637
4638    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4639       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4640                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4641
4642    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4643    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4644    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4645
4646    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4647
4648    aco_opcode op32, op64;
4649    switch (instr->intrinsic) {
4650       case nir_intrinsic_ssbo_atomic_add:
4651          op32 = aco_opcode::buffer_atomic_add;
4652          op64 = aco_opcode::buffer_atomic_add_x2;
4653          break;
4654       case nir_intrinsic_ssbo_atomic_imin:
4655          op32 = aco_opcode::buffer_atomic_smin;
4656          op64 = aco_opcode::buffer_atomic_smin_x2;
4657          break;
4658       case nir_intrinsic_ssbo_atomic_umin:
4659          op32 = aco_opcode::buffer_atomic_umin;
4660          op64 = aco_opcode::buffer_atomic_umin_x2;
4661          break;
4662       case nir_intrinsic_ssbo_atomic_imax:
4663          op32 = aco_opcode::buffer_atomic_smax;
4664          op64 = aco_opcode::buffer_atomic_smax_x2;
4665          break;
4666       case nir_intrinsic_ssbo_atomic_umax:
4667          op32 = aco_opcode::buffer_atomic_umax;
4668          op64 = aco_opcode::buffer_atomic_umax_x2;
4669          break;
4670       case nir_intrinsic_ssbo_atomic_and:
4671          op32 = aco_opcode::buffer_atomic_and;
4672          op64 = aco_opcode::buffer_atomic_and_x2;
4673          break;
4674       case nir_intrinsic_ssbo_atomic_or:
4675          op32 = aco_opcode::buffer_atomic_or;
4676          op64 = aco_opcode::buffer_atomic_or_x2;
4677          break;
4678       case nir_intrinsic_ssbo_atomic_xor:
4679          op32 = aco_opcode::buffer_atomic_xor;
4680          op64 = aco_opcode::buffer_atomic_xor_x2;
4681          break;
4682       case nir_intrinsic_ssbo_atomic_exchange:
4683          op32 = aco_opcode::buffer_atomic_swap;
4684          op64 = aco_opcode::buffer_atomic_swap_x2;
4685          break;
4686       case nir_intrinsic_ssbo_atomic_comp_swap:
4687          op32 = aco_opcode::buffer_atomic_cmpswap;
4688          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4689          break;
4690       default:
4691          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4692    }
4693    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4694    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4695    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4696    mubuf->operands[1] = Operand(rsrc);
4697    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4698    mubuf->operands[3] = Operand(data);
4699    if (return_previous)
4700       mubuf->definitions[0] = Definition(dst);
4701    mubuf->offset = 0;
4702    mubuf->offen = (offset.type() == RegType::vgpr);
4703    mubuf->glc = return_previous;
4704    mubuf->dlc = false; /* Not needed for atomics */
4705    mubuf->disable_wqm = true;
4706    mubuf->barrier = barrier_buffer;
4707    ctx->program->needs_exact = true;
4708    ctx->block->instructions.emplace_back(std::move(mubuf));
4709 }
4710
4711 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4712
4713    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4714    Builder bld(ctx->program, ctx->block);
4715    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4716    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4717 }
4718
4719 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4720 {
4721    Builder bld(ctx->program, ctx->block);
4722    unsigned num_components = instr->num_components;
4723    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4724
4725    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4726    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4727
4728    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4729    bool dlc = glc && ctx->options->chip_class >= GFX10;
4730    aco_opcode op;
4731    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4732       bool global = ctx->options->chip_class >= GFX9;
4733       aco_opcode op;
4734       switch (num_bytes) {
4735       case 4:
4736          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4737          break;
4738       case 8:
4739          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4740          break;
4741       case 12:
4742          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4743          break;
4744       case 16:
4745          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4746          break;
4747       default:
4748          unreachable("load_global not implemented for this size.");
4749       }
4750       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4751       flat->operands[0] = Operand(addr);
4752       flat->operands[1] = Operand(s1);
4753       flat->glc = glc;
4754       flat->dlc = dlc;
4755       flat->barrier = barrier_buffer;
4756
4757       if (dst.type() == RegType::sgpr) {
4758          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4759          flat->definitions[0] = Definition(vec);
4760          ctx->block->instructions.emplace_back(std::move(flat));
4761          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4762       } else {
4763          flat->definitions[0] = Definition(dst);
4764          ctx->block->instructions.emplace_back(std::move(flat));
4765       }
4766       emit_split_vector(ctx, dst, num_components);
4767    } else {
4768       switch (num_bytes) {
4769          case 4:
4770             op = aco_opcode::s_load_dword;
4771             break;
4772          case 8:
4773             op = aco_opcode::s_load_dwordx2;
4774             break;
4775          case 12:
4776          case 16:
4777             op = aco_opcode::s_load_dwordx4;
4778             break;
4779          default:
4780             unreachable("load_global not implemented for this size.");
4781       }
4782       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4783       load->operands[0] = Operand(addr);
4784       load->operands[1] = Operand(0u);
4785       load->definitions[0] = Definition(dst);
4786       load->glc = glc;
4787       load->dlc = dlc;
4788       load->barrier = barrier_buffer;
4789       assert(ctx->options->chip_class >= GFX8 || !glc);
4790
4791       if (dst.size() == 3) {
4792          /* trim vector */
4793          Temp vec = bld.tmp(s4);
4794          load->definitions[0] = Definition(vec);
4795          ctx->block->instructions.emplace_back(std::move(load));
4796          emit_split_vector(ctx, vec, 4);
4797
4798          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4799                     emit_extract_vector(ctx, vec, 0, s1),
4800                     emit_extract_vector(ctx, vec, 1, s1),
4801                     emit_extract_vector(ctx, vec, 2, s1));
4802       } else {
4803          ctx->block->instructions.emplace_back(std::move(load));
4804       }
4805    }
4806 }
4807
4808 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4809 {
4810    Builder bld(ctx->program, ctx->block);
4811    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4812
4813    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4814    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4815
4816    unsigned writemask = nir_intrinsic_write_mask(instr);
4817    while (writemask) {
4818       int start, count;
4819       u_bit_scan_consecutive_range(&writemask, &start, &count);
4820       unsigned num_bytes = count * elem_size_bytes;
4821
4822       Temp write_data = data;
4823       if (count != instr->num_components) {
4824          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4825          for (int i = 0; i < count; i++)
4826             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4827          write_data = bld.tmp(RegType::vgpr, count);
4828          vec->definitions[0] = Definition(write_data);
4829          ctx->block->instructions.emplace_back(std::move(vec));
4830       }
4831
4832       unsigned offset = start * elem_size_bytes;
4833       if (offset > 0 && ctx->options->chip_class < GFX9) {
4834          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4835          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4836          Temp carry = bld.tmp(bld.lm);
4837          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4838
4839          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4840                   Operand(offset), addr0);
4841          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4842                   Operand(0u), addr1,
4843                   carry).def(1).setHint(vcc);
4844
4845          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4846
4847          offset = 0;
4848       }
4849
4850       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4851       bool global = ctx->options->chip_class >= GFX9;
4852       aco_opcode op;
4853       switch (num_bytes) {
4854       case 4:
4855          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4856          break;
4857       case 8:
4858          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4859          break;
4860       case 12:
4861          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4862          break;
4863       case 16:
4864          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4865          break;
4866       default:
4867          unreachable("store_global not implemented for this size.");
4868       }
4869       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4870       flat->operands[0] = Operand(addr);
4871       flat->operands[1] = Operand(s1);
4872       flat->operands[2] = Operand(data);
4873       flat->glc = glc;
4874       flat->dlc = false;
4875       flat->offset = offset;
4876       flat->disable_wqm = true;
4877       flat->barrier = barrier_buffer;
4878       ctx->program->needs_exact = true;
4879       ctx->block->instructions.emplace_back(std::move(flat));
4880    }
4881 }
4882
4883 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4884 {
4885    /* return the previous value if dest is ever used */
4886    bool return_previous = false;
4887    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4888       return_previous = true;
4889       break;
4890    }
4891    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4892       return_previous = true;
4893       break;
4894    }
4895
4896    Builder bld(ctx->program, ctx->block);
4897    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4898    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4899
4900    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4901       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4902                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4903
4904    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4905
4906    bool global = ctx->options->chip_class >= GFX9;
4907    aco_opcode op32, op64;
4908    switch (instr->intrinsic) {
4909       case nir_intrinsic_global_atomic_add:
4910          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4911          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4912          break;
4913       case nir_intrinsic_global_atomic_imin:
4914          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4915          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4916          break;
4917       case nir_intrinsic_global_atomic_umin:
4918          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4919          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4920          break;
4921       case nir_intrinsic_global_atomic_imax:
4922          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4923          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4924          break;
4925       case nir_intrinsic_global_atomic_umax:
4926          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4927          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4928          break;
4929       case nir_intrinsic_global_atomic_and:
4930          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4931          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4932          break;
4933       case nir_intrinsic_global_atomic_or:
4934          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4935          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4936          break;
4937       case nir_intrinsic_global_atomic_xor:
4938          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4939          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4940          break;
4941       case nir_intrinsic_global_atomic_exchange:
4942          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4943          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4944          break;
4945       case nir_intrinsic_global_atomic_comp_swap:
4946          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4947          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4948          break;
4949       default:
4950          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4951    }
4952    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4953    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4954    flat->operands[0] = Operand(addr);
4955    flat->operands[1] = Operand(s1);
4956    flat->operands[2] = Operand(data);
4957    if (return_previous)
4958       flat->definitions[0] = Definition(dst);
4959    flat->glc = return_previous;
4960    flat->dlc = false; /* Not needed for atomics */
4961    flat->offset = 0;
4962    flat->disable_wqm = true;
4963    flat->barrier = barrier_buffer;
4964    ctx->program->needs_exact = true;
4965    ctx->block->instructions.emplace_back(std::move(flat));
4966 }
4967
4968 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4969    Builder bld(ctx->program, ctx->block);
4970    switch(instr->intrinsic) {
4971       case nir_intrinsic_group_memory_barrier:
4972       case nir_intrinsic_memory_barrier:
4973          bld.barrier(aco_opcode::p_memory_barrier_all);
4974          break;
4975       case nir_intrinsic_memory_barrier_buffer:
4976          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4977          break;
4978       case nir_intrinsic_memory_barrier_image:
4979          bld.barrier(aco_opcode::p_memory_barrier_image);
4980          break;
4981       case nir_intrinsic_memory_barrier_shared:
4982          bld.barrier(aco_opcode::p_memory_barrier_shared);
4983          break;
4984       default:
4985          unreachable("Unimplemented memory barrier intrinsic");
4986          break;
4987    }
4988 }
4989
4990 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4991 {
4992    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4993    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4994    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4995    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4996    Builder bld(ctx->program, ctx->block);
4997
4998    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4999    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5000    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5001 }
5002
5003 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5004 {
5005    unsigned writemask = nir_intrinsic_write_mask(instr);
5006    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5007    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5008    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5009    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5010
5011    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5012    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5013 }
5014
5015 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5016 {
5017    unsigned offset = nir_intrinsic_base(instr);
5018    Operand m = load_lds_size_m0(ctx);
5019    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5020    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5021
5022    unsigned num_operands = 3;
5023    aco_opcode op32, op64, op32_rtn, op64_rtn;
5024    switch(instr->intrinsic) {
5025       case nir_intrinsic_shared_atomic_add:
5026          op32 = aco_opcode::ds_add_u32;
5027          op64 = aco_opcode::ds_add_u64;
5028          op32_rtn = aco_opcode::ds_add_rtn_u32;
5029          op64_rtn = aco_opcode::ds_add_rtn_u64;
5030          break;
5031       case nir_intrinsic_shared_atomic_imin:
5032          op32 = aco_opcode::ds_min_i32;
5033          op64 = aco_opcode::ds_min_i64;
5034          op32_rtn = aco_opcode::ds_min_rtn_i32;
5035          op64_rtn = aco_opcode::ds_min_rtn_i64;
5036          break;
5037       case nir_intrinsic_shared_atomic_umin:
5038          op32 = aco_opcode::ds_min_u32;
5039          op64 = aco_opcode::ds_min_u64;
5040          op32_rtn = aco_opcode::ds_min_rtn_u32;
5041          op64_rtn = aco_opcode::ds_min_rtn_u64;
5042          break;
5043       case nir_intrinsic_shared_atomic_imax:
5044          op32 = aco_opcode::ds_max_i32;
5045          op64 = aco_opcode::ds_max_i64;
5046          op32_rtn = aco_opcode::ds_max_rtn_i32;
5047          op64_rtn = aco_opcode::ds_max_rtn_i64;
5048          break;
5049       case nir_intrinsic_shared_atomic_umax:
5050          op32 = aco_opcode::ds_max_u32;
5051          op64 = aco_opcode::ds_max_u64;
5052          op32_rtn = aco_opcode::ds_max_rtn_u32;
5053          op64_rtn = aco_opcode::ds_max_rtn_u64;
5054          break;
5055       case nir_intrinsic_shared_atomic_and:
5056          op32 = aco_opcode::ds_and_b32;
5057          op64 = aco_opcode::ds_and_b64;
5058          op32_rtn = aco_opcode::ds_and_rtn_b32;
5059          op64_rtn = aco_opcode::ds_and_rtn_b64;
5060          break;
5061       case nir_intrinsic_shared_atomic_or:
5062          op32 = aco_opcode::ds_or_b32;
5063          op64 = aco_opcode::ds_or_b64;
5064          op32_rtn = aco_opcode::ds_or_rtn_b32;
5065          op64_rtn = aco_opcode::ds_or_rtn_b64;
5066          break;
5067       case nir_intrinsic_shared_atomic_xor:
5068          op32 = aco_opcode::ds_xor_b32;
5069          op64 = aco_opcode::ds_xor_b64;
5070          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5071          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5072          break;
5073       case nir_intrinsic_shared_atomic_exchange:
5074          op32 = aco_opcode::ds_write_b32;
5075          op64 = aco_opcode::ds_write_b64;
5076          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5077          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5078          break;
5079       case nir_intrinsic_shared_atomic_comp_swap:
5080          op32 = aco_opcode::ds_cmpst_b32;
5081          op64 = aco_opcode::ds_cmpst_b64;
5082          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5083          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5084          num_operands = 4;
5085          break;
5086       default:
5087          unreachable("Unhandled shared atomic intrinsic");
5088    }
5089
5090    /* return the previous value if dest is ever used */
5091    bool return_previous = false;
5092    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5093       return_previous = true;
5094       break;
5095    }
5096    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5097       return_previous = true;
5098       break;
5099    }
5100
5101    aco_opcode op;
5102    if (data.size() == 1) {
5103       assert(instr->dest.ssa.bit_size == 32);
5104       op = return_previous ? op32_rtn : op32;
5105    } else {
5106       assert(instr->dest.ssa.bit_size == 64);
5107       op = return_previous ? op64_rtn : op64;
5108    }
5109
5110    if (offset > 65535) {
5111       Builder bld(ctx->program, ctx->block);
5112       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5113       offset = 0;
5114    }
5115
5116    aco_ptr<DS_instruction> ds;
5117    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5118    ds->operands[0] = Operand(address);
5119    ds->operands[1] = Operand(data);
5120    if (num_operands == 4)
5121       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5122    ds->operands[num_operands - 1] = m;
5123    ds->offset0 = offset;
5124    if (return_previous)
5125       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5126    ctx->block->instructions.emplace_back(std::move(ds));
5127 }
5128
5129 Temp get_scratch_resource(isel_context *ctx)
5130 {
5131    Builder bld(ctx->program, ctx->block);
5132    Temp scratch_addr = ctx->program->private_segment_buffer;
5133    if (ctx->stage != compute_cs)
5134       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5135
5136    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5137                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5138
5139    if (ctx->program->chip_class >= GFX10) {
5140       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5141                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5142                    S_008F0C_RESOURCE_LEVEL(1);
5143    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5144       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5145                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5146    }
5147
5148    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5149    if (ctx->program->chip_class <= GFX8)
5150       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5151
5152    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5153 }
5154
5155 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5156    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5157    Builder bld(ctx->program, ctx->block);
5158    Temp rsrc = get_scratch_resource(ctx);
5159    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5160    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5161
5162    aco_opcode op;
5163    switch (dst.size()) {
5164       case 1:
5165          op = aco_opcode::buffer_load_dword;
5166          break;
5167       case 2:
5168          op = aco_opcode::buffer_load_dwordx2;
5169          break;
5170       case 3:
5171          op = aco_opcode::buffer_load_dwordx3;
5172          break;
5173       case 4:
5174          op = aco_opcode::buffer_load_dwordx4;
5175          break;
5176       case 6:
5177       case 8: {
5178          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5179          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5180                                 bld.def(v4), offset, rsrc,
5181                                 ctx->program->scratch_offset, 0, true);
5182          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5183                                                   aco_opcode::buffer_load_dwordx4,
5184                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5185                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5186          emit_split_vector(ctx, lower, 2);
5187          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5188          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5189          if (dst.size() == 8) {
5190             emit_split_vector(ctx, upper, 2);
5191             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5192             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5193          } else {
5194             elems[2] = upper;
5195          }
5196
5197          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5198                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5199          for (unsigned i = 0; i < dst.size() / 2; i++)
5200             vec->operands[i] = Operand(elems[i]);
5201          vec->definitions[0] = Definition(dst);
5202          bld.insert(std::move(vec));
5203          ctx->allocated_vec.emplace(dst.id(), elems);
5204          return;
5205       }
5206       default:
5207          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5208    }
5209
5210    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5211    emit_split_vector(ctx, dst, instr->num_components);
5212 }
5213
5214 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5215    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5216    Builder bld(ctx->program, ctx->block);
5217    Temp rsrc = get_scratch_resource(ctx);
5218    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5219    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5220
5221    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5222    unsigned writemask = nir_intrinsic_write_mask(instr);
5223
5224    while (writemask) {
5225       int start, count;
5226       u_bit_scan_consecutive_range(&writemask, &start, &count);
5227       int num_bytes = count * elem_size_bytes;
5228
5229       if (num_bytes > 16) {
5230          assert(elem_size_bytes == 8);
5231          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5232          count = 2;
5233          num_bytes = 16;
5234       }
5235
5236       // TODO: check alignment of sub-dword stores
5237       // TODO: split 3 bytes. there is no store instruction for that
5238
5239       Temp write_data;
5240       if (count != instr->num_components) {
5241          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5242          for (int i = 0; i < count; i++) {
5243             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5244             vec->operands[i] = Operand(elem);
5245          }
5246          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5247          vec->definitions[0] = Definition(write_data);
5248          ctx->block->instructions.emplace_back(std::move(vec));
5249       } else {
5250          write_data = data;
5251       }
5252
5253       aco_opcode op;
5254       switch (num_bytes) {
5255          case 4:
5256             op = aco_opcode::buffer_store_dword;
5257             break;
5258          case 8:
5259             op = aco_opcode::buffer_store_dwordx2;
5260             break;
5261          case 12:
5262             op = aco_opcode::buffer_store_dwordx3;
5263             break;
5264          case 16:
5265             op = aco_opcode::buffer_store_dwordx4;
5266             break;
5267          default:
5268             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5269       }
5270
5271       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5272    }
5273 }
5274
5275 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5276    uint8_t log2_ps_iter_samples;
5277    if (ctx->program->info->ps.force_persample) {
5278       log2_ps_iter_samples =
5279          util_logbase2(ctx->options->key.fs.num_samples);
5280    } else {
5281       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5282    }
5283
5284    /* The bit pattern matches that used by fixed function fragment
5285     * processing. */
5286    static const unsigned ps_iter_masks[] = {
5287       0xffff, /* not used */
5288       0x5555,
5289       0x1111,
5290       0x0101,
5291       0x0001,
5292    };
5293    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5294
5295    Builder bld(ctx->program, ctx->block);
5296
5297    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5298                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5299    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5300    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5301    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5302    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5303 }
5304
5305 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5306 {
5307    Builder bld(ctx->program, ctx->block);
5308
5309    if (cluster_size == 1) {
5310       return src;
5311    } if (op == nir_op_iand && cluster_size == 4) {
5312       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5313       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5314       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5315                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5316    } else if (op == nir_op_ior && cluster_size == 4) {
5317       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5318       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5319                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5320    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5321       //subgroupAnd(val) -> (exec & ~val) == 0
5322       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5323       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5324       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5325    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5326       //subgroupOr(val) -> (val & exec) != 0
5327       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5328       return bool_to_vector_condition(ctx, tmp);
5329    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5330       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5331       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5332       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5333       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5334       return bool_to_vector_condition(ctx, tmp);
5335    } else {
5336       //subgroupClustered{And,Or,Xor}(val, n) ->
5337       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5338       //cluster_offset = ~(n - 1) & lane_id
5339       //cluster_mask = ((1 << n) - 1)
5340       //subgroupClusteredAnd():
5341       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5342       //subgroupClusteredOr():
5343       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5344       //subgroupClusteredXor():
5345       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5346       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5347       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5348
5349       Temp tmp;
5350       if (op == nir_op_iand)
5351          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5352       else
5353          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5354
5355       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5356
5357       if (ctx->program->chip_class <= GFX7)
5358          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5359       else if (ctx->program->wave_size == 64)
5360          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5361       else
5362          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5363       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5364       if (cluster_mask != 0xffffffff)
5365          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5366
5367       Definition cmp_def = Definition();
5368       if (op == nir_op_iand) {
5369          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5370       } else if (op == nir_op_ior) {
5371          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5372       } else if (op == nir_op_ixor) {
5373          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5374                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5375          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5376       }
5377       cmp_def.setHint(vcc);
5378       return cmp_def.getTemp();
5379    }
5380 }
5381
5382 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5383 {
5384    Builder bld(ctx->program, ctx->block);
5385
5386    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5387    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5388    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5389    Temp tmp;
5390    if (op == nir_op_iand)
5391       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5392    else
5393       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5394
5395    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5396    Temp lo = lohi.def(0).getTemp();
5397    Temp hi = lohi.def(1).getTemp();
5398    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5399
5400    Definition cmp_def = Definition();
5401    if (op == nir_op_iand)
5402       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5403    else if (op == nir_op_ior)
5404       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5405    else if (op == nir_op_ixor)
5406       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5407                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5408    cmp_def.setHint(vcc);
5409    return cmp_def.getTemp();
5410 }
5411
5412 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5413 {
5414    Builder bld(ctx->program, ctx->block);
5415
5416    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5417    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5418    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5419    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5420    if (op == nir_op_iand)
5421       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5422    else if (op == nir_op_ior)
5423       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5424    else if (op == nir_op_ixor)
5425       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5426
5427    assert(false);
5428    return Temp();
5429 }
5430
5431 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5432 {
5433    Builder bld(ctx->program, ctx->block);
5434    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5435    if (src.regClass().type() == RegType::vgpr) {
5436       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5437    } else if (src.regClass() == s1) {
5438       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5439    } else if (src.regClass() == s2) {
5440       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5441    } else {
5442       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5443       nir_print_instr(&instr->instr, stderr);
5444       fprintf(stderr, "\n");
5445    }
5446 }
5447
5448 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5449 {
5450    Builder bld(ctx->program, ctx->block);
5451    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5452    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5453    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5454
5455    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5456    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5457    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5458    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5459
5460    /* Build DD X/Y */
5461    if (ctx->program->chip_class >= GFX8) {
5462       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5463       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5464       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5465       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5466       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5467       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5468    } else {
5469       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5470       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5471       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5472       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5473       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5474       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5475       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5476       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5477       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5478       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5479    }
5480
5481    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5482    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5483    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5484    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5485    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5486    Temp wqm1 = bld.tmp(v1);
5487    emit_wqm(ctx, tmp1, wqm1, true);
5488    Temp wqm2 = bld.tmp(v1);
5489    emit_wqm(ctx, tmp2, wqm2, true);
5490    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5491    return;
5492 }
5493
5494 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5495 {
5496    Builder bld(ctx->program, ctx->block);
5497    switch(instr->intrinsic) {
5498    case nir_intrinsic_load_barycentric_sample:
5499    case nir_intrinsic_load_barycentric_pixel:
5500    case nir_intrinsic_load_barycentric_centroid: {
5501       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5502       Temp bary = Temp(0, s2);
5503       switch (mode) {
5504       case INTERP_MODE_SMOOTH:
5505       case INTERP_MODE_NONE:
5506          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5507             bary = get_arg(ctx, ctx->args->ac.persp_center);
5508          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5509             bary = ctx->persp_centroid;
5510          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5511             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5512          break;
5513       case INTERP_MODE_NOPERSPECTIVE:
5514          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5515             bary = get_arg(ctx, ctx->args->ac.linear_center);
5516          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5517             bary = ctx->linear_centroid;
5518          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5519             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5520          break;
5521       default:
5522          break;
5523       }
5524       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5525       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5526       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5527       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5528                  Operand(p1), Operand(p2));
5529       emit_split_vector(ctx, dst, 2);
5530       break;
5531    }
5532    case nir_intrinsic_load_barycentric_at_sample: {
5533       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5534       switch (ctx->options->key.fs.num_samples) {
5535          case 2: sample_pos_offset += 1 << 3; break;
5536          case 4: sample_pos_offset += 3 << 3; break;
5537          case 8: sample_pos_offset += 7 << 3; break;
5538          default: break;
5539       }
5540       Temp sample_pos;
5541       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5542       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5543       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5544       if (addr.type() == RegType::sgpr) {
5545          Operand offset;
5546          if (const_addr) {
5547             sample_pos_offset += const_addr->u32 << 3;
5548             offset = Operand(sample_pos_offset);
5549          } else if (ctx->options->chip_class >= GFX9) {
5550             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5551          } else {
5552             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5553             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5554          }
5555          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5556
5557       } else if (ctx->options->chip_class >= GFX9) {
5558          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5559          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5560       } else if (ctx->options->chip_class >= GFX7) {
5561          /* addr += private_segment_buffer + sample_pos_offset */
5562          Temp tmp0 = bld.tmp(s1);
5563          Temp tmp1 = bld.tmp(s1);
5564          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5565          Definition scc_tmp = bld.def(s1, scc);
5566          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5567          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5568          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5569          Temp pck0 = bld.tmp(v1);
5570          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5571          tmp1 = as_vgpr(ctx, tmp1);
5572          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5573          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5574
5575          /* sample_pos = flat_load_dwordx2 addr */
5576          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5577       } else {
5578          assert(ctx->options->chip_class == GFX6);
5579
5580          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5581                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5582          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(rsrc_conf));
5583
5584          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5585          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
5586
5587          sample_pos = bld.tmp(v2);
5588
5589          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
5590          load->definitions[0] = Definition(sample_pos);
5591          load->operands[0] = Operand(addr);
5592          load->operands[1] = Operand(rsrc);
5593          load->operands[2] = Operand(0u);
5594          load->offset = sample_pos_offset;
5595          load->offen = 0;
5596          load->addr64 = true;
5597          load->glc = false;
5598          load->dlc = false;
5599          load->disable_wqm = false;
5600          load->barrier = barrier_none;
5601          load->can_reorder = true;
5602          ctx->block->instructions.emplace_back(std::move(load));
5603       }
5604
5605       /* sample_pos -= 0.5 */
5606       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5607       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5608       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5609       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5610       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5611
5612       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5613       break;
5614    }
5615    case nir_intrinsic_load_barycentric_at_offset: {
5616       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5617       RegClass rc = RegClass(offset.type(), 1);
5618       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5619       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5620       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5621       break;
5622    }
5623    case nir_intrinsic_load_front_face: {
5624       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5625                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5626       break;
5627    }
5628    case nir_intrinsic_load_view_index:
5629    case nir_intrinsic_load_layer_id: {
5630       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5631          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5632          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5633          break;
5634       }
5635
5636       unsigned idx = nir_intrinsic_base(instr);
5637       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5638                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5639       break;
5640    }
5641    case nir_intrinsic_load_frag_coord: {
5642       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5643       break;
5644    }
5645    case nir_intrinsic_load_sample_pos: {
5646       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5647       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5648       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5649                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5650                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5651       break;
5652    }
5653    case nir_intrinsic_load_interpolated_input:
5654       visit_load_interpolated_input(ctx, instr);
5655       break;
5656    case nir_intrinsic_store_output:
5657       visit_store_output(ctx, instr);
5658       break;
5659    case nir_intrinsic_load_input:
5660       visit_load_input(ctx, instr);
5661       break;
5662    case nir_intrinsic_load_ubo:
5663       visit_load_ubo(ctx, instr);
5664       break;
5665    case nir_intrinsic_load_push_constant:
5666       visit_load_push_constant(ctx, instr);
5667       break;
5668    case nir_intrinsic_load_constant:
5669       visit_load_constant(ctx, instr);
5670       break;
5671    case nir_intrinsic_vulkan_resource_index:
5672       visit_load_resource(ctx, instr);
5673       break;
5674    case nir_intrinsic_discard:
5675       visit_discard(ctx, instr);
5676       break;
5677    case nir_intrinsic_discard_if:
5678       visit_discard_if(ctx, instr);
5679       break;
5680    case nir_intrinsic_load_shared:
5681       visit_load_shared(ctx, instr);
5682       break;
5683    case nir_intrinsic_store_shared:
5684       visit_store_shared(ctx, instr);
5685       break;
5686    case nir_intrinsic_shared_atomic_add:
5687    case nir_intrinsic_shared_atomic_imin:
5688    case nir_intrinsic_shared_atomic_umin:
5689    case nir_intrinsic_shared_atomic_imax:
5690    case nir_intrinsic_shared_atomic_umax:
5691    case nir_intrinsic_shared_atomic_and:
5692    case nir_intrinsic_shared_atomic_or:
5693    case nir_intrinsic_shared_atomic_xor:
5694    case nir_intrinsic_shared_atomic_exchange:
5695    case nir_intrinsic_shared_atomic_comp_swap:
5696       visit_shared_atomic(ctx, instr);
5697       break;
5698    case nir_intrinsic_image_deref_load:
5699       visit_image_load(ctx, instr);
5700       break;
5701    case nir_intrinsic_image_deref_store:
5702       visit_image_store(ctx, instr);
5703       break;
5704    case nir_intrinsic_image_deref_atomic_add:
5705    case nir_intrinsic_image_deref_atomic_umin:
5706    case nir_intrinsic_image_deref_atomic_imin:
5707    case nir_intrinsic_image_deref_atomic_umax:
5708    case nir_intrinsic_image_deref_atomic_imax:
5709    case nir_intrinsic_image_deref_atomic_and:
5710    case nir_intrinsic_image_deref_atomic_or:
5711    case nir_intrinsic_image_deref_atomic_xor:
5712    case nir_intrinsic_image_deref_atomic_exchange:
5713    case nir_intrinsic_image_deref_atomic_comp_swap:
5714       visit_image_atomic(ctx, instr);
5715       break;
5716    case nir_intrinsic_image_deref_size:
5717       visit_image_size(ctx, instr);
5718       break;
5719    case nir_intrinsic_load_ssbo:
5720       visit_load_ssbo(ctx, instr);
5721       break;
5722    case nir_intrinsic_store_ssbo:
5723       visit_store_ssbo(ctx, instr);
5724       break;
5725    case nir_intrinsic_load_global:
5726       visit_load_global(ctx, instr);
5727       break;
5728    case nir_intrinsic_store_global:
5729       visit_store_global(ctx, instr);
5730       break;
5731    case nir_intrinsic_global_atomic_add:
5732    case nir_intrinsic_global_atomic_imin:
5733    case nir_intrinsic_global_atomic_umin:
5734    case nir_intrinsic_global_atomic_imax:
5735    case nir_intrinsic_global_atomic_umax:
5736    case nir_intrinsic_global_atomic_and:
5737    case nir_intrinsic_global_atomic_or:
5738    case nir_intrinsic_global_atomic_xor:
5739    case nir_intrinsic_global_atomic_exchange:
5740    case nir_intrinsic_global_atomic_comp_swap:
5741       visit_global_atomic(ctx, instr);
5742       break;
5743    case nir_intrinsic_ssbo_atomic_add:
5744    case nir_intrinsic_ssbo_atomic_imin:
5745    case nir_intrinsic_ssbo_atomic_umin:
5746    case nir_intrinsic_ssbo_atomic_imax:
5747    case nir_intrinsic_ssbo_atomic_umax:
5748    case nir_intrinsic_ssbo_atomic_and:
5749    case nir_intrinsic_ssbo_atomic_or:
5750    case nir_intrinsic_ssbo_atomic_xor:
5751    case nir_intrinsic_ssbo_atomic_exchange:
5752    case nir_intrinsic_ssbo_atomic_comp_swap:
5753       visit_atomic_ssbo(ctx, instr);
5754       break;
5755    case nir_intrinsic_load_scratch:
5756       visit_load_scratch(ctx, instr);
5757       break;
5758    case nir_intrinsic_store_scratch:
5759       visit_store_scratch(ctx, instr);
5760       break;
5761    case nir_intrinsic_get_buffer_size:
5762       visit_get_buffer_size(ctx, instr);
5763       break;
5764    case nir_intrinsic_control_barrier: {
5765       unsigned* bsize = ctx->program->info->cs.block_size;
5766       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5767       if (workgroup_size > ctx->program->wave_size)
5768          bld.sopp(aco_opcode::s_barrier);
5769       break;
5770    }
5771    case nir_intrinsic_group_memory_barrier:
5772    case nir_intrinsic_memory_barrier:
5773    case nir_intrinsic_memory_barrier_buffer:
5774    case nir_intrinsic_memory_barrier_image:
5775    case nir_intrinsic_memory_barrier_shared:
5776       emit_memory_barrier(ctx, instr);
5777       break;
5778    case nir_intrinsic_memory_barrier_tcs_patch:
5779       break;
5780    case nir_intrinsic_load_num_work_groups: {
5781       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5782       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5783       emit_split_vector(ctx, dst, 3);
5784       break;
5785    }
5786    case nir_intrinsic_load_local_invocation_id: {
5787       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5788       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5789       emit_split_vector(ctx, dst, 3);
5790       break;
5791    }
5792    case nir_intrinsic_load_work_group_id: {
5793       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5794       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5795       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5796                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5797                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5798                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5799       emit_split_vector(ctx, dst, 3);
5800       break;
5801    }
5802    case nir_intrinsic_load_local_invocation_index: {
5803       Temp id = emit_mbcnt(ctx, bld.def(v1));
5804
5805       /* The tg_size bits [6:11] contain the subgroup id,
5806        * we need this multiplied by the wave size, and then OR the thread id to it.
5807        */
5808       if (ctx->program->wave_size == 64) {
5809          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5810          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5811                                 get_arg(ctx, ctx->args->ac.tg_size));
5812          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5813       } else {
5814          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5815          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5816                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5817          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5818       }
5819       break;
5820    }
5821    case nir_intrinsic_load_subgroup_id: {
5822       if (ctx->stage == compute_cs) {
5823          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5824                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5825       } else {
5826          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5827       }
5828       break;
5829    }
5830    case nir_intrinsic_load_subgroup_invocation: {
5831       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5832       break;
5833    }
5834    case nir_intrinsic_load_num_subgroups: {
5835       if (ctx->stage == compute_cs)
5836          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5837                   get_arg(ctx, ctx->args->ac.tg_size));
5838       else
5839          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5840       break;
5841    }
5842    case nir_intrinsic_ballot: {
5843       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5844       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5845       Definition tmp = bld.def(dst.regClass());
5846       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5847       if (instr->src[0].ssa->bit_size == 1) {
5848          assert(src.regClass() == bld.lm);
5849          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5850       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5851          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5852       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5853          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5854       } else {
5855          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5856          nir_print_instr(&instr->instr, stderr);
5857          fprintf(stderr, "\n");
5858       }
5859       if (dst.size() != bld.lm.size()) {
5860          /* Wave32 with ballot size set to 64 */
5861          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5862       }
5863       emit_wqm(ctx, tmp.getTemp(), dst);
5864       break;
5865    }
5866    case nir_intrinsic_shuffle:
5867    case nir_intrinsic_read_invocation: {
5868       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5869       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5870          emit_uniform_subgroup(ctx, instr, src);
5871       } else {
5872          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5873          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5874             tid = bld.as_uniform(tid);
5875          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5876          if (src.regClass() == v1) {
5877             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5878          } else if (src.regClass() == v2) {
5879             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5880             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5881             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5882             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5883             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5884             emit_split_vector(ctx, dst, 2);
5885          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5886             assert(src.regClass() == bld.lm);
5887             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5888             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5889          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5890             assert(src.regClass() == bld.lm);
5891             Temp tmp;
5892             if (ctx->program->chip_class <= GFX7)
5893                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
5894             else if (ctx->program->wave_size == 64)
5895                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5896             else
5897                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5898             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5899             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5900             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5901          } else {
5902             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5903             nir_print_instr(&instr->instr, stderr);
5904             fprintf(stderr, "\n");
5905          }
5906       }
5907       break;
5908    }
5909    case nir_intrinsic_load_sample_id: {
5910       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5911                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5912       break;
5913    }
5914    case nir_intrinsic_load_sample_mask_in: {
5915       visit_load_sample_mask_in(ctx, instr);
5916       break;
5917    }
5918    case nir_intrinsic_read_first_invocation: {
5919       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5920       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5921       if (src.regClass() == v1) {
5922          emit_wqm(ctx,
5923                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5924                   dst);
5925       } else if (src.regClass() == v2) {
5926          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5927          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5928          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5929          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5930          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5931          emit_split_vector(ctx, dst, 2);
5932       } else if (instr->dest.ssa.bit_size == 1) {
5933          assert(src.regClass() == bld.lm);
5934          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5935                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5936          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5937       } else if (src.regClass() == s1) {
5938          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5939       } else if (src.regClass() == s2) {
5940          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5941       } else {
5942          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5943          nir_print_instr(&instr->instr, stderr);
5944          fprintf(stderr, "\n");
5945       }
5946       break;
5947    }
5948    case nir_intrinsic_vote_all: {
5949       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5950       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5951       assert(src.regClass() == bld.lm);
5952       assert(dst.regClass() == bld.lm);
5953
5954       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5955       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5956       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
5957       break;
5958    }
5959    case nir_intrinsic_vote_any: {
5960       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5961       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5962       assert(src.regClass() == bld.lm);
5963       assert(dst.regClass() == bld.lm);
5964
5965       Temp tmp = bool_to_scalar_condition(ctx, src);
5966       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5967       break;
5968    }
5969    case nir_intrinsic_reduce:
5970    case nir_intrinsic_inclusive_scan:
5971    case nir_intrinsic_exclusive_scan: {
5972       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5973       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5974       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5975       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5976          nir_intrinsic_cluster_size(instr) : 0;
5977       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5978
5979       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5980          emit_uniform_subgroup(ctx, instr, src);
5981       } else if (instr->dest.ssa.bit_size == 1) {
5982          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5983             op = nir_op_iand;
5984          else if (op == nir_op_iadd)
5985             op = nir_op_ixor;
5986          else if (op == nir_op_umax || op == nir_op_imax)
5987             op = nir_op_ior;
5988          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5989
5990          switch (instr->intrinsic) {
5991          case nir_intrinsic_reduce:
5992             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5993             break;
5994          case nir_intrinsic_exclusive_scan:
5995             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5996             break;
5997          case nir_intrinsic_inclusive_scan:
5998             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5999             break;
6000          default:
6001             assert(false);
6002          }
6003       } else if (cluster_size == 1) {
6004          bld.copy(Definition(dst), src);
6005       } else {
6006          src = as_vgpr(ctx, src);
6007
6008          ReduceOp reduce_op;
6009          switch (op) {
6010          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6011             CASE(iadd)
6012             CASE(imul)
6013             CASE(fadd)
6014             CASE(fmul)
6015             CASE(imin)
6016             CASE(umin)
6017             CASE(fmin)
6018             CASE(imax)
6019             CASE(umax)
6020             CASE(fmax)
6021             CASE(iand)
6022             CASE(ior)
6023             CASE(ixor)
6024             default:
6025                unreachable("unknown reduction op");
6026          #undef CASE
6027          }
6028
6029          aco_opcode aco_op;
6030          switch (instr->intrinsic) {
6031             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6032             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6033             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6034             default:
6035                unreachable("unknown reduce intrinsic");
6036          }
6037
6038          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6039          reduce->operands[0] = Operand(src);
6040          // filled in by aco_reduce_assign.cpp, used internally as part of the
6041          // reduce sequence
6042          assert(dst.size() == 1 || dst.size() == 2);
6043          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6044          reduce->operands[2] = Operand(v1.as_linear());
6045
6046          Temp tmp_dst = bld.tmp(dst.regClass());
6047          reduce->definitions[0] = Definition(tmp_dst);
6048          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6049          reduce->definitions[2] = Definition();
6050          reduce->definitions[3] = Definition(scc, s1);
6051          reduce->definitions[4] = Definition();
6052          reduce->reduce_op = reduce_op;
6053          reduce->cluster_size = cluster_size;
6054          ctx->block->instructions.emplace_back(std::move(reduce));
6055
6056          emit_wqm(ctx, tmp_dst, dst);
6057       }
6058       break;
6059    }
6060    case nir_intrinsic_quad_broadcast: {
6061       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6062       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6063          emit_uniform_subgroup(ctx, instr, src);
6064       } else {
6065          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6066          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6067          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6068
6069          if (instr->dest.ssa.bit_size == 1) {
6070             assert(src.regClass() == bld.lm);
6071             assert(dst.regClass() == bld.lm);
6072             uint32_t half_mask = 0x11111111u << lane;
6073             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6074             Temp tmp = bld.tmp(bld.lm);
6075             bld.sop1(Builder::s_wqm, Definition(tmp),
6076                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6077                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6078             emit_wqm(ctx, tmp, dst);
6079          } else if (instr->dest.ssa.bit_size == 32) {
6080             if (ctx->program->chip_class >= GFX8)
6081                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6082             else
6083                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6084          } else if (instr->dest.ssa.bit_size == 64) {
6085             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6086             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6087             if (ctx->program->chip_class >= GFX8) {
6088                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6089                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6090             } else {
6091                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6092                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6093             }
6094             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6095             emit_split_vector(ctx, dst, 2);
6096          } else {
6097             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6098             nir_print_instr(&instr->instr, stderr);
6099             fprintf(stderr, "\n");
6100          }
6101       }
6102       break;
6103    }
6104    case nir_intrinsic_quad_swap_horizontal:
6105    case nir_intrinsic_quad_swap_vertical:
6106    case nir_intrinsic_quad_swap_diagonal:
6107    case nir_intrinsic_quad_swizzle_amd: {
6108       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6109       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6110          emit_uniform_subgroup(ctx, instr, src);
6111          break;
6112       }
6113       uint16_t dpp_ctrl = 0;
6114       switch (instr->intrinsic) {
6115       case nir_intrinsic_quad_swap_horizontal:
6116          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6117          break;
6118       case nir_intrinsic_quad_swap_vertical:
6119          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6120          break;
6121       case nir_intrinsic_quad_swap_diagonal:
6122          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6123          break;
6124       case nir_intrinsic_quad_swizzle_amd:
6125          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6126          break;
6127       default:
6128          break;
6129       }
6130       if (ctx->program->chip_class < GFX8)
6131          dpp_ctrl |= (1 << 15);
6132
6133       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6134       if (instr->dest.ssa.bit_size == 1) {
6135          assert(src.regClass() == bld.lm);
6136          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6137          if (ctx->program->chip_class >= GFX8)
6138             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6139          else
6140             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6141          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6142          emit_wqm(ctx, tmp, dst);
6143       } else if (instr->dest.ssa.bit_size == 32) {
6144          Temp tmp;
6145          if (ctx->program->chip_class >= GFX8)
6146             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6147          else
6148             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6149          emit_wqm(ctx, tmp, dst);
6150       } else if (instr->dest.ssa.bit_size == 64) {
6151          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6152          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6153          if (ctx->program->chip_class >= GFX8) {
6154             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6155             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6156          } else {
6157             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6158             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6159          }
6160          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6161          emit_split_vector(ctx, dst, 2);
6162       } else {
6163          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6164          nir_print_instr(&instr->instr, stderr);
6165          fprintf(stderr, "\n");
6166       }
6167       break;
6168    }
6169    case nir_intrinsic_masked_swizzle_amd: {
6170       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6171       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6172          emit_uniform_subgroup(ctx, instr, src);
6173          break;
6174       }
6175       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6176       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6177       if (dst.regClass() == v1) {
6178          emit_wqm(ctx,
6179                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6180                   dst);
6181       } else if (dst.regClass() == v2) {
6182          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6183          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6184          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6185          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6186          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6187          emit_split_vector(ctx, dst, 2);
6188       } else {
6189          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6190          nir_print_instr(&instr->instr, stderr);
6191          fprintf(stderr, "\n");
6192       }
6193       break;
6194    }
6195    case nir_intrinsic_write_invocation_amd: {
6196       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6197       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6198       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6199       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6200       if (dst.regClass() == v1) {
6201          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6202          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6203       } else if (dst.regClass() == v2) {
6204          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6205          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6206          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6207          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6208          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6209          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6210          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6211          emit_split_vector(ctx, dst, 2);
6212       } else {
6213          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6214          nir_print_instr(&instr->instr, stderr);
6215          fprintf(stderr, "\n");
6216       }
6217       break;
6218    }
6219    case nir_intrinsic_mbcnt_amd: {
6220       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6221       RegClass rc = RegClass(src.type(), 1);
6222       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6223       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6224       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6225       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6226       emit_wqm(ctx, wqm_tmp, dst);
6227       break;
6228    }
6229    case nir_intrinsic_load_helper_invocation: {
6230       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6231       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6232       ctx->block->kind |= block_kind_needs_lowering;
6233       ctx->program->needs_exact = true;
6234       break;
6235    }
6236    case nir_intrinsic_is_helper_invocation: {
6237       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6238       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6239       ctx->block->kind |= block_kind_needs_lowering;
6240       ctx->program->needs_exact = true;
6241       break;
6242    }
6243    case nir_intrinsic_demote:
6244       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6245
6246       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6247          ctx->cf_info.exec_potentially_empty = true;
6248       ctx->block->kind |= block_kind_uses_demote;
6249       ctx->program->needs_exact = true;
6250       break;
6251    case nir_intrinsic_demote_if: {
6252       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6253       assert(src.regClass() == bld.lm);
6254       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6255       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6256
6257       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6258          ctx->cf_info.exec_potentially_empty = true;
6259       ctx->block->kind |= block_kind_uses_demote;
6260       ctx->program->needs_exact = true;
6261       break;
6262    }
6263    case nir_intrinsic_first_invocation: {
6264       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6265                get_ssa_temp(ctx, &instr->dest.ssa));
6266       break;
6267    }
6268    case nir_intrinsic_shader_clock:
6269       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6270       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6271       break;
6272    case nir_intrinsic_load_vertex_id_zero_base: {
6273       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6274       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6275       break;
6276    }
6277    case nir_intrinsic_load_first_vertex: {
6278       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6279       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6280       break;
6281    }
6282    case nir_intrinsic_load_base_instance: {
6283       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6284       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6285       break;
6286    }
6287    case nir_intrinsic_load_instance_id: {
6288       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6289       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6290       break;
6291    }
6292    case nir_intrinsic_load_draw_id: {
6293       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6294       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6295       break;
6296    }
6297    default:
6298       fprintf(stderr, "Unimplemented intrinsic instr: ");
6299       nir_print_instr(&instr->instr, stderr);
6300       fprintf(stderr, "\n");
6301       abort();
6302
6303       break;
6304    }
6305 }
6306
6307
6308 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6309                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6310                     enum glsl_base_type *stype)
6311 {
6312    nir_deref_instr *texture_deref_instr = NULL;
6313    nir_deref_instr *sampler_deref_instr = NULL;
6314    int plane = -1;
6315
6316    for (unsigned i = 0; i < instr->num_srcs; i++) {
6317       switch (instr->src[i].src_type) {
6318       case nir_tex_src_texture_deref:
6319          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6320          break;
6321       case nir_tex_src_sampler_deref:
6322          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6323          break;
6324       case nir_tex_src_plane:
6325          plane = nir_src_as_int(instr->src[i].src);
6326          break;
6327       default:
6328          break;
6329       }
6330    }
6331
6332    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6333
6334    if (!sampler_deref_instr)
6335       sampler_deref_instr = texture_deref_instr;
6336
6337    if (plane >= 0) {
6338       assert(instr->op != nir_texop_txf_ms &&
6339              instr->op != nir_texop_samples_identical);
6340       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6341       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6342    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6343       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6344    } else {
6345       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6346    }
6347    if (samp_ptr) {
6348       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6349
6350       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6351          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6352          Builder bld(ctx->program, ctx->block);
6353
6354          /* to avoid unnecessary moves, we split and recombine sampler and image */
6355          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6356                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6357          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6358          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6359                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6360                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6361          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6362                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6363
6364          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6365          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6366                                img[0], img[1], img[2], img[3],
6367                                img[4], img[5], img[6], img[7]);
6368          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6369                                 samp[0], samp[1], samp[2], samp[3]);
6370       }
6371    }
6372    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6373                      instr->op == nir_texop_samples_identical))
6374       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6375 }
6376
6377 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6378                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6379 {
6380    Builder bld(ctx->program, ctx->block);
6381
6382    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6383    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6384    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6385
6386    Operand neg_one(0xbf800000u);
6387    Operand one(0x3f800000u);
6388    Operand two(0x40000000u);
6389    Operand four(0x40800000u);
6390
6391    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6392    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6393    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6394
6395    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6396    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6397    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6398    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6399
6400    // select sc
6401    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6402    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6403                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6404                        one, is_ma_y);
6405    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6406
6407    // select tc
6408    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6409    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6410    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6411
6412    // select ma
6413    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6414                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6415                   deriv_z, is_ma_z);
6416    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6417    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6418 }
6419
6420 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6421 {
6422    Builder bld(ctx->program, ctx->block);
6423    Temp coord_args[4], ma, tc, sc, id;
6424    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6425       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6426
6427    if (is_array) {
6428       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6429
6430       // see comment in ac_prepare_cube_coords()
6431       if (ctx->options->chip_class <= GFX8)
6432          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6433    }
6434
6435    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6436
6437    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6438    vop3a->operands[0] = Operand(ma);
6439    vop3a->abs[0] = true;
6440    Temp invma = bld.tmp(v1);
6441    vop3a->definitions[0] = Definition(invma);
6442    ctx->block->instructions.emplace_back(std::move(vop3a));
6443
6444    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6445    if (!is_deriv)
6446       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6447
6448    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6449    if (!is_deriv)
6450       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6451
6452    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6453
6454    if (is_deriv) {
6455       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6456       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6457
6458       for (unsigned i = 0; i < 2; i++) {
6459          // see comment in ac_prepare_cube_coords()
6460          Temp deriv_ma;
6461          Temp deriv_sc, deriv_tc;
6462          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6463                            &deriv_ma, &deriv_sc, &deriv_tc);
6464
6465          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6466
6467          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6468                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6469                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6470          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6471                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6472                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6473          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6474       }
6475
6476       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6477       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6478    }
6479
6480    if (is_array)
6481       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6482    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6483
6484 }
6485
6486 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6487 {
6488    Temp coord_vec[3];
6489    for (unsigned i = 0; i < coords.size(); i++)
6490       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6491
6492    Builder bld(ctx->program, ctx->block);
6493    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6494
6495    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6496    for (unsigned i = 0; i < coords.size(); i++)
6497       vec->operands[i] = Operand(coord_vec[i]);
6498    Temp res = bld.tmp(RegType::vgpr, coords.size());
6499    vec->definitions[0] = Definition(res);
6500    ctx->block->instructions.emplace_back(std::move(vec));
6501    return res;
6502 }
6503
6504 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6505 {
6506    if (vec->parent_instr->type != nir_instr_type_alu)
6507       return;
6508    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6509    if (vec_instr->op != nir_op_vec(vec->num_components))
6510       return;
6511
6512    for (unsigned i = 0; i < vec->num_components; i++) {
6513       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6514               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6515    }
6516 }
6517
6518 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6519 {
6520    Builder bld(ctx->program, ctx->block);
6521    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6522         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6523    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6524         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6525    nir_const_value *sample_index_cv = NULL;
6526    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6527    enum glsl_base_type stype;
6528    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6529
6530    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6531                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6532    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6533                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6534
6535    for (unsigned i = 0; i < instr->num_srcs; i++) {
6536       switch (instr->src[i].src_type) {
6537       case nir_tex_src_coord:
6538          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6539          break;
6540       case nir_tex_src_bias:
6541          if (instr->op == nir_texop_txb) {
6542             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6543             has_bias = true;
6544          }
6545          break;
6546       case nir_tex_src_lod: {
6547          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6548
6549          if (val && val->f32 <= 0.0) {
6550             level_zero = true;
6551          } else {
6552             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6553             has_lod = true;
6554          }
6555          break;
6556       }
6557       case nir_tex_src_comparator:
6558          if (instr->is_shadow) {
6559             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6560             has_compare = true;
6561          }
6562          break;
6563       case nir_tex_src_offset:
6564          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6565          get_const_vec(instr->src[i].src.ssa, const_offset);
6566          has_offset = true;
6567          break;
6568       case nir_tex_src_ddx:
6569          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6570          has_ddx = true;
6571          break;
6572       case nir_tex_src_ddy:
6573          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6574          has_ddy = true;
6575          break;
6576       case nir_tex_src_ms_index:
6577          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6578          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6579          has_sample_index = true;
6580          break;
6581       case nir_tex_src_texture_offset:
6582       case nir_tex_src_sampler_offset:
6583       default:
6584          break;
6585       }
6586    }
6587 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6588    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6589       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6590
6591    if (instr->op == nir_texop_texture_samples) {
6592       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6593
6594       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6595       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6596       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6597       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6598
6599       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6600                samples, Operand(1u), bld.scc(is_msaa));
6601       return;
6602    }
6603
6604    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6605       aco_ptr<Instruction> tmp_instr;
6606       Temp acc, pack = Temp();
6607
6608       uint32_t pack_const = 0;
6609       for (unsigned i = 0; i < offset.size(); i++) {
6610          if (!const_offset[i])
6611             continue;
6612          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6613       }
6614
6615       if (offset.type() == RegType::sgpr) {
6616          for (unsigned i = 0; i < offset.size(); i++) {
6617             if (const_offset[i])
6618                continue;
6619
6620             acc = emit_extract_vector(ctx, offset, i, s1);
6621             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6622
6623             if (i) {
6624                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6625             }
6626
6627             if (pack == Temp()) {
6628                pack = acc;
6629             } else {
6630                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6631             }
6632          }
6633
6634          if (pack_const && pack != Temp())
6635             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6636       } else {
6637          for (unsigned i = 0; i < offset.size(); i++) {
6638             if (const_offset[i])
6639                continue;
6640
6641             acc = emit_extract_vector(ctx, offset, i, v1);
6642             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6643
6644             if (i) {
6645                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6646             }
6647
6648             if (pack == Temp()) {
6649                pack = acc;
6650             } else {
6651                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6652             }
6653          }
6654
6655          if (pack_const && pack != Temp())
6656             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6657       }
6658       if (pack_const && pack == Temp())
6659          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6660       else if (pack == Temp())
6661          has_offset = false;
6662       else
6663          offset = pack;
6664    }
6665
6666    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6667       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6668
6669    /* pack derivatives */
6670    if (has_ddx || has_ddy) {
6671       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6672          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6673                              ddx, Operand(0u), ddy, Operand(0u));
6674       } else {
6675          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6676       }
6677       has_derivs = true;
6678    }
6679
6680    if (instr->coord_components > 1 &&
6681        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6682        instr->is_array &&
6683        instr->op != nir_texop_txf)
6684       coords = apply_round_slice(ctx, coords, 1);
6685
6686    if (instr->coord_components > 2 &&
6687       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6688        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6689        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6690        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6691        instr->is_array &&
6692        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6693       coords = apply_round_slice(ctx, coords, 2);
6694
6695    if (ctx->options->chip_class == GFX9 &&
6696        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6697        instr->op != nir_texop_lod && instr->coord_components) {
6698       assert(coords.size() > 0 && coords.size() < 3);
6699
6700       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6701       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6702       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6703       if (coords.size() > 1)
6704          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6705       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6706       vec->definitions[0] = Definition(coords);
6707       ctx->block->instructions.emplace_back(std::move(vec));
6708    }
6709
6710    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6711
6712    if (instr->op == nir_texop_samples_identical)
6713       resource = fmask_ptr;
6714
6715    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6716              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6717             instr->op != nir_texop_txs) {
6718       assert(has_sample_index);
6719       Operand op(sample_index);
6720       if (sample_index_cv)
6721          op = Operand(sample_index_cv->u32);
6722       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6723    }
6724
6725    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6726       Temp split_coords[coords.size()];
6727       emit_split_vector(ctx, coords, coords.size());
6728       for (unsigned i = 0; i < coords.size(); i++)
6729          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6730
6731       unsigned i = 0;
6732       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6733          Temp off = emit_extract_vector(ctx, offset, i, v1);
6734          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6735       }
6736
6737       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6738       for (unsigned i = 0; i < coords.size(); i++)
6739          vec->operands[i] = Operand(split_coords[i]);
6740       coords = bld.tmp(coords.regClass());
6741       vec->definitions[0] = Definition(coords);
6742       ctx->block->instructions.emplace_back(std::move(vec));
6743
6744       has_offset = false;
6745    }
6746
6747    /* Build tex instruction */
6748    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6749    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6750                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6751                   : 0;
6752    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6753    Temp tmp_dst = dst;
6754
6755    /* gather4 selects the component by dmask and always returns vec4 */
6756    if (instr->op == nir_texop_tg4) {
6757       assert(instr->dest.ssa.num_components == 4);
6758       if (instr->is_shadow)
6759          dmask = 1;
6760       else
6761          dmask = 1 << instr->component;
6762       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6763          tmp_dst = bld.tmp(v4);
6764    } else if (instr->op == nir_texop_samples_identical) {
6765       tmp_dst = bld.tmp(v1);
6766    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6767       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6768    }
6769
6770    aco_ptr<MIMG_instruction> tex;
6771    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6772       if (!has_lod)
6773          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6774
6775       bool div_by_6 = instr->op == nir_texop_txs &&
6776                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6777                       instr->is_array &&
6778                       (dmask & (1 << 2));
6779       if (tmp_dst.id() == dst.id() && div_by_6)
6780          tmp_dst = bld.tmp(tmp_dst.regClass());
6781
6782       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6783       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6784       tex->operands[1] = Operand(resource);
6785       if (ctx->options->chip_class == GFX9 &&
6786           instr->op == nir_texop_txs &&
6787           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6788           instr->is_array) {
6789          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6790       } else if (instr->op == nir_texop_query_levels) {
6791          tex->dmask = 1 << 3;
6792       } else {
6793          tex->dmask = dmask;
6794       }
6795       tex->da = da;
6796       tex->definitions[0] = Definition(tmp_dst);
6797       tex->dim = dim;
6798       tex->can_reorder = true;
6799       ctx->block->instructions.emplace_back(std::move(tex));
6800
6801       if (div_by_6) {
6802          /* divide 3rd value by 6 by multiplying with magic number */
6803          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6804          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6805          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6806          assert(instr->dest.ssa.num_components == 3);
6807          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6808          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6809                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6810                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6811                               by_6);
6812
6813       }
6814
6815       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6816       return;
6817    }
6818
6819    Temp tg4_compare_cube_wa64 = Temp();
6820
6821    if (tg4_integer_workarounds) {
6822       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6823       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6824       tex->operands[1] = Operand(resource);
6825       tex->dim = dim;
6826       tex->dmask = 0x3;
6827       tex->da = da;
6828       Temp size = bld.tmp(v2);
6829       tex->definitions[0] = Definition(size);
6830       tex->can_reorder = true;
6831       ctx->block->instructions.emplace_back(std::move(tex));
6832       emit_split_vector(ctx, size, size.size());
6833
6834       Temp half_texel[2];
6835       for (unsigned i = 0; i < 2; i++) {
6836          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6837          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6838          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6839          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6840       }
6841
6842       Temp orig_coords[2] = {
6843          emit_extract_vector(ctx, coords, 0, v1),
6844          emit_extract_vector(ctx, coords, 1, v1)};
6845       Temp new_coords[2] = {
6846          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6847          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6848       };
6849
6850       if (tg4_integer_cube_workaround) {
6851          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6852          Temp desc[resource.size()];
6853          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6854                                                                            Format::PSEUDO, 1, resource.size())};
6855          split->operands[0] = Operand(resource);
6856          for (unsigned i = 0; i < resource.size(); i++) {
6857             desc[i] = bld.tmp(s1);
6858             split->definitions[i] = Definition(desc[i]);
6859          }
6860          ctx->block->instructions.emplace_back(std::move(split));
6861
6862          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6863          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6864                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6865
6866          Temp nfmt;
6867          if (stype == GLSL_TYPE_UINT) {
6868             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6869                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6870                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6871                             bld.scc(compare_cube_wa));
6872          } else {
6873             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6874                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6875                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6876                             bld.scc(compare_cube_wa));
6877          }
6878          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6879          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6880
6881          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6882
6883          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6884                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6885          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6886
6887          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6888                                                                          Format::PSEUDO, resource.size(), 1)};
6889          for (unsigned i = 0; i < resource.size(); i++)
6890             vec->operands[i] = Operand(desc[i]);
6891          resource = bld.tmp(resource.regClass());
6892          vec->definitions[0] = Definition(resource);
6893          ctx->block->instructions.emplace_back(std::move(vec));
6894
6895          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6896                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6897          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6898                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6899       }
6900
6901       if (coords.size() == 3) {
6902          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6903                              new_coords[0], new_coords[1],
6904                              emit_extract_vector(ctx, coords, 2, v1));
6905       } else {
6906          assert(coords.size() == 2);
6907          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6908                              new_coords[0], new_coords[1]);
6909       }
6910    }
6911
6912    std::vector<Operand> args;
6913    if (has_offset)
6914       args.emplace_back(Operand(offset));
6915    if (has_bias)
6916       args.emplace_back(Operand(bias));
6917    if (has_compare)
6918       args.emplace_back(Operand(compare));
6919    if (has_derivs)
6920       args.emplace_back(Operand(derivs));
6921    args.emplace_back(Operand(coords));
6922    if (has_sample_index)
6923       args.emplace_back(Operand(sample_index));
6924    if (has_lod)
6925       args.emplace_back(lod);
6926
6927    Temp arg;
6928    if (args.size() > 1) {
6929       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6930       unsigned size = 0;
6931       for (unsigned i = 0; i < args.size(); i++) {
6932          size += args[i].size();
6933          vec->operands[i] = args[i];
6934       }
6935       RegClass rc = RegClass(RegType::vgpr, size);
6936       Temp tmp = bld.tmp(rc);
6937       vec->definitions[0] = Definition(tmp);
6938       ctx->block->instructions.emplace_back(std::move(vec));
6939       arg = tmp;
6940    } else {
6941       assert(args[0].isTemp());
6942       arg = as_vgpr(ctx, args[0].getTemp());
6943    }
6944
6945    /* we don't need the bias, sample index, compare value or offset to be
6946     * computed in WQM but if the p_create_vector copies the coordinates, then it
6947     * needs to be in WQM */
6948    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6949        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6950        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6951       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6952
6953    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6954       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6955
6956       assert(coords.size() == 1);
6957       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6958       aco_opcode op;
6959       switch (last_bit) {
6960       case 1:
6961          op = aco_opcode::buffer_load_format_x; break;
6962       case 2:
6963          op = aco_opcode::buffer_load_format_xy; break;
6964       case 3:
6965          op = aco_opcode::buffer_load_format_xyz; break;
6966       case 4:
6967          op = aco_opcode::buffer_load_format_xyzw; break;
6968       default:
6969          unreachable("Tex instruction loads more than 4 components.");
6970       }
6971
6972       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6973       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6974          tmp_dst = dst;
6975       else
6976          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6977
6978       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6979       mubuf->operands[0] = Operand(coords);
6980       mubuf->operands[1] = Operand(resource);
6981       mubuf->operands[2] = Operand((uint32_t) 0);
6982       mubuf->definitions[0] = Definition(tmp_dst);
6983       mubuf->idxen = true;
6984       mubuf->can_reorder = true;
6985       ctx->block->instructions.emplace_back(std::move(mubuf));
6986
6987       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6988       return;
6989    }
6990
6991
6992    if (instr->op == nir_texop_txf ||
6993        instr->op == nir_texop_txf_ms ||
6994        instr->op == nir_texop_samples_identical) {
6995       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6996       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6997       tex->operands[0] = Operand(arg);
6998       tex->operands[1] = Operand(resource);
6999       tex->dim = dim;
7000       tex->dmask = dmask;
7001       tex->unrm = true;
7002       tex->da = da;
7003       tex->definitions[0] = Definition(tmp_dst);
7004       tex->can_reorder = true;
7005       ctx->block->instructions.emplace_back(std::move(tex));
7006
7007       if (instr->op == nir_texop_samples_identical) {
7008          assert(dmask == 1 && dst.regClass() == v1);
7009          assert(dst.id() != tmp_dst.id());
7010
7011          Temp tmp = bld.tmp(bld.lm);
7012          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7013          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7014
7015       } else {
7016          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7017       }
7018       return;
7019    }
7020
7021    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7022    aco_opcode opcode = aco_opcode::image_sample;
7023    if (has_offset) { /* image_sample_*_o */
7024       if (has_compare) {
7025          opcode = aco_opcode::image_sample_c_o;
7026          if (has_derivs)
7027             opcode = aco_opcode::image_sample_c_d_o;
7028          if (has_bias)
7029             opcode = aco_opcode::image_sample_c_b_o;
7030          if (level_zero)
7031             opcode = aco_opcode::image_sample_c_lz_o;
7032          if (has_lod)
7033             opcode = aco_opcode::image_sample_c_l_o;
7034       } else {
7035          opcode = aco_opcode::image_sample_o;
7036          if (has_derivs)
7037             opcode = aco_opcode::image_sample_d_o;
7038          if (has_bias)
7039             opcode = aco_opcode::image_sample_b_o;
7040          if (level_zero)
7041             opcode = aco_opcode::image_sample_lz_o;
7042          if (has_lod)
7043             opcode = aco_opcode::image_sample_l_o;
7044       }
7045    } else { /* no offset */
7046       if (has_compare) {
7047          opcode = aco_opcode::image_sample_c;
7048          if (has_derivs)
7049             opcode = aco_opcode::image_sample_c_d;
7050          if (has_bias)
7051             opcode = aco_opcode::image_sample_c_b;
7052          if (level_zero)
7053             opcode = aco_opcode::image_sample_c_lz;
7054          if (has_lod)
7055             opcode = aco_opcode::image_sample_c_l;
7056       } else {
7057          opcode = aco_opcode::image_sample;
7058          if (has_derivs)
7059             opcode = aco_opcode::image_sample_d;
7060          if (has_bias)
7061             opcode = aco_opcode::image_sample_b;
7062          if (level_zero)
7063             opcode = aco_opcode::image_sample_lz;
7064          if (has_lod)
7065             opcode = aco_opcode::image_sample_l;
7066       }
7067    }
7068
7069    if (instr->op == nir_texop_tg4) {
7070       if (has_offset) {
7071          opcode = aco_opcode::image_gather4_lz_o;
7072          if (has_compare)
7073             opcode = aco_opcode::image_gather4_c_lz_o;
7074       } else {
7075          opcode = aco_opcode::image_gather4_lz;
7076          if (has_compare)
7077             opcode = aco_opcode::image_gather4_c_lz;
7078       }
7079    } else if (instr->op == nir_texop_lod) {
7080       opcode = aco_opcode::image_get_lod;
7081    }
7082
7083    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7084    tex->operands[0] = Operand(arg);
7085    tex->operands[1] = Operand(resource);
7086    tex->operands[2] = Operand(sampler);
7087    tex->dim = dim;
7088    tex->dmask = dmask;
7089    tex->da = da;
7090    tex->definitions[0] = Definition(tmp_dst);
7091    tex->can_reorder = true;
7092    ctx->block->instructions.emplace_back(std::move(tex));
7093
7094    if (tg4_integer_cube_workaround) {
7095       assert(tmp_dst.id() != dst.id());
7096       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7097
7098       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7099       Temp val[4];
7100       for (unsigned i = 0; i < dst.size(); i++) {
7101          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7102          Temp cvt_val;
7103          if (stype == GLSL_TYPE_UINT)
7104             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7105          else
7106             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7107          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7108       }
7109       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7110       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7111                            val[0], val[1], val[2], val[3]);
7112    }
7113    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7114    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7115
7116 }
7117
7118
7119 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7120 {
7121    Temp tmp = get_ssa_temp(ctx, ssa);
7122    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7123       return Operand(tmp.regClass());
7124    else
7125       return Operand(tmp);
7126 }
7127
7128 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7129 {
7130    aco_ptr<Pseudo_instruction> phi;
7131    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7132    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7133
7134    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7135    logical |= ctx->block->kind & block_kind_merge;
7136    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7137
7138    /* we want a sorted list of sources, since the predecessor list is also sorted */
7139    std::map<unsigned, nir_ssa_def*> phi_src;
7140    nir_foreach_phi_src(src, instr)
7141       phi_src[src->pred->index] = src->src.ssa;
7142
7143    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7144    unsigned num_operands = 0;
7145    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7146    unsigned num_defined = 0;
7147    unsigned cur_pred_idx = 0;
7148    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7149       if (cur_pred_idx < preds.size()) {
7150          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7151          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7152          unsigned skipped = 0;
7153          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7154             skipped++;
7155          if (cur_pred_idx + skipped < preds.size()) {
7156             for (unsigned i = 0; i < skipped; i++)
7157                operands[num_operands++] = Operand(dst.regClass());
7158             cur_pred_idx += skipped;
7159          } else {
7160             continue;
7161          }
7162       }
7163       cur_pred_idx++;
7164       Operand op = get_phi_operand(ctx, src.second);
7165       operands[num_operands++] = op;
7166       num_defined += !op.isUndefined();
7167    }
7168    /* handle block_kind_continue_or_break at loop exit blocks */
7169    while (cur_pred_idx++ < preds.size())
7170       operands[num_operands++] = Operand(dst.regClass());
7171
7172    if (num_defined == 0) {
7173       Builder bld(ctx->program, ctx->block);
7174       if (dst.regClass() == s1) {
7175          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7176       } else if (dst.regClass() == v1) {
7177          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7178       } else {
7179          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7180          for (unsigned i = 0; i < dst.size(); i++)
7181             vec->operands[i] = Operand(0u);
7182          vec->definitions[0] = Definition(dst);
7183          ctx->block->instructions.emplace_back(std::move(vec));
7184       }
7185       return;
7186    }
7187
7188    /* we can use a linear phi in some cases if one src is undef */
7189    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7190       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7191
7192       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7193       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7194       assert(invert->kind & block_kind_invert);
7195
7196       unsigned then_block = invert->linear_preds[0];
7197
7198       Block* insert_block = NULL;
7199       for (unsigned i = 0; i < num_operands; i++) {
7200          Operand op = operands[i];
7201          if (op.isUndefined())
7202             continue;
7203          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7204          phi->operands[0] = op;
7205          break;
7206       }
7207       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7208       phi->operands[1] = Operand(dst.regClass());
7209       phi->definitions[0] = Definition(dst);
7210       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7211       return;
7212    }
7213
7214    /* try to scalarize vector phis */
7215    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7216       // TODO: scalarize linear phis on divergent ifs
7217       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7218       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7219       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7220          Operand src = operands[i];
7221          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7222             can_scalarize = false;
7223       }
7224       if (can_scalarize) {
7225          unsigned num_components = instr->dest.ssa.num_components;
7226          assert(dst.size() % num_components == 0);
7227          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7228
7229          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7230          for (unsigned k = 0; k < num_components; k++) {
7231             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7232             for (unsigned i = 0; i < num_operands; i++) {
7233                Operand src = operands[i];
7234                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7235             }
7236             Temp phi_dst = {ctx->program->allocateId(), rc};
7237             phi->definitions[0] = Definition(phi_dst);
7238             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7239             new_vec[k] = phi_dst;
7240             vec->operands[k] = Operand(phi_dst);
7241          }
7242          vec->definitions[0] = Definition(dst);
7243          ctx->block->instructions.emplace_back(std::move(vec));
7244          ctx->allocated_vec.emplace(dst.id(), new_vec);
7245          return;
7246       }
7247    }
7248
7249    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7250    for (unsigned i = 0; i < num_operands; i++)
7251       phi->operands[i] = operands[i];
7252    phi->definitions[0] = Definition(dst);
7253    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7254 }
7255
7256
7257 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7258 {
7259    Temp dst = get_ssa_temp(ctx, &instr->def);
7260
7261    assert(dst.type() == RegType::sgpr);
7262
7263    if (dst.size() == 1) {
7264       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7265    } else {
7266       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7267       for (unsigned i = 0; i < dst.size(); i++)
7268          vec->operands[i] = Operand(0u);
7269       vec->definitions[0] = Definition(dst);
7270       ctx->block->instructions.emplace_back(std::move(vec));
7271    }
7272 }
7273
7274 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7275 {
7276    Builder bld(ctx->program, ctx->block);
7277    Block *logical_target;
7278    append_logical_end(ctx->block);
7279    unsigned idx = ctx->block->index;
7280
7281    switch (instr->type) {
7282    case nir_jump_break:
7283       logical_target = ctx->cf_info.parent_loop.exit;
7284       add_logical_edge(idx, logical_target);
7285       ctx->block->kind |= block_kind_break;
7286
7287       if (!ctx->cf_info.parent_if.is_divergent &&
7288           !ctx->cf_info.parent_loop.has_divergent_continue) {
7289          /* uniform break - directly jump out of the loop */
7290          ctx->block->kind |= block_kind_uniform;
7291          ctx->cf_info.has_branch = true;
7292          bld.branch(aco_opcode::p_branch);
7293          add_linear_edge(idx, logical_target);
7294          return;
7295       }
7296       ctx->cf_info.parent_loop.has_divergent_branch = true;
7297       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7298       break;
7299    case nir_jump_continue:
7300       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7301       add_logical_edge(idx, logical_target);
7302       ctx->block->kind |= block_kind_continue;
7303
7304       if (ctx->cf_info.parent_if.is_divergent) {
7305          /* for potential uniform breaks after this continue,
7306             we must ensure that they are handled correctly */
7307          ctx->cf_info.parent_loop.has_divergent_continue = true;
7308          ctx->cf_info.parent_loop.has_divergent_branch = true;
7309          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7310       } else {
7311          /* uniform continue - directly jump to the loop header */
7312          ctx->block->kind |= block_kind_uniform;
7313          ctx->cf_info.has_branch = true;
7314          bld.branch(aco_opcode::p_branch);
7315          add_linear_edge(idx, logical_target);
7316          return;
7317       }
7318       break;
7319    default:
7320       fprintf(stderr, "Unknown NIR jump instr: ");
7321       nir_print_instr(&instr->instr, stderr);
7322       fprintf(stderr, "\n");
7323       abort();
7324    }
7325
7326    /* remove critical edges from linear CFG */
7327    bld.branch(aco_opcode::p_branch);
7328    Block* break_block = ctx->program->create_and_insert_block();
7329    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7330    break_block->kind |= block_kind_uniform;
7331    add_linear_edge(idx, break_block);
7332    /* the loop_header pointer might be invalidated by this point */
7333    if (instr->type == nir_jump_continue)
7334       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7335    add_linear_edge(break_block->index, logical_target);
7336    bld.reset(break_block);
7337    bld.branch(aco_opcode::p_branch);
7338
7339    Block* continue_block = ctx->program->create_and_insert_block();
7340    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7341    add_linear_edge(idx, continue_block);
7342    append_logical_start(continue_block);
7343    ctx->block = continue_block;
7344    return;
7345 }
7346
7347 void visit_block(isel_context *ctx, nir_block *block)
7348 {
7349    nir_foreach_instr(instr, block) {
7350       switch (instr->type) {
7351       case nir_instr_type_alu:
7352          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7353          break;
7354       case nir_instr_type_load_const:
7355          visit_load_const(ctx, nir_instr_as_load_const(instr));
7356          break;
7357       case nir_instr_type_intrinsic:
7358          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7359          break;
7360       case nir_instr_type_tex:
7361          visit_tex(ctx, nir_instr_as_tex(instr));
7362          break;
7363       case nir_instr_type_phi:
7364          visit_phi(ctx, nir_instr_as_phi(instr));
7365          break;
7366       case nir_instr_type_ssa_undef:
7367          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7368          break;
7369       case nir_instr_type_deref:
7370          break;
7371       case nir_instr_type_jump:
7372          visit_jump(ctx, nir_instr_as_jump(instr));
7373          break;
7374       default:
7375          fprintf(stderr, "Unknown NIR instr type: ");
7376          nir_print_instr(instr, stderr);
7377          fprintf(stderr, "\n");
7378          //abort();
7379       }
7380    }
7381
7382    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7383       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7384 }
7385
7386
7387
7388 static void visit_loop(isel_context *ctx, nir_loop *loop)
7389 {
7390    append_logical_end(ctx->block);
7391    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7392    Builder bld(ctx->program, ctx->block);
7393    bld.branch(aco_opcode::p_branch);
7394    unsigned loop_preheader_idx = ctx->block->index;
7395
7396    Block loop_exit = Block();
7397    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7398    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7399
7400    Block* loop_header = ctx->program->create_and_insert_block();
7401    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7402    loop_header->kind |= block_kind_loop_header;
7403    add_edge(loop_preheader_idx, loop_header);
7404    ctx->block = loop_header;
7405
7406    /* emit loop body */
7407    unsigned loop_header_idx = loop_header->index;
7408    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7409    append_logical_start(ctx->block);
7410    visit_cf_list(ctx, &loop->body);
7411
7412    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7413    if (!ctx->cf_info.has_branch) {
7414       append_logical_end(ctx->block);
7415       if (ctx->cf_info.exec_potentially_empty) {
7416          /* Discards can result in code running with an empty exec mask.
7417           * This would result in divergent breaks not ever being taken. As a
7418           * workaround, break the loop when the loop mask is empty instead of
7419           * always continuing. */
7420          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7421          unsigned block_idx = ctx->block->index;
7422
7423          /* create helper blocks to avoid critical edges */
7424          Block *break_block = ctx->program->create_and_insert_block();
7425          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7426          break_block->kind = block_kind_uniform;
7427          bld.reset(break_block);
7428          bld.branch(aco_opcode::p_branch);
7429          add_linear_edge(block_idx, break_block);
7430          add_linear_edge(break_block->index, &loop_exit);
7431
7432          Block *continue_block = ctx->program->create_and_insert_block();
7433          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7434          continue_block->kind = block_kind_uniform;
7435          bld.reset(continue_block);
7436          bld.branch(aco_opcode::p_branch);
7437          add_linear_edge(block_idx, continue_block);
7438          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7439
7440          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7441          ctx->block = &ctx->program->blocks[block_idx];
7442       } else {
7443          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7444          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7445             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7446          else
7447             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7448       }
7449
7450       bld.reset(ctx->block);
7451       bld.branch(aco_opcode::p_branch);
7452    }
7453
7454    /* fixup phis in loop header from unreachable blocks */
7455    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7456       bool linear = ctx->cf_info.has_branch;
7457       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7458       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7459          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7460              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7461             /* the last operand should be the one that needs to be removed */
7462             instr->operands.pop_back();
7463          } else if (!is_phi(instr)) {
7464             break;
7465          }
7466       }
7467    }
7468
7469    ctx->cf_info.has_branch = false;
7470
7471    // TODO: if the loop has not a single exit, we must add one °°
7472    /* emit loop successor block */
7473    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7474    append_logical_start(ctx->block);
7475
7476    #if 0
7477    // TODO: check if it is beneficial to not branch on continues
7478    /* trim linear phis in loop header */
7479    for (auto&& instr : loop_entry->instructions) {
7480       if (instr->opcode == aco_opcode::p_linear_phi) {
7481          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7482          new_phi->definitions[0] = instr->definitions[0];
7483          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7484             new_phi->operands[i] = instr->operands[i];
7485          /* check that the remaining operands are all the same */
7486          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7487             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7488          instr.swap(new_phi);
7489       } else if (instr->opcode == aco_opcode::p_phi) {
7490          continue;
7491       } else {
7492          break;
7493       }
7494    }
7495    #endif
7496 }
7497
7498 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7499 {
7500    ic->cond = cond;
7501
7502    append_logical_end(ctx->block);
7503    ctx->block->kind |= block_kind_branch;
7504
7505    /* branch to linear then block */
7506    assert(cond.regClass() == ctx->program->lane_mask);
7507    aco_ptr<Pseudo_branch_instruction> branch;
7508    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7509    branch->operands[0] = Operand(cond);
7510    ctx->block->instructions.push_back(std::move(branch));
7511
7512    ic->BB_if_idx = ctx->block->index;
7513    ic->BB_invert = Block();
7514    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7515    /* Invert blocks are intentionally not marked as top level because they
7516     * are not part of the logical cfg. */
7517    ic->BB_invert.kind |= block_kind_invert;
7518    ic->BB_endif = Block();
7519    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7520    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7521
7522    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7523    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7524    ctx->cf_info.parent_if.is_divergent = true;
7525    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7526
7527    /** emit logical then block */
7528    Block* BB_then_logical = ctx->program->create_and_insert_block();
7529    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7530    add_edge(ic->BB_if_idx, BB_then_logical);
7531    ctx->block = BB_then_logical;
7532    append_logical_start(BB_then_logical);
7533 }
7534
7535 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7536 {
7537    Block *BB_then_logical = ctx->block;
7538    append_logical_end(BB_then_logical);
7539     /* branch from logical then block to invert block */
7540    aco_ptr<Pseudo_branch_instruction> branch;
7541    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7542    BB_then_logical->instructions.emplace_back(std::move(branch));
7543    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7544    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7545       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7546    BB_then_logical->kind |= block_kind_uniform;
7547    assert(!ctx->cf_info.has_branch);
7548    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7549    ctx->cf_info.parent_loop.has_divergent_branch = false;
7550
7551    /** emit linear then block */
7552    Block* BB_then_linear = ctx->program->create_and_insert_block();
7553    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7554    BB_then_linear->kind |= block_kind_uniform;
7555    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7556    /* branch from linear then block to invert block */
7557    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7558    BB_then_linear->instructions.emplace_back(std::move(branch));
7559    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7560
7561    /** emit invert merge block */
7562    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7563    ic->invert_idx = ctx->block->index;
7564
7565    /* branch to linear else block (skip else) */
7566    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7567    branch->operands[0] = Operand(ic->cond);
7568    ctx->block->instructions.push_back(std::move(branch));
7569
7570    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7571    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7572
7573    /** emit logical else block */
7574    Block* BB_else_logical = ctx->program->create_and_insert_block();
7575    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7576    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7577    add_linear_edge(ic->invert_idx, BB_else_logical);
7578    ctx->block = BB_else_logical;
7579    append_logical_start(BB_else_logical);
7580 }
7581
7582 static void end_divergent_if(isel_context *ctx, if_context *ic)
7583 {
7584    Block *BB_else_logical = ctx->block;
7585    append_logical_end(BB_else_logical);
7586
7587    /* branch from logical else block to endif block */
7588    aco_ptr<Pseudo_branch_instruction> branch;
7589    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7590    BB_else_logical->instructions.emplace_back(std::move(branch));
7591    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7592    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7593       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7594    BB_else_logical->kind |= block_kind_uniform;
7595
7596    assert(!ctx->cf_info.has_branch);
7597    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7598
7599
7600    /** emit linear else block */
7601    Block* BB_else_linear = ctx->program->create_and_insert_block();
7602    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7603    BB_else_linear->kind |= block_kind_uniform;
7604    add_linear_edge(ic->invert_idx, BB_else_linear);
7605
7606    /* branch from linear else block to endif block */
7607    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7608    BB_else_linear->instructions.emplace_back(std::move(branch));
7609    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7610
7611
7612    /** emit endif merge block */
7613    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7614    append_logical_start(ctx->block);
7615
7616
7617    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7618    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7619    /* uniform control flow never has an empty exec-mask */
7620    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7621       ctx->cf_info.exec_potentially_empty = false;
7622 }
7623
7624 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7625 {
7626    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7627    Builder bld(ctx->program, ctx->block);
7628    aco_ptr<Pseudo_branch_instruction> branch;
7629
7630    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7631       /**
7632        * Uniform conditionals are represented in the following way*) :
7633        *
7634        * The linear and logical CFG:
7635        *                        BB_IF
7636        *                        /    \
7637        *       BB_THEN (logical)      BB_ELSE (logical)
7638        *                        \    /
7639        *                        BB_ENDIF
7640        *
7641        * *) Exceptions may be due to break and continue statements within loops
7642        *    If a break/continue happens within uniform control flow, it branches
7643        *    to the loop exit/entry block. Otherwise, it branches to the next
7644        *    merge block.
7645        **/
7646       append_logical_end(ctx->block);
7647       ctx->block->kind |= block_kind_uniform;
7648
7649       /* emit branch */
7650       assert(cond.regClass() == bld.lm);
7651       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7652       cond = bool_to_scalar_condition(ctx, cond);
7653
7654       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7655       branch->operands[0] = Operand(cond);
7656       branch->operands[0].setFixed(scc);
7657       ctx->block->instructions.emplace_back(std::move(branch));
7658
7659       unsigned BB_if_idx = ctx->block->index;
7660       Block BB_endif = Block();
7661       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7662       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7663
7664       /** emit then block */
7665       Block* BB_then = ctx->program->create_and_insert_block();
7666       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7667       add_edge(BB_if_idx, BB_then);
7668       append_logical_start(BB_then);
7669       ctx->block = BB_then;
7670       visit_cf_list(ctx, &if_stmt->then_list);
7671       BB_then = ctx->block;
7672       bool then_branch = ctx->cf_info.has_branch;
7673       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7674
7675       if (!then_branch) {
7676          append_logical_end(BB_then);
7677          /* branch from then block to endif block */
7678          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7679          BB_then->instructions.emplace_back(std::move(branch));
7680          add_linear_edge(BB_then->index, &BB_endif);
7681          if (!then_branch_divergent)
7682             add_logical_edge(BB_then->index, &BB_endif);
7683          BB_then->kind |= block_kind_uniform;
7684       }
7685
7686       ctx->cf_info.has_branch = false;
7687       ctx->cf_info.parent_loop.has_divergent_branch = false;
7688
7689       /** emit else block */
7690       Block* BB_else = ctx->program->create_and_insert_block();
7691       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7692       add_edge(BB_if_idx, BB_else);
7693       append_logical_start(BB_else);
7694       ctx->block = BB_else;
7695       visit_cf_list(ctx, &if_stmt->else_list);
7696       BB_else = ctx->block;
7697
7698       if (!ctx->cf_info.has_branch) {
7699          append_logical_end(BB_else);
7700          /* branch from then block to endif block */
7701          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7702          BB_else->instructions.emplace_back(std::move(branch));
7703          add_linear_edge(BB_else->index, &BB_endif);
7704          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7705             add_logical_edge(BB_else->index, &BB_endif);
7706          BB_else->kind |= block_kind_uniform;
7707       }
7708
7709       ctx->cf_info.has_branch &= then_branch;
7710       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7711
7712       /** emit endif merge block */
7713       if (!ctx->cf_info.has_branch) {
7714          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7715          append_logical_start(ctx->block);
7716       }
7717    } else { /* non-uniform condition */
7718       /**
7719        * To maintain a logical and linear CFG without critical edges,
7720        * non-uniform conditionals are represented in the following way*) :
7721        *
7722        * The linear CFG:
7723        *                        BB_IF
7724        *                        /    \
7725        *       BB_THEN (logical)      BB_THEN (linear)
7726        *                        \    /
7727        *                        BB_INVERT (linear)
7728        *                        /    \
7729        *       BB_ELSE (logical)      BB_ELSE (linear)
7730        *                        \    /
7731        *                        BB_ENDIF
7732        *
7733        * The logical CFG:
7734        *                        BB_IF
7735        *                        /    \
7736        *       BB_THEN (logical)      BB_ELSE (logical)
7737        *                        \    /
7738        *                        BB_ENDIF
7739        *
7740        * *) Exceptions may be due to break and continue statements within loops
7741        **/
7742
7743       if_context ic;
7744
7745       begin_divergent_if_then(ctx, &ic, cond);
7746       visit_cf_list(ctx, &if_stmt->then_list);
7747
7748       begin_divergent_if_else(ctx, &ic);
7749       visit_cf_list(ctx, &if_stmt->else_list);
7750
7751       end_divergent_if(ctx, &ic);
7752    }
7753 }
7754
7755 static void visit_cf_list(isel_context *ctx,
7756                           struct exec_list *list)
7757 {
7758    foreach_list_typed(nir_cf_node, node, node, list) {
7759       switch (node->type) {
7760       case nir_cf_node_block:
7761          visit_block(ctx, nir_cf_node_as_block(node));
7762          break;
7763       case nir_cf_node_if:
7764          visit_if(ctx, nir_cf_node_as_if(node));
7765          break;
7766       case nir_cf_node_loop:
7767          visit_loop(ctx, nir_cf_node_as_loop(node));
7768          break;
7769       default:
7770          unreachable("unimplemented cf list type");
7771       }
7772    }
7773 }
7774
7775 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7776 {
7777    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7778    uint64_t mask = ctx->vs_output.mask[slot];
7779    if (!is_pos && !mask)
7780       return;
7781    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7782       return;
7783    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7784    exp->enabled_mask = mask;
7785    for (unsigned i = 0; i < 4; ++i) {
7786       if (mask & (1 << i))
7787          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7788       else
7789          exp->operands[i] = Operand(v1);
7790    }
7791    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
7792     * Setting valid_mask=1 prevents it and has no other effect.
7793     */
7794    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
7795    exp->done = false;
7796    exp->compressed = false;
7797    if (is_pos)
7798       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7799    else
7800       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7801    ctx->block->instructions.emplace_back(std::move(exp));
7802 }
7803
7804 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7805 {
7806    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7807    exp->enabled_mask = 0;
7808    for (unsigned i = 0; i < 4; ++i)
7809       exp->operands[i] = Operand(v1);
7810    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7811       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7812       exp->enabled_mask |= 0x1;
7813    }
7814    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7815       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7816       exp->enabled_mask |= 0x4;
7817    }
7818    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7819       if (ctx->options->chip_class < GFX9) {
7820          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7821          exp->enabled_mask |= 0x8;
7822       } else {
7823          Builder bld(ctx->program, ctx->block);
7824
7825          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7826                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7827          if (exp->operands[2].isTemp())
7828             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7829
7830          exp->operands[2] = Operand(out);
7831          exp->enabled_mask |= 0x4;
7832       }
7833    }
7834    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
7835    exp->done = false;
7836    exp->compressed = false;
7837    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7838    ctx->block->instructions.emplace_back(std::move(exp));
7839 }
7840
7841 static void create_vs_exports(isel_context *ctx)
7842 {
7843    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7844
7845    if (outinfo->export_prim_id) {
7846       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7847       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7848    }
7849
7850    if (ctx->options->key.has_multiview_view_index) {
7851       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7852       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7853    }
7854
7855    /* the order these position exports are created is important */
7856    int next_pos = 0;
7857    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7858    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7859       export_vs_psiz_layer_viewport(ctx, &next_pos);
7860    }
7861    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7862       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7863    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7864       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7865
7866    if (ctx->options->key.vs_common_out.export_clip_dists) {
7867       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7868          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7869       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7870          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7871    }
7872
7873    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7874       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7875           i != VARYING_SLOT_PRIMITIVE_ID)
7876          continue;
7877
7878       export_vs_varying(ctx, i, false, NULL);
7879    }
7880 }
7881
7882 static void emit_stream_output(isel_context *ctx,
7883                                Temp const *so_buffers,
7884                                Temp const *so_write_offset,
7885                                const struct radv_stream_output *output)
7886 {
7887    unsigned num_comps = util_bitcount(output->component_mask);
7888    unsigned writemask = (1 << num_comps) - 1;
7889    unsigned loc = output->location;
7890    unsigned buf = output->buffer;
7891
7892    assert(num_comps && num_comps <= 4);
7893    if (!num_comps || num_comps > 4)
7894       return;
7895
7896    unsigned start = ffs(output->component_mask) - 1;
7897
7898    Temp out[4];
7899    bool all_undef = true;
7900    assert(ctx->stage == vertex_vs);
7901    for (unsigned i = 0; i < num_comps; i++) {
7902       out[i] = ctx->vs_output.outputs[loc][start + i];
7903       all_undef = all_undef && !out[i].id();
7904    }
7905    if (all_undef)
7906       return;
7907
7908    while (writemask) {
7909       int start, count;
7910       u_bit_scan_consecutive_range(&writemask, &start, &count);
7911       if (count == 3 && ctx->options->chip_class == GFX6) {
7912          /* GFX6 doesn't support storing vec3, split it. */
7913          writemask |= 1u << (start + 2);
7914          count = 2;
7915       }
7916
7917       unsigned offset = output->offset + start * 4;
7918
7919       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
7920       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
7921       for (int i = 0; i < count; ++i)
7922          vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
7923       vec->definitions[0] = Definition(write_data);
7924       ctx->block->instructions.emplace_back(std::move(vec));
7925
7926       aco_opcode opcode;
7927       switch (count) {
7928       case 1:
7929          opcode = aco_opcode::buffer_store_dword;
7930          break;
7931       case 2:
7932          opcode = aco_opcode::buffer_store_dwordx2;
7933          break;
7934       case 3:
7935          opcode = aco_opcode::buffer_store_dwordx3;
7936          break;
7937       case 4:
7938          opcode = aco_opcode::buffer_store_dwordx4;
7939          break;
7940       default:
7941          unreachable("Unsupported dword count.");
7942       }
7943
7944       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7945       store->operands[0] = Operand(so_write_offset[buf]);
7946       store->operands[1] = Operand(so_buffers[buf]);
7947       store->operands[2] = Operand((uint32_t) 0);
7948       store->operands[3] = Operand(write_data);
7949       if (offset > 4095) {
7950          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7951          Builder bld(ctx->program, ctx->block);
7952          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7953       } else {
7954          store->offset = offset;
7955       }
7956       store->offen = true;
7957       store->glc = true;
7958       store->dlc = false;
7959       store->slc = true;
7960       store->can_reorder = true;
7961       ctx->block->instructions.emplace_back(std::move(store));
7962    }
7963 }
7964
7965 static void emit_streamout(isel_context *ctx, unsigned stream)
7966 {
7967    Builder bld(ctx->program, ctx->block);
7968
7969    Temp so_buffers[4];
7970    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7971    for (unsigned i = 0; i < 4; i++) {
7972       unsigned stride = ctx->program->info->so.strides[i];
7973       if (!stride)
7974          continue;
7975
7976       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7977    }
7978
7979    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7980                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7981
7982    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7983
7984    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
7985
7986    if_context ic;
7987    begin_divergent_if_then(ctx, &ic, can_emit);
7988
7989    bld.reset(ctx->block);
7990
7991    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7992
7993    Temp so_write_offset[4];
7994
7995    for (unsigned i = 0; i < 4; i++) {
7996       unsigned stride = ctx->program->info->so.strides[i];
7997       if (!stride)
7998          continue;
7999
8000       if (stride == 1) {
8001          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8002                                 get_arg(ctx, ctx->args->streamout_write_idx),
8003                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8004          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8005
8006          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8007       } else {
8008          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8009          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8010                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8011          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8012       }
8013    }
8014
8015    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8016       struct radv_stream_output *output =
8017          &ctx->program->info->so.outputs[i];
8018       if (stream != output->stream)
8019          continue;
8020
8021       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8022    }
8023
8024    begin_divergent_if_else(ctx, &ic);
8025    end_divergent_if(ctx, &ic);
8026 }
8027
8028 } /* end namespace */
8029
8030 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8031 {
8032    /* Split all arguments except for the first (ring_offsets) and the last
8033     * (exec) so that the dead channels don't stay live throughout the program.
8034     */
8035    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8036       if (startpgm->definitions[i].regClass().size() > 1) {
8037          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8038                            startpgm->definitions[i].regClass().size());
8039       }
8040    }
8041 }
8042
8043 void handle_bc_optimize(isel_context *ctx)
8044 {
8045    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8046    Builder bld(ctx->program, ctx->block);
8047    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8048    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8049    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8050    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8051    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8052    if (uses_center && uses_centroid) {
8053       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8054                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8055
8056       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8057          Temp new_coord[2];
8058          for (unsigned i = 0; i < 2; i++) {
8059             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8060             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8061             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8062                                     persp_centroid, persp_center, sel);
8063          }
8064          ctx->persp_centroid = bld.tmp(v2);
8065          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8066                     Operand(new_coord[0]), Operand(new_coord[1]));
8067          emit_split_vector(ctx, ctx->persp_centroid, 2);
8068       }
8069
8070       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8071          Temp new_coord[2];
8072          for (unsigned i = 0; i < 2; i++) {
8073             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8074             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8075             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8076                                     linear_centroid, linear_center, sel);
8077          }
8078          ctx->linear_centroid = bld.tmp(v2);
8079          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8080                     Operand(new_coord[0]), Operand(new_coord[1]));
8081          emit_split_vector(ctx, ctx->linear_centroid, 2);
8082       }
8083    }
8084 }
8085
8086 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8087 {
8088    Program *program = ctx->program;
8089
8090    unsigned float_controls = shader->info.float_controls_execution_mode;
8091
8092    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8093       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8094    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8095       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8096                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8097
8098    program->next_fp_mode.must_flush_denorms32 =
8099       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8100    program->next_fp_mode.must_flush_denorms16_64 =
8101       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8102                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8103
8104    program->next_fp_mode.care_about_round32 =
8105       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8106
8107    program->next_fp_mode.care_about_round16_64 =
8108       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8109                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8110
8111    /* default to preserving fp16 and fp64 denorms, since it's free */
8112    if (program->next_fp_mode.must_flush_denorms16_64)
8113       program->next_fp_mode.denorm16_64 = 0;
8114    else
8115       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8116
8117    /* preserving fp32 denorms is expensive, so only do it if asked */
8118    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8119       program->next_fp_mode.denorm32 = fp_denorm_keep;
8120    else
8121       program->next_fp_mode.denorm32 = 0;
8122
8123    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8124       program->next_fp_mode.round32 = fp_round_tz;
8125    else
8126       program->next_fp_mode.round32 = fp_round_ne;
8127
8128    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8129       program->next_fp_mode.round16_64 = fp_round_tz;
8130    else
8131       program->next_fp_mode.round16_64 = fp_round_ne;
8132
8133    ctx->block->fp_mode = program->next_fp_mode;
8134 }
8135
8136 void select_program(Program *program,
8137                     unsigned shader_count,
8138                     struct nir_shader *const *shaders,
8139                     ac_shader_config* config,
8140                     struct radv_shader_args *args)
8141 {
8142    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
8143
8144    for (unsigned i = 0; i < shader_count; i++) {
8145       nir_shader *nir = shaders[i];
8146       init_context(&ctx, nir);
8147
8148       setup_fp_mode(&ctx, nir);
8149
8150       if (!i) {
8151          /* needs to be after init_context() for FS */
8152          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8153          append_logical_start(ctx.block);
8154          split_arguments(&ctx, startpgm);
8155       }
8156
8157       if_context ic;
8158       if (shader_count >= 2) {
8159          Builder bld(ctx.program, ctx.block);
8160          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
8161          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8162          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8163
8164          begin_divergent_if_then(&ctx, &ic, cond);
8165       }
8166
8167       if (i) {
8168          Builder bld(ctx.program, ctx.block);
8169          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
8170          bld.sopp(aco_opcode::s_barrier);
8171       }
8172
8173       if (ctx.stage == fragment_fs)
8174          handle_bc_optimize(&ctx);
8175
8176       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8177       visit_cf_list(&ctx, &func->body);
8178
8179       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8180          emit_streamout(&ctx, 0);
8181
8182       if (ctx.stage == vertex_vs)
8183          create_vs_exports(&ctx);
8184
8185       if (shader_count >= 2) {
8186          begin_divergent_if_else(&ctx, &ic);
8187          end_divergent_if(&ctx, &ic);
8188       }
8189
8190       ralloc_free(ctx.divergent_vals);
8191    }
8192
8193    program->config->float_mode = program->blocks[0].fp_mode.val;
8194
8195    append_logical_end(ctx.block);
8196    ctx.block->kind |= block_kind_uniform;
8197    Builder bld(ctx.program, ctx.block);
8198    if (ctx.program->wb_smem_l1_on_end)
8199       bld.smem(aco_opcode::s_dcache_wb, false);
8200    bld.sopp(aco_opcode::s_endpgm);
8201
8202    /* cleanup CFG */
8203    for (Block& BB : program->blocks) {
8204       for (unsigned idx : BB.linear_preds)
8205          program->blocks[idx].linear_succs.emplace_back(BB.index);
8206       for (unsigned idx : BB.logical_preds)
8207          program->blocks[idx].logical_succs.emplace_back(BB.index);
8208    }
8209 }
8210 }