src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.readlane(bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 439                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 440 {
 441    Builder bld(ctx->program, ctx->block);
 442    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 443    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 444    if (src1.type() == RegType::sgpr) {
 445       if (commutative && src0.type() == RegType::vgpr) {
 446          Temp t = src0;
 447          src0 = src1;
 448          src1 = t;
 449       } else if (src0.type() == RegType::vgpr &&
 450                  op != aco_opcode::v_madmk_f32 &&
 451                  op != aco_opcode::v_madak_f32 &&
 452                  op != aco_opcode::v_madmk_f16 &&
 453                  op != aco_opcode::v_madak_f16) {
 454          /* If the instruction is not commutative, we emit a VOP3A instruction */
 455          bld.vop2_e64(op, Definition(dst), src0, src1);
 456          return;
 457       } else {
 458          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 459       }
 460    }
 461
 462    if (flush_denorms && ctx->program->chip_class < GFX9) {
 463       assert(dst.size() == 1);
 464       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 465       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 466    } else {
 467       bld.vop2(op, Definition(dst), src0, src1);
 468    }
 469 }
 470
 471 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 472                             bool flush_denorms = false)
 473 {
 474    Temp src0 = get_alu_src(ctx, instr->src[0]);
 475    Temp src1 = get_alu_src(ctx, instr->src[1]);
 476    Temp src2 = get_alu_src(ctx, instr->src[2]);
 477
 478    /* ensure that the instruction has at most 1 sgpr operand
 479     * The optimizer will inline constants for us */
 480    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 481       src0 = as_vgpr(ctx, src0);
 482    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 483       src1 = as_vgpr(ctx, src1);
 484    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 485       src2 = as_vgpr(ctx, src2);
 486
 487    Builder bld(ctx->program, ctx->block);
 488    if (flush_denorms && ctx->program->chip_class < GFX9) {
 489       assert(dst.size() == 1);
 490       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 491       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 492    } else {
 493       bld.vop3(op, Definition(dst), src0, src1, src2);
 494    }
 495 }
 496
 497 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 498 {
 499    Builder bld(ctx->program, ctx->block);
 500    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 501 }
 502
 503 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 504 {
 505    Temp src0 = get_alu_src(ctx, instr->src[0]);
 506    Temp src1 = get_alu_src(ctx, instr->src[1]);
 507    assert(src0.size() == src1.size());
 508
 509    aco_ptr<Instruction> vopc;
 510    if (src1.type() == RegType::sgpr) {
 511       if (src0.type() == RegType::vgpr) {
 512          /* to swap the operands, we might also have to change the opcode */
 513          switch (op) {
 514             case aco_opcode::v_cmp_lt_f32:
 515                op = aco_opcode::v_cmp_gt_f32;
 516                break;
 517             case aco_opcode::v_cmp_ge_f32:
 518                op = aco_opcode::v_cmp_le_f32;
 519                break;
 520             case aco_opcode::v_cmp_lt_i32:
 521                op = aco_opcode::v_cmp_gt_i32;
 522                break;
 523             case aco_opcode::v_cmp_ge_i32:
 524                op = aco_opcode::v_cmp_le_i32;
 525                break;
 526             case aco_opcode::v_cmp_lt_u32:
 527                op = aco_opcode::v_cmp_gt_u32;
 528                break;
 529             case aco_opcode::v_cmp_ge_u32:
 530                op = aco_opcode::v_cmp_le_u32;
 531                break;
 532             case aco_opcode::v_cmp_lt_f64:
 533                op = aco_opcode::v_cmp_gt_f64;
 534                break;
 535             case aco_opcode::v_cmp_ge_f64:
 536                op = aco_opcode::v_cmp_le_f64;
 537                break;
 538             case aco_opcode::v_cmp_lt_i64:
 539                op = aco_opcode::v_cmp_gt_i64;
 540                break;
 541             case aco_opcode::v_cmp_ge_i64:
 542                op = aco_opcode::v_cmp_le_i64;
 543                break;
 544             case aco_opcode::v_cmp_lt_u64:
 545                op = aco_opcode::v_cmp_gt_u64;
 546                break;
 547             case aco_opcode::v_cmp_ge_u64:
 548                op = aco_opcode::v_cmp_le_u64;
 549                break;
 550             default: /* eq and ne are commutative */
 551                break;
 552          }
 553          Temp t = src0;
 554          src0 = src1;
 555          src1 = t;
 556       } else {
 557          src1 = as_vgpr(ctx, src1);
 558       }
 559    }
 560
 561    Builder bld(ctx->program, ctx->block);
 562    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 563 }
 564
 565 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 566 {
 567    Temp src0 = get_alu_src(ctx, instr->src[0]);
 568    Temp src1 = get_alu_src(ctx, instr->src[1]);
 569    Builder bld(ctx->program, ctx->block);
 570
 571    assert(dst.regClass() == bld.lm);
 572    assert(src0.type() == RegType::sgpr);
 573    assert(src1.type() == RegType::sgpr);
 574    assert(src0.regClass() == src1.regClass());
 575
 576    /* Emit the SALU comparison instruction */
 577    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 578    /* Turn the result into a per-lane bool */
 579    bool_to_vector_condition(ctx, cmp, dst);
 580 }
 581
 582 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 583                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 584 {
 585    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 586    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 587    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 588    bool use_valu = s_op == aco_opcode::num_opcodes ||
 589                    divergent_vals ||
 590                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 591                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 592    aco_opcode op = use_valu ? v_op : s_op;
 593    assert(op != aco_opcode::num_opcodes);
 594    assert(dst.regClass() == ctx->program->lane_mask);
 595
 596    if (use_valu)
 597       emit_vopc_instruction(ctx, instr, op, dst);
 598    else
 599       emit_sopc_instruction(ctx, instr, op, dst);
 600 }
 601
 602 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 603 {
 604    Builder bld(ctx->program, ctx->block);
 605    Temp src0 = get_alu_src(ctx, instr->src[0]);
 606    Temp src1 = get_alu_src(ctx, instr->src[1]);
 607
 608    assert(dst.regClass() == bld.lm);
 609    assert(src0.regClass() == bld.lm);
 610    assert(src1.regClass() == bld.lm);
 611
 612    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 613 }
 614
 615 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 616 {
 617    Builder bld(ctx->program, ctx->block);
 618    Temp cond = get_alu_src(ctx, instr->src[0]);
 619    Temp then = get_alu_src(ctx, instr->src[1]);
 620    Temp els = get_alu_src(ctx, instr->src[2]);
 621
 622    assert(cond.regClass() == bld.lm);
 623
 624    if (dst.type() == RegType::vgpr) {
 625       aco_ptr<Instruction> bcsel;
 626       if (dst.size() == 1) {
 627          then = as_vgpr(ctx, then);
 628          els = as_vgpr(ctx, els);
 629
 630          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 631       } else if (dst.size() == 2) {
 632          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 633          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 634          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 635          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 636
 637          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 638          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 639
 640          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 641       } else {
 642          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 643          nir_print_instr(&instr->instr, stderr);
 644          fprintf(stderr, "\n");
 645       }
 646       return;
 647    }
 648
 649    if (instr->dest.dest.ssa.bit_size == 1) {
 650       assert(dst.regClass() == bld.lm);
 651       assert(then.regClass() == bld.lm);
 652       assert(els.regClass() == bld.lm);
 653    }
 654
 655    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 656       if (dst.regClass() == s1 || dst.regClass() == s2) {
 657          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 658          assert(dst.size() == then.size());
 659          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 660          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 661       } else {
 662          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 663          nir_print_instr(&instr->instr, stderr);
 664          fprintf(stderr, "\n");
 665       }
 666       return;
 667    }
 668
 669    /* divergent boolean bcsel
 670     * this implements bcsel on bools: dst = s0 ? s1 : s2
 671     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 672    assert(instr->dest.dest.ssa.bit_size == 1);
 673
 674    if (cond.id() != then.id())
 675       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 676
 677    if (cond.id() == els.id())
 678       bld.sop1(Builder::s_mov, Definition(dst), then);
 679    else
 680       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 681                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 682 }
 683
 684 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 685                     aco_opcode op, uint32_t undo)
 686 {
 687    /* multiply by 16777216 to handle denormals */
 688    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 689                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 690    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 691    scaled = bld.vop1(op, bld.def(v1), scaled);
 692    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 693
 694    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 695
 696    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 697 }
 698
 699 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 700 {
 701    if (ctx->block->fp_mode.denorm32 == 0) {
 702       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 703       return;
 704    }
 705
 706    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 707 }
 708
 709 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 710 {
 711    if (ctx->block->fp_mode.denorm32 == 0) {
 712       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 713       return;
 714    }
 715
 716    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 717 }
 718
 719 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 720 {
 721    if (ctx->block->fp_mode.denorm32 == 0) {
 722       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 723       return;
 724    }
 725
 726    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 727 }
 728
 729 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 730 {
 731    if (ctx->block->fp_mode.denorm32 == 0) {
 732       bld.vop1(aco_opcode::v_log_f32, dst, val);
 733       return;
 734    }
 735
 736    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 737 }
 738
 739 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 740 {
 741    if (!instr->dest.dest.is_ssa) {
 742       fprintf(stderr, "nir alu dst not in ssa: ");
 743       nir_print_instr(&instr->instr, stderr);
 744       fprintf(stderr, "\n");
 745       abort();
 746    }
 747    Builder bld(ctx->program, ctx->block);
 748    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 749    switch(instr->op) {
 750    case nir_op_vec2:
 751    case nir_op_vec3:
 752    case nir_op_vec4: {
 753       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 754       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 755       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 756          elems[i] = get_alu_src(ctx, instr->src[i]);
 757          vec->operands[i] = Operand{elems[i]};
 758       }
 759       vec->definitions[0] = Definition(dst);
 760       ctx->block->instructions.emplace_back(std::move(vec));
 761       ctx->allocated_vec.emplace(dst.id(), elems);
 762       break;
 763    }
 764    case nir_op_mov: {
 765       Temp src = get_alu_src(ctx, instr->src[0]);
 766       aco_ptr<Instruction> mov;
 767       if (dst.type() == RegType::sgpr) {
 768          if (src.type() == RegType::vgpr)
 769             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 770          else if (src.regClass() == s1)
 771             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 772          else if (src.regClass() == s2)
 773             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 774          else
 775             unreachable("wrong src register class for nir_op_imov");
 776       } else if (dst.regClass() == v1) {
 777          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 778       } else if (dst.regClass() == v2) {
 779          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 780       } else {
 781          nir_print_instr(&instr->instr, stderr);
 782          unreachable("Should have been lowered to scalar.");
 783       }
 784       break;
 785    }
 786    case nir_op_inot: {
 787       Temp src = get_alu_src(ctx, instr->src[0]);
 788       if (instr->dest.dest.ssa.bit_size == 1) {
 789          assert(src.regClass() == bld.lm);
 790          assert(dst.regClass() == bld.lm);
 791          bld.sop2(Builder::s_andn2, Definition(dst), bld.def(s1, scc), Operand(exec, bld.lm), src);
 792       } else if (dst.regClass() == v1) {
 793          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 794       } else if (dst.type() == RegType::sgpr) {
 795          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 796          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 797       } else {
 798          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 799          nir_print_instr(&instr->instr, stderr);
 800          fprintf(stderr, "\n");
 801       }
 802       break;
 803    }
 804    case nir_op_ineg: {
 805       Temp src = get_alu_src(ctx, instr->src[0]);
 806       if (dst.regClass() == v1) {
 807          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 808       } else if (dst.regClass() == s1) {
 809          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 810       } else if (dst.size() == 2) {
 811          Temp src0 = bld.tmp(dst.type(), 1);
 812          Temp src1 = bld.tmp(dst.type(), 1);
 813          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 814
 815          if (dst.regClass() == s2) {
 816             Temp carry = bld.tmp(s1);
 817             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 818             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 819             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 820          } else {
 821             Temp lower = bld.tmp(v1);
 822             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 823             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 824             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 825          }
 826       } else {
 827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 828          nir_print_instr(&instr->instr, stderr);
 829          fprintf(stderr, "\n");
 830       }
 831       break;
 832    }
 833    case nir_op_iabs: {
 834       if (dst.regClass() == s1) {
 835          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 836       } else if (dst.regClass() == v1) {
 837          Temp src = get_alu_src(ctx, instr->src[0]);
 838          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 839       } else {
 840          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 841          nir_print_instr(&instr->instr, stderr);
 842          fprintf(stderr, "\n");
 843       }
 844       break;
 845    }
 846    case nir_op_isign: {
 847       Temp src = get_alu_src(ctx, instr->src[0]);
 848       if (dst.regClass() == s1) {
 849          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 850          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 851          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 852       } else if (dst.regClass() == s2) {
 853          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 854          Temp neqz;
 855          if (ctx->program->chip_class >= GFX8)
 856             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 857          else
 858             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 859          /* SCC gets zero-extended to 64 bit */
 860          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 861       } else if (dst.regClass() == v1) {
 862          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 863          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 864          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 865       } else if (dst.regClass() == v2) {
 866          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 867          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 868          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 869          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 870          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 871          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 872       } else {
 873          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 874          nir_print_instr(&instr->instr, stderr);
 875          fprintf(stderr, "\n");
 876       }
 877       break;
 878    }
 879    case nir_op_imax: {
 880       if (dst.regClass() == v1) {
 881          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 882       } else if (dst.regClass() == s1) {
 883          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 884       } else {
 885          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 886          nir_print_instr(&instr->instr, stderr);
 887          fprintf(stderr, "\n");
 888       }
 889       break;
 890    }
 891    case nir_op_umax: {
 892       if (dst.regClass() == v1) {
 893          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 894       } else if (dst.regClass() == s1) {
 895          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 896       } else {
 897          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 898          nir_print_instr(&instr->instr, stderr);
 899          fprintf(stderr, "\n");
 900       }
 901       break;
 902    }
 903    case nir_op_imin: {
 904       if (dst.regClass() == v1) {
 905          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 906       } else if (dst.regClass() == s1) {
 907          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 908       } else {
 909          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 910          nir_print_instr(&instr->instr, stderr);
 911          fprintf(stderr, "\n");
 912       }
 913       break;
 914    }
 915    case nir_op_umin: {
 916       if (dst.regClass() == v1) {
 917          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 918       } else if (dst.regClass() == s1) {
 919          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 920       } else {
 921          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 922          nir_print_instr(&instr->instr, stderr);
 923          fprintf(stderr, "\n");
 924       }
 925       break;
 926    }
 927    case nir_op_ior: {
 928       if (instr->dest.dest.ssa.bit_size == 1) {
 929          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 930       } else if (dst.regClass() == v1) {
 931          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 932       } else if (dst.regClass() == s1) {
 933          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 934       } else if (dst.regClass() == s2) {
 935          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 936       } else {
 937          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 938          nir_print_instr(&instr->instr, stderr);
 939          fprintf(stderr, "\n");
 940       }
 941       break;
 942    }
 943    case nir_op_iand: {
 944       if (instr->dest.dest.ssa.bit_size == 1) {
 945          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 946       } else if (dst.regClass() == v1) {
 947          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 948       } else if (dst.regClass() == s1) {
 949          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 950       } else if (dst.regClass() == s2) {
 951          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_ixor: {
 960       if (instr->dest.dest.ssa.bit_size == 1) {
 961          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 962       } else if (dst.regClass() == v1) {
 963          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 964       } else if (dst.regClass() == s1) {
 965          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 966       } else if (dst.regClass() == s2) {
 967          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 968       } else {
 969          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 970          nir_print_instr(&instr->instr, stderr);
 971          fprintf(stderr, "\n");
 972       }
 973       break;
 974    }
 975    case nir_op_ushr: {
 976       if (dst.regClass() == v1) {
 977          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 978       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 979          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 980                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 981       } else if (dst.regClass() == v2) {
 982          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
 983                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
 984       } else if (dst.regClass() == s2) {
 985          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 986       } else if (dst.regClass() == s1) {
 987          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 988       } else {
 989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 990          nir_print_instr(&instr->instr, stderr);
 991          fprintf(stderr, "\n");
 992       }
 993       break;
 994    }
 995    case nir_op_ishl: {
 996       if (dst.regClass() == v1) {
 997          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 998       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 999          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1000                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1001       } else if (dst.regClass() == v2) {
1002          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1003                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1004       } else if (dst.regClass() == s1) {
1005          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1006       } else if (dst.regClass() == s2) {
1007          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1008       } else {
1009          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1010          nir_print_instr(&instr->instr, stderr);
1011          fprintf(stderr, "\n");
1012       }
1013       break;
1014    }
1015    case nir_op_ishr: {
1016       if (dst.regClass() == v1) {
1017          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1018       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1019          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1020                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1021       } else if (dst.regClass() == v2) {
1022          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1023                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1024       } else if (dst.regClass() == s1) {
1025          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1026       } else if (dst.regClass() == s2) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1028       } else {
1029          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1030          nir_print_instr(&instr->instr, stderr);
1031          fprintf(stderr, "\n");
1032       }
1033       break;
1034    }
1035    case nir_op_find_lsb: {
1036       Temp src = get_alu_src(ctx, instr->src[0]);
1037       if (src.regClass() == s1) {
1038          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1039       } else if (src.regClass() == v1) {
1040          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1041       } else if (src.regClass() == s2) {
1042          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1043       } else {
1044          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1045          nir_print_instr(&instr->instr, stderr);
1046          fprintf(stderr, "\n");
1047       }
1048       break;
1049    }
1050    case nir_op_ufind_msb:
1051    case nir_op_ifind_msb: {
1052       Temp src = get_alu_src(ctx, instr->src[0]);
1053       if (src.regClass() == s1 || src.regClass() == s2) {
1054          aco_opcode op = src.regClass() == s2 ?
1055                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1056                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1057          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1058
1059          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1060                                         Operand(src.size() * 32u - 1u), msb_rev);
1061          Temp msb = sub.def(0).getTemp();
1062          Temp carry = sub.def(1).getTemp();
1063
1064          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1065       } else if (src.regClass() == v1) {
1066          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1067          Temp msb_rev = bld.tmp(v1);
1068          emit_vop1_instruction(ctx, instr, op, msb_rev);
1069          Temp msb = bld.tmp(v1);
1070          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1071          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1072       } else {
1073          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1074          nir_print_instr(&instr->instr, stderr);
1075          fprintf(stderr, "\n");
1076       }
1077       break;
1078    }
1079    case nir_op_bitfield_reverse: {
1080       if (dst.regClass() == s1) {
1081          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1082       } else if (dst.regClass() == v1) {
1083          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1084       } else {
1085          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1086          nir_print_instr(&instr->instr, stderr);
1087          fprintf(stderr, "\n");
1088       }
1089       break;
1090    }
1091    case nir_op_iadd: {
1092       if (dst.regClass() == s1) {
1093          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1094          break;
1095       }
1096
1097       Temp src0 = get_alu_src(ctx, instr->src[0]);
1098       Temp src1 = get_alu_src(ctx, instr->src[1]);
1099       if (dst.regClass() == v1) {
1100          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1101          break;
1102       }
1103
1104       assert(src0.size() == 2 && src1.size() == 2);
1105       Temp src00 = bld.tmp(src0.type(), 1);
1106       Temp src01 = bld.tmp(dst.type(), 1);
1107       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1108       Temp src10 = bld.tmp(src1.type(), 1);
1109       Temp src11 = bld.tmp(dst.type(), 1);
1110       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1111
1112       if (dst.regClass() == s2) {
1113          Temp carry = bld.tmp(s1);
1114          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1115          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1116          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1117       } else if (dst.regClass() == v2) {
1118          Temp dst0 = bld.tmp(v1);
1119          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1120          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1121          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1122       } else {
1123          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1124          nir_print_instr(&instr->instr, stderr);
1125          fprintf(stderr, "\n");
1126       }
1127       break;
1128    }
1129    case nir_op_uadd_sat: {
1130       Temp src0 = get_alu_src(ctx, instr->src[0]);
1131       Temp src1 = get_alu_src(ctx, instr->src[1]);
1132       if (dst.regClass() == s1) {
1133          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1134          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1135                   src0, src1);
1136          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1137       } else if (dst.regClass() == v1) {
1138          if (ctx->options->chip_class >= GFX9) {
1139             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1140             add->operands[0] = Operand(src0);
1141             add->operands[1] = Operand(src1);
1142             add->definitions[0] = Definition(dst);
1143             add->clamp = 1;
1144             ctx->block->instructions.emplace_back(std::move(add));
1145          } else {
1146             if (src1.regClass() != v1)
1147                std::swap(src0, src1);
1148             assert(src1.regClass() == v1);
1149             Temp tmp = bld.tmp(v1);
1150             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1151             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1152          }
1153       } else {
1154          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1155          nir_print_instr(&instr->instr, stderr);
1156          fprintf(stderr, "\n");
1157       }
1158       break;
1159    }
1160    case nir_op_uadd_carry: {
1161       Temp src0 = get_alu_src(ctx, instr->src[0]);
1162       Temp src1 = get_alu_src(ctx, instr->src[1]);
1163       if (dst.regClass() == s1) {
1164          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1165          break;
1166       }
1167       if (dst.regClass() == v1) {
1168          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1169          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1170          break;
1171       }
1172
1173       Temp src00 = bld.tmp(src0.type(), 1);
1174       Temp src01 = bld.tmp(dst.type(), 1);
1175       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1176       Temp src10 = bld.tmp(src1.type(), 1);
1177       Temp src11 = bld.tmp(dst.type(), 1);
1178       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1179       if (dst.regClass() == s2) {
1180          Temp carry = bld.tmp(s1);
1181          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1182          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1183          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1184       } else if (dst.regClass() == v2) {
1185          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1186          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1187          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1188          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1189       } else {
1190          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1191          nir_print_instr(&instr->instr, stderr);
1192          fprintf(stderr, "\n");
1193       }
1194       break;
1195    }
1196    case nir_op_isub: {
1197       if (dst.regClass() == s1) {
1198          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1199          break;
1200       }
1201
1202       Temp src0 = get_alu_src(ctx, instr->src[0]);
1203       Temp src1 = get_alu_src(ctx, instr->src[1]);
1204       if (dst.regClass() == v1) {
1205          bld.vsub32(Definition(dst), src0, src1);
1206          break;
1207       }
1208
1209       Temp src00 = bld.tmp(src0.type(), 1);
1210       Temp src01 = bld.tmp(dst.type(), 1);
1211       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1212       Temp src10 = bld.tmp(src1.type(), 1);
1213       Temp src11 = bld.tmp(dst.type(), 1);
1214       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1215       if (dst.regClass() == s2) {
1216          Temp carry = bld.tmp(s1);
1217          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1218          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1219          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1220       } else if (dst.regClass() == v2) {
1221          Temp lower = bld.tmp(v1);
1222          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1223          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1224          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1225       } else {
1226          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1227          nir_print_instr(&instr->instr, stderr);
1228          fprintf(stderr, "\n");
1229       }
1230       break;
1231    }
1232    case nir_op_usub_borrow: {
1233       Temp src0 = get_alu_src(ctx, instr->src[0]);
1234       Temp src1 = get_alu_src(ctx, instr->src[1]);
1235       if (dst.regClass() == s1) {
1236          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1237          break;
1238       } else if (dst.regClass() == v1) {
1239          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1240          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1241          break;
1242       }
1243
1244       Temp src00 = bld.tmp(src0.type(), 1);
1245       Temp src01 = bld.tmp(dst.type(), 1);
1246       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1247       Temp src10 = bld.tmp(src1.type(), 1);
1248       Temp src11 = bld.tmp(dst.type(), 1);
1249       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1250       if (dst.regClass() == s2) {
1251          Temp borrow = bld.tmp(s1);
1252          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1253          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1254          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1255       } else if (dst.regClass() == v2) {
1256          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1257          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1258          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1259          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1260       } else {
1261          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1262          nir_print_instr(&instr->instr, stderr);
1263          fprintf(stderr, "\n");
1264       }
1265       break;
1266    }
1267    case nir_op_imul: {
1268       if (dst.regClass() == v1) {
1269          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1270                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1271       } else if (dst.regClass() == s1) {
1272          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1273       } else {
1274          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1275          nir_print_instr(&instr->instr, stderr);
1276          fprintf(stderr, "\n");
1277       }
1278       break;
1279    }
1280    case nir_op_umul_high: {
1281       if (dst.regClass() == v1) {
1282          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1283       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1284          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1285       } else if (dst.regClass() == s1) {
1286          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1287                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1288          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1289       } else {
1290          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1291          nir_print_instr(&instr->instr, stderr);
1292          fprintf(stderr, "\n");
1293       }
1294       break;
1295    }
1296    case nir_op_imul_high: {
1297       if (dst.regClass() == v1) {
1298          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1299       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1300          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1301       } else if (dst.regClass() == s1) {
1302          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1303                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1304          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1305       } else {
1306          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1307          nir_print_instr(&instr->instr, stderr);
1308          fprintf(stderr, "\n");
1309       }
1310       break;
1311    }
1312    case nir_op_fmul: {
1313       if (dst.size() == 1) {
1314          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1315       } else if (dst.size() == 2) {
1316          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1317                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1318       } else {
1319          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1320          nir_print_instr(&instr->instr, stderr);
1321          fprintf(stderr, "\n");
1322       }
1323       break;
1324    }
1325    case nir_op_fadd: {
1326       if (dst.size() == 1) {
1327          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1328       } else if (dst.size() == 2) {
1329          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1330                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1331       } else {
1332          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1333          nir_print_instr(&instr->instr, stderr);
1334          fprintf(stderr, "\n");
1335       }
1336       break;
1337    }
1338    case nir_op_fsub: {
1339       Temp src0 = get_alu_src(ctx, instr->src[0]);
1340       Temp src1 = get_alu_src(ctx, instr->src[1]);
1341       if (dst.size() == 1) {
1342          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1343             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1344          else
1345             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1346       } else if (dst.size() == 2) {
1347          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1348                                      get_alu_src(ctx, instr->src[0]),
1349                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1350          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1351          sub->neg[1] = true;
1352       } else {
1353          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1354          nir_print_instr(&instr->instr, stderr);
1355          fprintf(stderr, "\n");
1356       }
1357       break;
1358    }
1359    case nir_op_fmax: {
1360       if (dst.size() == 1) {
1361          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1362       } else if (dst.size() == 2) {
1363          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1364             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1365                                 get_alu_src(ctx, instr->src[0]),
1366                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1367             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1368          } else {
1369             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1370                      get_alu_src(ctx, instr->src[0]),
1371                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1372          }
1373       } else {
1374          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1375          nir_print_instr(&instr->instr, stderr);
1376          fprintf(stderr, "\n");
1377       }
1378       break;
1379    }
1380    case nir_op_fmin: {
1381       if (dst.size() == 1) {
1382          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1383       } else if (dst.size() == 2) {
1384          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1385             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1386                                 get_alu_src(ctx, instr->src[0]),
1387                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1388             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1389          } else {
1390             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1391                      get_alu_src(ctx, instr->src[0]),
1392                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1393          }
1394       } else {
1395          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1396          nir_print_instr(&instr->instr, stderr);
1397          fprintf(stderr, "\n");
1398       }
1399       break;
1400    }
1401    case nir_op_fmax3: {
1402       if (dst.size() == 1) {
1403          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1404       } else {
1405          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1406          nir_print_instr(&instr->instr, stderr);
1407          fprintf(stderr, "\n");
1408       }
1409       break;
1410    }
1411    case nir_op_fmin3: {
1412       if (dst.size() == 1) {
1413          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1414       } else {
1415          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1416          nir_print_instr(&instr->instr, stderr);
1417          fprintf(stderr, "\n");
1418       }
1419       break;
1420    }
1421    case nir_op_fmed3: {
1422       if (dst.size() == 1) {
1423          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1424       } else {
1425          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1426          nir_print_instr(&instr->instr, stderr);
1427          fprintf(stderr, "\n");
1428       }
1429       break;
1430    }
1431    case nir_op_umax3: {
1432       if (dst.size() == 1) {
1433          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1434       } else {
1435          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1436          nir_print_instr(&instr->instr, stderr);
1437          fprintf(stderr, "\n");
1438       }
1439       break;
1440    }
1441    case nir_op_umin3: {
1442       if (dst.size() == 1) {
1443          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1444       } else {
1445          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1446          nir_print_instr(&instr->instr, stderr);
1447          fprintf(stderr, "\n");
1448       }
1449       break;
1450    }
1451    case nir_op_umed3: {
1452       if (dst.size() == 1) {
1453          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1454       } else {
1455          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1456          nir_print_instr(&instr->instr, stderr);
1457          fprintf(stderr, "\n");
1458       }
1459       break;
1460    }
1461    case nir_op_imax3: {
1462       if (dst.size() == 1) {
1463          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1464       } else {
1465          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1466          nir_print_instr(&instr->instr, stderr);
1467          fprintf(stderr, "\n");
1468       }
1469       break;
1470    }
1471    case nir_op_imin3: {
1472       if (dst.size() == 1) {
1473          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_imed3: {
1482       if (dst.size() == 1) {
1483          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_cube_face_coord: {
1492       Temp in = get_alu_src(ctx, instr->src[0], 3);
1493       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1494                       emit_extract_vector(ctx, in, 1, v1),
1495                       emit_extract_vector(ctx, in, 2, v1) };
1496       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1497       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1498       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1499       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1500       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1501       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1502       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1503       break;
1504    }
1505    case nir_op_cube_face_index: {
1506       Temp in = get_alu_src(ctx, instr->src[0], 3);
1507       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1508                       emit_extract_vector(ctx, in, 1, v1),
1509                       emit_extract_vector(ctx, in, 2, v1) };
1510       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1511       break;
1512    }
1513    case nir_op_bcsel: {
1514       emit_bcsel(ctx, instr, dst);
1515       break;
1516    }
1517    case nir_op_frsq: {
1518       if (dst.size() == 1) {
1519          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1520       } else if (dst.size() == 2) {
1521          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1522       } else {
1523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1524          nir_print_instr(&instr->instr, stderr);
1525          fprintf(stderr, "\n");
1526       }
1527       break;
1528    }
1529    case nir_op_fneg: {
1530       Temp src = get_alu_src(ctx, instr->src[0]);
1531       if (dst.size() == 1) {
1532          if (ctx->block->fp_mode.must_flush_denorms32)
1533             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1534          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1535       } else if (dst.size() == 2) {
1536          if (ctx->block->fp_mode.must_flush_denorms16_64)
1537             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1538          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1539          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1540          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1541          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_fabs: {
1550       Temp src = get_alu_src(ctx, instr->src[0]);
1551       if (dst.size() == 1) {
1552          if (ctx->block->fp_mode.must_flush_denorms32)
1553             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1554          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1555       } else if (dst.size() == 2) {
1556          if (ctx->block->fp_mode.must_flush_denorms16_64)
1557             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1558          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1559          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1560          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1561          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1562       } else {
1563          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1564          nir_print_instr(&instr->instr, stderr);
1565          fprintf(stderr, "\n");
1566       }
1567       break;
1568    }
1569    case nir_op_fsat: {
1570       Temp src = get_alu_src(ctx, instr->src[0]);
1571       if (dst.size() == 1) {
1572          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1573          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1574          // TODO: confirm that this holds under any circumstances
1575       } else if (dst.size() == 2) {
1576          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1577          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1578          vop3->clamp = true;
1579       } else {
1580          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1581          nir_print_instr(&instr->instr, stderr);
1582          fprintf(stderr, "\n");
1583       }
1584       break;
1585    }
1586    case nir_op_flog2: {
1587       if (dst.size() == 1) {
1588          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_frcp: {
1597       if (dst.size() == 1) {
1598          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1599       } else if (dst.size() == 2) {
1600          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1601       } else {
1602          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1603          nir_print_instr(&instr->instr, stderr);
1604          fprintf(stderr, "\n");
1605       }
1606       break;
1607    }
1608    case nir_op_fexp2: {
1609       if (dst.size() == 1) {
1610          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1611       } else {
1612          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1613          nir_print_instr(&instr->instr, stderr);
1614          fprintf(stderr, "\n");
1615       }
1616       break;
1617    }
1618    case nir_op_fsqrt: {
1619       if (dst.size() == 1) {
1620          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1621       } else if (dst.size() == 2) {
1622          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1623       } else {
1624          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1625          nir_print_instr(&instr->instr, stderr);
1626          fprintf(stderr, "\n");
1627       }
1628       break;
1629    }
1630    case nir_op_ffract: {
1631       if (dst.size() == 1) {
1632          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1633       } else if (dst.size() == 2) {
1634          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1635       } else {
1636          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1637          nir_print_instr(&instr->instr, stderr);
1638          fprintf(stderr, "\n");
1639       }
1640       break;
1641    }
1642    case nir_op_ffloor: {
1643       if (dst.size() == 1) {
1644          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1645       } else if (dst.size() == 2) {
1646          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1647       } else {
1648          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1649          nir_print_instr(&instr->instr, stderr);
1650          fprintf(stderr, "\n");
1651       }
1652       break;
1653    }
1654    case nir_op_fceil: {
1655       if (dst.size() == 1) {
1656          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1657       } else if (dst.size() == 2) {
1658          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_ftrunc: {
1667       if (dst.size() == 1) {
1668          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1669       } else if (dst.size() == 2) {
1670          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1671       } else {
1672          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1673          nir_print_instr(&instr->instr, stderr);
1674          fprintf(stderr, "\n");
1675       }
1676       break;
1677    }
1678    case nir_op_fround_even: {
1679       if (dst.size() == 1) {
1680          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1681       } else if (dst.size() == 2) {
1682          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1683       } else {
1684          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1685          nir_print_instr(&instr->instr, stderr);
1686          fprintf(stderr, "\n");
1687       }
1688       break;
1689    }
1690    case nir_op_fsin:
1691    case nir_op_fcos: {
1692       Temp src = get_alu_src(ctx, instr->src[0]);
1693       aco_ptr<Instruction> norm;
1694       if (dst.size() == 1) {
1695          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1696          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1697
1698          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1699          if (ctx->options->chip_class < GFX9)
1700             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1701
1702          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1703          bld.vop1(opcode, Definition(dst), tmp);
1704       } else {
1705          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1706          nir_print_instr(&instr->instr, stderr);
1707          fprintf(stderr, "\n");
1708       }
1709       break;
1710    }
1711    case nir_op_ldexp: {
1712       if (dst.size() == 1) {
1713          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1714                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1715                   get_alu_src(ctx, instr->src[1]));
1716       } else if (dst.size() == 2) {
1717          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1718                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1719                   get_alu_src(ctx, instr->src[1]));
1720       } else {
1721          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1722          nir_print_instr(&instr->instr, stderr);
1723          fprintf(stderr, "\n");
1724       }
1725       break;
1726    }
1727    case nir_op_frexp_sig: {
1728       if (dst.size() == 1) {
1729          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1730                   get_alu_src(ctx, instr->src[0]));
1731       } else if (dst.size() == 2) {
1732          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1733                   get_alu_src(ctx, instr->src[0]));
1734       } else {
1735          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1736          nir_print_instr(&instr->instr, stderr);
1737          fprintf(stderr, "\n");
1738       }
1739       break;
1740    }
1741    case nir_op_frexp_exp: {
1742       if (instr->src[0].src.ssa->bit_size == 32) {
1743          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1744                   get_alu_src(ctx, instr->src[0]));
1745       } else if (instr->src[0].src.ssa->bit_size == 64) {
1746          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1747                   get_alu_src(ctx, instr->src[0]));
1748       } else {
1749          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1750          nir_print_instr(&instr->instr, stderr);
1751          fprintf(stderr, "\n");
1752       }
1753       break;
1754    }
1755    case nir_op_fsign: {
1756       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1757       if (dst.size() == 1) {
1758          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1759          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1760          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1761          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1762       } else if (dst.size() == 2) {
1763          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1764          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1765          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1766
1767          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1768          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1769          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1770
1771          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1772       } else {
1773          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1774          nir_print_instr(&instr->instr, stderr);
1775          fprintf(stderr, "\n");
1776       }
1777       break;
1778    }
1779    case nir_op_f2f32: {
1780       if (instr->src[0].src.ssa->bit_size == 64) {
1781          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1782       } else {
1783          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1784          nir_print_instr(&instr->instr, stderr);
1785          fprintf(stderr, "\n");
1786       }
1787       break;
1788    }
1789    case nir_op_f2f64: {
1790       if (instr->src[0].src.ssa->bit_size == 32) {
1791          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1792       } else {
1793          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1794          nir_print_instr(&instr->instr, stderr);
1795          fprintf(stderr, "\n");
1796       }
1797       break;
1798    }
1799    case nir_op_i2f32: {
1800       assert(dst.size() == 1);
1801       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1802       break;
1803    }
1804    case nir_op_i2f64: {
1805       if (instr->src[0].src.ssa->bit_size == 32) {
1806          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1807       } else if (instr->src[0].src.ssa->bit_size == 64) {
1808          Temp src = get_alu_src(ctx, instr->src[0]);
1809          RegClass rc = RegClass(src.type(), 1);
1810          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1811          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1812          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1813          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1814          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1815          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1816
1817       } else {
1818          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1819          nir_print_instr(&instr->instr, stderr);
1820          fprintf(stderr, "\n");
1821       }
1822       break;
1823    }
1824    case nir_op_u2f32: {
1825       assert(dst.size() == 1);
1826       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1827       break;
1828    }
1829    case nir_op_u2f64: {
1830       if (instr->src[0].src.ssa->bit_size == 32) {
1831          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1832       } else if (instr->src[0].src.ssa->bit_size == 64) {
1833          Temp src = get_alu_src(ctx, instr->src[0]);
1834          RegClass rc = RegClass(src.type(), 1);
1835          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1836          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1837          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1838          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1839          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1840          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1841       } else {
1842          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1843          nir_print_instr(&instr->instr, stderr);
1844          fprintf(stderr, "\n");
1845       }
1846       break;
1847    }
1848    case nir_op_f2i32: {
1849       Temp src = get_alu_src(ctx, instr->src[0]);
1850       if (instr->src[0].src.ssa->bit_size == 32) {
1851          if (dst.type() == RegType::vgpr)
1852             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1853          else
1854             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1855                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1856
1857       } else if (instr->src[0].src.ssa->bit_size == 64) {
1858          if (dst.type() == RegType::vgpr)
1859             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1860          else
1861             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1862                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1863
1864       } else {
1865          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1866          nir_print_instr(&instr->instr, stderr);
1867          fprintf(stderr, "\n");
1868       }
1869       break;
1870    }
1871    case nir_op_f2u32: {
1872       Temp src = get_alu_src(ctx, instr->src[0]);
1873       if (instr->src[0].src.ssa->bit_size == 32) {
1874          if (dst.type() == RegType::vgpr)
1875             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1876          else
1877             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1878                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1879
1880       } else if (instr->src[0].src.ssa->bit_size == 64) {
1881          if (dst.type() == RegType::vgpr)
1882             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1883          else
1884             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1885                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1886
1887       } else {
1888          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1889          nir_print_instr(&instr->instr, stderr);
1890          fprintf(stderr, "\n");
1891       }
1892       break;
1893    }
1894    case nir_op_f2i64: {
1895       Temp src = get_alu_src(ctx, instr->src[0]);
1896       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1897          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1898          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1899          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1900          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1901          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1902          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1903          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1904          Temp new_exponent = bld.tmp(v1);
1905          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1906          if (ctx->program->chip_class >= GFX8)
1907             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1908          else
1909             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
1910          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1911          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1912          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1913          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1914          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1915          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1916          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1917          Temp new_lower = bld.tmp(v1);
1918          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1919          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1920          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1921
1922       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1923          if (src.type() == RegType::vgpr)
1924             src = bld.as_uniform(src);
1925          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1926          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1927          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1928          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1929          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1930          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1931          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1932          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1933          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1934          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1935          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1936          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1937          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1938          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1939          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1940          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1941          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1942          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1943          Temp borrow = bld.tmp(s1);
1944          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1945          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1946          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1947
1948       } else if (instr->src[0].src.ssa->bit_size == 64) {
1949          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1950          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1951          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1952          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1953          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1954          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1955          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1956          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1957          if (dst.type() == RegType::sgpr) {
1958             lower = bld.as_uniform(lower);
1959             upper = bld.as_uniform(upper);
1960          }
1961          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1962
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_f2u64: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1973          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1974          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1975          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1976          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1977          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1978          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1979          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1980          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1981          Temp new_exponent = bld.tmp(v1);
1982          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1983          if (ctx->program->chip_class >= GFX8)
1984             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1985          else
1986             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
1987          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1988          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1989          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1990          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1991          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1992          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1993          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1994
1995       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1996          if (src.type() == RegType::vgpr)
1997             src = bld.as_uniform(src);
1998          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1999          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2000          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2001          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2002          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2003          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2004          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2005          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2006          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2007          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2008          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2009          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2010          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2011          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2012          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2013          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2014          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2015          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2016
2017       } else if (instr->src[0].src.ssa->bit_size == 64) {
2018          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2019          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
2020          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2021          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2022          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
2023          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2024          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2025          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2026          if (dst.type() == RegType::sgpr) {
2027             lower = bld.as_uniform(lower);
2028             upper = bld.as_uniform(upper);
2029          }
2030          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2031
2032       } else {
2033          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2034          nir_print_instr(&instr->instr, stderr);
2035          fprintf(stderr, "\n");
2036       }
2037       break;
2038    }
2039    case nir_op_b2f32: {
2040       Temp src = get_alu_src(ctx, instr->src[0]);
2041       assert(src.regClass() == bld.lm);
2042
2043       if (dst.regClass() == s1) {
2044          src = bool_to_scalar_condition(ctx, src);
2045          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2046       } else if (dst.regClass() == v1) {
2047          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2048       } else {
2049          unreachable("Wrong destination register class for nir_op_b2f32.");
2050       }
2051       break;
2052    }
2053    case nir_op_b2f64: {
2054       Temp src = get_alu_src(ctx, instr->src[0]);
2055       assert(src.regClass() == bld.lm);
2056
2057       if (dst.regClass() == s2) {
2058          src = bool_to_scalar_condition(ctx, src);
2059          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2060       } else if (dst.regClass() == v2) {
2061          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2062          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2063          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2064       } else {
2065          unreachable("Wrong destination register class for nir_op_b2f64.");
2066       }
2067       break;
2068    }
2069    case nir_op_i2i32: {
2070       Temp src = get_alu_src(ctx, instr->src[0]);
2071       if (instr->src[0].src.ssa->bit_size == 64) {
2072          /* we can actually just say dst = src, as it would map the lower register */
2073          emit_extract_vector(ctx, src, 0, dst);
2074       } else {
2075          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2076          nir_print_instr(&instr->instr, stderr);
2077          fprintf(stderr, "\n");
2078       }
2079       break;
2080    }
2081    case nir_op_u2u32: {
2082       Temp src = get_alu_src(ctx, instr->src[0]);
2083       if (instr->src[0].src.ssa->bit_size == 16) {
2084          if (dst.regClass() == s1) {
2085             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2086          } else {
2087             // TODO: do better with SDWA
2088             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2089          }
2090       } else if (instr->src[0].src.ssa->bit_size == 64) {
2091          /* we can actually just say dst = src, as it would map the lower register */
2092          emit_extract_vector(ctx, src, 0, dst);
2093       } else {
2094          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2095          nir_print_instr(&instr->instr, stderr);
2096          fprintf(stderr, "\n");
2097       }
2098       break;
2099    }
2100    case nir_op_i2i64: {
2101       Temp src = get_alu_src(ctx, instr->src[0]);
2102       if (src.regClass() == s1) {
2103          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2104          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2105       } else if (src.regClass() == v1) {
2106          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2107          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2108       } else {
2109          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2110          nir_print_instr(&instr->instr, stderr);
2111          fprintf(stderr, "\n");
2112       }
2113       break;
2114    }
2115    case nir_op_u2u64: {
2116       Temp src = get_alu_src(ctx, instr->src[0]);
2117       if (instr->src[0].src.ssa->bit_size == 32) {
2118          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2119       } else {
2120          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2121          nir_print_instr(&instr->instr, stderr);
2122          fprintf(stderr, "\n");
2123       }
2124       break;
2125    }
2126    case nir_op_b2i32: {
2127       Temp src = get_alu_src(ctx, instr->src[0]);
2128       assert(src.regClass() == bld.lm);
2129
2130       if (dst.regClass() == s1) {
2131          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2132          bool_to_scalar_condition(ctx, src, dst);
2133       } else if (dst.regClass() == v1) {
2134          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2135       } else {
2136          unreachable("Invalid register class for b2i32");
2137       }
2138       break;
2139    }
2140    case nir_op_i2b1: {
2141       Temp src = get_alu_src(ctx, instr->src[0]);
2142       assert(dst.regClass() == bld.lm);
2143
2144       if (src.type() == RegType::vgpr) {
2145          assert(src.regClass() == v1 || src.regClass() == v2);
2146          assert(dst.regClass() == bld.lm);
2147          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2148                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2149       } else {
2150          assert(src.regClass() == s1 || src.regClass() == s2);
2151          Temp tmp;
2152          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2153             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2154          } else {
2155             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2156                            bld.scc(bld.def(s1)), Operand(0u), src);
2157          }
2158          bool_to_vector_condition(ctx, tmp, dst);
2159       }
2160       break;
2161    }
2162    case nir_op_pack_64_2x32_split: {
2163       Temp src0 = get_alu_src(ctx, instr->src[0]);
2164       Temp src1 = get_alu_src(ctx, instr->src[1]);
2165
2166       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2167       break;
2168    }
2169    case nir_op_unpack_64_2x32_split_x:
2170       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2171       break;
2172    case nir_op_unpack_64_2x32_split_y:
2173       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2174       break;
2175    case nir_op_pack_half_2x16: {
2176       Temp src = get_alu_src(ctx, instr->src[0], 2);
2177
2178       if (dst.regClass() == v1) {
2179          Temp src0 = bld.tmp(v1);
2180          Temp src1 = bld.tmp(v1);
2181          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2182          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2183             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2184          else
2185             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2186                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2187                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2188       } else {
2189          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2190          nir_print_instr(&instr->instr, stderr);
2191          fprintf(stderr, "\n");
2192       }
2193       break;
2194    }
2195    case nir_op_unpack_half_2x16_split_x: {
2196       if (dst.regClass() == v1) {
2197          Builder bld(ctx->program, ctx->block);
2198          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2199       } else {
2200          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2201          nir_print_instr(&instr->instr, stderr);
2202          fprintf(stderr, "\n");
2203       }
2204       break;
2205    }
2206    case nir_op_unpack_half_2x16_split_y: {
2207       if (dst.regClass() == v1) {
2208          Builder bld(ctx->program, ctx->block);
2209          /* TODO: use SDWA here */
2210          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2211                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2212       } else {
2213          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2214          nir_print_instr(&instr->instr, stderr);
2215          fprintf(stderr, "\n");
2216       }
2217       break;
2218    }
2219    case nir_op_fquantize2f16: {
2220       Temp src = get_alu_src(ctx, instr->src[0]);
2221       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2222       Temp f32, cmp_res;
2223
2224       if (ctx->program->chip_class >= GFX8) {
2225          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2226          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2227          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2228       } else {
2229          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2230           * so compare the result and flush to 0 if it's smaller.
2231           */
2232          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2233          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2234          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2235          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2236          cmp_res = vop3->definitions[0].getTemp();
2237       }
2238
2239       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2240          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2241          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2242       } else {
2243          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2244       }
2245       break;
2246    }
2247    case nir_op_bfm: {
2248       Temp bits = get_alu_src(ctx, instr->src[0]);
2249       Temp offset = get_alu_src(ctx, instr->src[1]);
2250
2251       if (dst.regClass() == s1) {
2252          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2253       } else if (dst.regClass() == v1) {
2254          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2255       } else {
2256          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2257          nir_print_instr(&instr->instr, stderr);
2258          fprintf(stderr, "\n");
2259       }
2260       break;
2261    }
2262    case nir_op_bitfield_select: {
2263       /* (mask & insert) | (~mask & base) */
2264       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2265       Temp insert = get_alu_src(ctx, instr->src[1]);
2266       Temp base = get_alu_src(ctx, instr->src[2]);
2267
2268       /* dst = (insert & bitmask) | (base & ~bitmask) */
2269       if (dst.regClass() == s1) {
2270          aco_ptr<Instruction> sop2;
2271          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2272          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2273          Operand lhs;
2274          if (const_insert && const_bitmask) {
2275             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2276          } else {
2277             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2278             lhs = Operand(insert);
2279          }
2280
2281          Operand rhs;
2282          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2283          if (const_base && const_bitmask) {
2284             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2285          } else {
2286             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2287             rhs = Operand(base);
2288          }
2289
2290          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2291
2292       } else if (dst.regClass() == v1) {
2293          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2294             base = as_vgpr(ctx, base);
2295          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2296             insert = as_vgpr(ctx, insert);
2297
2298          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2299
2300       } else {
2301          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2302          nir_print_instr(&instr->instr, stderr);
2303          fprintf(stderr, "\n");
2304       }
2305       break;
2306    }
2307    case nir_op_ubfe:
2308    case nir_op_ibfe: {
2309       Temp base = get_alu_src(ctx, instr->src[0]);
2310       Temp offset = get_alu_src(ctx, instr->src[1]);
2311       Temp bits = get_alu_src(ctx, instr->src[2]);
2312
2313       if (dst.type() == RegType::sgpr) {
2314          Operand extract;
2315          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2316          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2317          if (const_offset && const_bits) {
2318             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2319             extract = Operand(const_extract);
2320          } else {
2321             Operand width;
2322             if (const_bits) {
2323                width = Operand(const_bits->u32 << 16);
2324             } else {
2325                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2326             }
2327             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2328          }
2329
2330          aco_opcode opcode;
2331          if (dst.regClass() == s1) {
2332             if (instr->op == nir_op_ubfe)
2333                opcode = aco_opcode::s_bfe_u32;
2334             else
2335                opcode = aco_opcode::s_bfe_i32;
2336          } else if (dst.regClass() == s2) {
2337             if (instr->op == nir_op_ubfe)
2338                opcode = aco_opcode::s_bfe_u64;
2339             else
2340                opcode = aco_opcode::s_bfe_i64;
2341          } else {
2342             unreachable("Unsupported BFE bit size");
2343          }
2344
2345          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2346
2347       } else {
2348          aco_opcode opcode;
2349          if (dst.regClass() == v1) {
2350             if (instr->op == nir_op_ubfe)
2351                opcode = aco_opcode::v_bfe_u32;
2352             else
2353                opcode = aco_opcode::v_bfe_i32;
2354          } else {
2355             unreachable("Unsupported BFE bit size");
2356          }
2357
2358          emit_vop3a_instruction(ctx, instr, opcode, dst);
2359       }
2360       break;
2361    }
2362    case nir_op_bit_count: {
2363       Temp src = get_alu_src(ctx, instr->src[0]);
2364       if (src.regClass() == s1) {
2365          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2366       } else if (src.regClass() == v1) {
2367          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2368       } else if (src.regClass() == v2) {
2369          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2370                   emit_extract_vector(ctx, src, 1, v1),
2371                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2372                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2373       } else if (src.regClass() == s2) {
2374          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2375       } else {
2376          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2377          nir_print_instr(&instr->instr, stderr);
2378          fprintf(stderr, "\n");
2379       }
2380       break;
2381    }
2382    case nir_op_flt: {
2383       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2384       break;
2385    }
2386    case nir_op_fge: {
2387       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2388       break;
2389    }
2390    case nir_op_feq: {
2391       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2392       break;
2393    }
2394    case nir_op_fne: {
2395       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2396       break;
2397    }
2398    case nir_op_ilt: {
2399       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2400       break;
2401    }
2402    case nir_op_ige: {
2403       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2404       break;
2405    }
2406    case nir_op_ieq: {
2407       if (instr->src[0].src.ssa->bit_size == 1)
2408          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2409       else
2410          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2411                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2412       break;
2413    }
2414    case nir_op_ine: {
2415       if (instr->src[0].src.ssa->bit_size == 1)
2416          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2417       else
2418          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2419                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2420       break;
2421    }
2422    case nir_op_ult: {
2423       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2424       break;
2425    }
2426    case nir_op_uge: {
2427       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2428       break;
2429    }
2430    case nir_op_fddx:
2431    case nir_op_fddy:
2432    case nir_op_fddx_fine:
2433    case nir_op_fddy_fine:
2434    case nir_op_fddx_coarse:
2435    case nir_op_fddy_coarse: {
2436       Temp src = get_alu_src(ctx, instr->src[0]);
2437       uint16_t dpp_ctrl1, dpp_ctrl2;
2438       if (instr->op == nir_op_fddx_fine) {
2439          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2440          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2441       } else if (instr->op == nir_op_fddy_fine) {
2442          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2443          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2444       } else {
2445          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2446          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2447             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2448          else
2449             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2450       }
2451
2452       Temp tmp;
2453       if (ctx->program->chip_class >= GFX8) {
2454          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2455          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2456       } else {
2457          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2458          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2459          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2460       }
2461       emit_wqm(ctx, tmp, dst, true);
2462       break;
2463    }
2464    default:
2465       fprintf(stderr, "Unknown NIR ALU instr: ");
2466       nir_print_instr(&instr->instr, stderr);
2467       fprintf(stderr, "\n");
2468    }
2469 }
2470
2471 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2472 {
2473    Temp dst = get_ssa_temp(ctx, &instr->def);
2474
2475    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2476    // which get truncated the lsb if double and msb if int
2477    // for now, we only use s_mov_b64 with 64bit inline constants
2478    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2479    assert(dst.type() == RegType::sgpr);
2480
2481    Builder bld(ctx->program, ctx->block);
2482
2483    if (instr->def.bit_size == 1) {
2484       assert(dst.regClass() == bld.lm);
2485       int val = instr->value[0].b ? -1 : 0;
2486       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2487       bld.sop1(Builder::s_mov, Definition(dst), op);
2488    } else if (dst.size() == 1) {
2489       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2490    } else {
2491       assert(dst.size() != 1);
2492       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2493       if (instr->def.bit_size == 64)
2494          for (unsigned i = 0; i < dst.size(); i++)
2495             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2496       else {
2497          for (unsigned i = 0; i < dst.size(); i++)
2498             vec->operands[i] = Operand{instr->value[i].u32};
2499       }
2500       vec->definitions[0] = Definition(dst);
2501       ctx->block->instructions.emplace_back(std::move(vec));
2502    }
2503 }
2504
2505 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2506 {
2507    uint32_t new_mask = 0;
2508    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2509       if (mask & (1u << i))
2510          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2511    return new_mask;
2512 }
2513
2514 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2515 {
2516    /* This wouldn't work inside control flow or with indirect offsets but
2517     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2518
2519    unsigned write_mask = nir_intrinsic_write_mask(instr);
2520    unsigned component = nir_intrinsic_component(instr);
2521    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2522    unsigned idx = nir_intrinsic_base(instr) + component;
2523
2524    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2525    if (off_instr->type != nir_instr_type_load_const) {
2526       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2527       nir_print_instr(off_instr, stderr);
2528       fprintf(stderr, "\n");
2529    }
2530    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2531
2532    if (instr->src[0].ssa->bit_size == 64)
2533       write_mask = widen_mask(write_mask, 2);
2534
2535    for (unsigned i = 0; i < 8; ++i) {
2536       if (write_mask & (1 << i)) {
2537          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2538          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2539       }
2540       idx++;
2541    }
2542 }
2543
2544 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2545 {
2546    Builder bld(ctx->program, ctx->block);
2547    unsigned write_mask = nir_intrinsic_write_mask(instr);
2548    Operand values[4];
2549    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2550    for (unsigned i = 0; i < 4; ++i) {
2551       if (write_mask & (1 << i)) {
2552          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2553          values[i] = Operand(tmp);
2554       } else {
2555          values[i] = Operand(v1);
2556       }
2557    }
2558
2559    unsigned index = nir_intrinsic_base(instr) / 4;
2560    unsigned target, col_format;
2561    unsigned enabled_channels = 0xF;
2562    aco_opcode compr_op = (aco_opcode)0;
2563
2564    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2565    assert(offset && "Non-const offsets on exports not yet supported");
2566    index += offset->u32;
2567
2568    assert(index != FRAG_RESULT_COLOR);
2569
2570    /* Unlike vertex shader exports, it's fine to use multiple exports to
2571     * export separate channels of one target. So shaders which export both
2572     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2573     * TODO: combine the exports in those cases and create better code
2574     */
2575
2576    if (index == FRAG_RESULT_SAMPLE_MASK) {
2577
2578       if (ctx->program->info->ps.writes_z) {
2579          target = V_008DFC_SQ_EXP_MRTZ;
2580          enabled_channels = 0x4;
2581          col_format = (unsigned) -1;
2582
2583          values[2] = values[0];
2584          values[0] = Operand(v1);
2585       } else {
2586          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2587                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2588          return;
2589       }
2590
2591    } else if (index == FRAG_RESULT_DEPTH) {
2592
2593       target = V_008DFC_SQ_EXP_MRTZ;
2594       enabled_channels = 0x1;
2595       col_format = (unsigned) -1;
2596
2597    } else if (index == FRAG_RESULT_STENCIL) {
2598
2599       if (ctx->program->info->ps.writes_z) {
2600          target = V_008DFC_SQ_EXP_MRTZ;
2601          enabled_channels = 0x2;
2602          col_format = (unsigned) -1;
2603
2604          values[1] = values[0];
2605          values[0] = Operand(v1);
2606       } else {
2607          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2608          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2609                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2610          return;
2611       }
2612
2613    } else {
2614       index -= FRAG_RESULT_DATA0;
2615       target = V_008DFC_SQ_EXP_MRT + index;
2616       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2617    }
2618    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2619    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2620
2621    switch (col_format)
2622    {
2623    case V_028714_SPI_SHADER_ZERO:
2624       enabled_channels = 0; /* writemask */
2625       target = V_008DFC_SQ_EXP_NULL;
2626       break;
2627
2628    case V_028714_SPI_SHADER_32_R:
2629       enabled_channels = 1;
2630       break;
2631
2632    case V_028714_SPI_SHADER_32_GR:
2633       enabled_channels = 0x3;
2634       break;
2635
2636    case V_028714_SPI_SHADER_32_AR:
2637       if (ctx->options->chip_class >= GFX10) {
2638          /* Special case: on GFX10, the outputs are different for 32_AR */
2639          enabled_channels = 0x3;
2640          values[1] = values[3];
2641          values[3] = Operand(v1);
2642       } else {
2643          enabled_channels = 0x9;
2644       }
2645       break;
2646
2647    case V_028714_SPI_SHADER_FP16_ABGR:
2648       enabled_channels = 0x5;
2649       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2650       break;
2651
2652    case V_028714_SPI_SHADER_UNORM16_ABGR:
2653       enabled_channels = 0x5;
2654       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2655       break;
2656
2657    case V_028714_SPI_SHADER_SNORM16_ABGR:
2658       enabled_channels = 0x5;
2659       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2660       break;
2661
2662    case V_028714_SPI_SHADER_UINT16_ABGR: {
2663       enabled_channels = 0x5;
2664       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2665       if (is_int8 || is_int10) {
2666          /* clamp */
2667          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2668          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2669
2670          for (unsigned i = 0; i < 4; i++) {
2671             if ((write_mask >> i) & 1) {
2672                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2673                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2674                                     values[i]);
2675             }
2676          }
2677       }
2678       break;
2679    }
2680
2681    case V_028714_SPI_SHADER_SINT16_ABGR:
2682       enabled_channels = 0x5;
2683       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2684       if (is_int8 || is_int10) {
2685          /* clamp */
2686          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2687          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2688          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2689          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2690
2691          for (unsigned i = 0; i < 4; i++) {
2692             if ((write_mask >> i) & 1) {
2693                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2694                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2695                                     values[i]);
2696                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2697                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2698                                     values[i]);
2699             }
2700          }
2701       }
2702       break;
2703
2704    case V_028714_SPI_SHADER_32_ABGR:
2705       enabled_channels = 0xF;
2706       break;
2707
2708    default:
2709       break;
2710    }
2711
2712    if (target == V_008DFC_SQ_EXP_NULL)
2713       return;
2714
2715    if ((bool) compr_op) {
2716       for (int i = 0; i < 2; i++) {
2717          /* check if at least one of the values to be compressed is enabled */
2718          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2719          if (enabled) {
2720             enabled_channels |= enabled << (i*2);
2721             values[i] = bld.vop3(compr_op, bld.def(v1),
2722                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2723                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2724          } else {
2725             values[i] = Operand(v1);
2726          }
2727       }
2728       values[2] = Operand(v1);
2729       values[3] = Operand(v1);
2730    } else {
2731       for (int i = 0; i < 4; i++)
2732          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2733    }
2734
2735    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2736            enabled_channels, target, (bool) compr_op);
2737 }
2738
2739 Operand load_lds_size_m0(isel_context *ctx)
2740 {
2741    /* TODO: m0 does not need to be initialized on GFX9+ */
2742    Builder bld(ctx->program, ctx->block);
2743    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2744 }
2745
2746 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2747               Temp address, unsigned base_offset, unsigned align)
2748 {
2749    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2750
2751    Builder bld(ctx->program, ctx->block);
2752
2753    Operand m = load_lds_size_m0(ctx);
2754
2755    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2756    unsigned bytes_read = 0;
2757    unsigned result_size = 0;
2758    unsigned total_bytes = num_components * elem_size_bytes;
2759    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2760
2761    while (bytes_read < total_bytes) {
2762       unsigned todo = total_bytes - bytes_read;
2763       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2764       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2765
2766       aco_opcode op = aco_opcode::last_opcode;
2767       bool read2 = false;
2768       if (todo >= 16 && aligned16) {
2769          op = aco_opcode::ds_read_b128;
2770          todo = 16;
2771       } else if (todo >= 16 && aligned8) {
2772          op = aco_opcode::ds_read2_b64;
2773          read2 = true;
2774          todo = 16;
2775       } else if (todo >= 12 && aligned16) {
2776          op = aco_opcode::ds_read_b96;
2777          todo = 12;
2778       } else if (todo >= 8 && aligned8) {
2779          op = aco_opcode::ds_read_b64;
2780          todo = 8;
2781       } else if (todo >= 8) {
2782          op = aco_opcode::ds_read2_b32;
2783          read2 = true;
2784          todo = 8;
2785       } else if (todo >= 4) {
2786          op = aco_opcode::ds_read_b32;
2787          todo = 4;
2788       } else {
2789          assert(false);
2790       }
2791       assert(todo % elem_size_bytes == 0);
2792       unsigned num_elements = todo / elem_size_bytes;
2793       unsigned offset = base_offset + bytes_read;
2794       unsigned max_offset = read2 ? 1019 : 65535;
2795
2796       Temp address_offset = address;
2797       if (offset > max_offset) {
2798          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2799          offset = bytes_read;
2800       }
2801       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2802
2803       Temp res;
2804       if (num_components == 1 && dst.type() == RegType::vgpr)
2805          res = dst;
2806       else
2807          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2808
2809       if (read2)
2810          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2811       else
2812          res = bld.ds(op, Definition(res), address_offset, m, offset);
2813
2814       if (num_components == 1) {
2815          assert(todo == total_bytes);
2816          if (dst.type() == RegType::sgpr)
2817             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2818          return;
2819       }
2820
2821       if (dst.type() == RegType::sgpr) {
2822          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2823          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2824          res = new_res;
2825       }
2826
2827       if (num_elements == 1) {
2828          result[result_size++] = res;
2829       } else {
2830          assert(res != dst && res.size() % num_elements == 0);
2831          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2832          split->operands[0] = Operand(res);
2833          for (unsigned i = 0; i < num_elements; i++)
2834             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2835          ctx->block->instructions.emplace_back(std::move(split));
2836       }
2837
2838       bytes_read += todo;
2839    }
2840
2841    assert(result_size == num_components && result_size > 1);
2842    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2843    for (unsigned i = 0; i < result_size; i++)
2844       vec->operands[i] = Operand(result[i]);
2845    vec->definitions[0] = Definition(dst);
2846    ctx->block->instructions.emplace_back(std::move(vec));
2847    ctx->allocated_vec.emplace(dst.id(), result);
2848 }
2849
2850 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2851 {
2852    if (start == 0 && size == data.size())
2853       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2854
2855    unsigned size_hint = 1;
2856    auto it = ctx->allocated_vec.find(data.id());
2857    if (it != ctx->allocated_vec.end())
2858       size_hint = it->second[0].size();
2859    if (size % size_hint || start % size_hint)
2860       size_hint = 1;
2861
2862    start /= size_hint;
2863    size /= size_hint;
2864
2865    Temp elems[size];
2866    for (unsigned i = 0; i < size; i++)
2867       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2868
2869    if (size == 1)
2870       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2871
2872    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2873    for (unsigned i = 0; i < size; i++)
2874       vec->operands[i] = Operand(elems[i]);
2875    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2876    vec->definitions[0] = Definition(res);
2877    ctx->block->instructions.emplace_back(std::move(vec));
2878    return res;
2879 }
2880
2881 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2882 {
2883    Builder bld(ctx->program, ctx->block);
2884    unsigned bytes_written = 0;
2885    while (bytes_written < total_size * 4) {
2886       unsigned todo = total_size * 4 - bytes_written;
2887       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2888       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2889
2890       aco_opcode op = aco_opcode::last_opcode;
2891       bool write2 = false;
2892       unsigned size = 0;
2893       if (todo >= 16 && aligned16) {
2894          op = aco_opcode::ds_write_b128;
2895          size = 4;
2896       } else if (todo >= 16 && aligned8) {
2897          op = aco_opcode::ds_write2_b64;
2898          write2 = true;
2899          size = 4;
2900       } else if (todo >= 12 && aligned16) {
2901          op = aco_opcode::ds_write_b96;
2902          size = 3;
2903       } else if (todo >= 8 && aligned8) {
2904          op = aco_opcode::ds_write_b64;
2905          size = 2;
2906       } else if (todo >= 8) {
2907          op = aco_opcode::ds_write2_b32;
2908          write2 = true;
2909          size = 2;
2910       } else if (todo >= 4) {
2911          op = aco_opcode::ds_write_b32;
2912          size = 1;
2913       } else {
2914          assert(false);
2915       }
2916
2917       unsigned offset = offset0 + offset1 + bytes_written;
2918       unsigned max_offset = write2 ? 1020 : 65535;
2919       Temp address_offset = address;
2920       if (offset > max_offset) {
2921          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2922          offset = offset1 + bytes_written;
2923       }
2924       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2925
2926       if (write2) {
2927          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2928          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2929          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2930       } else {
2931          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2932          bld.ds(op, address_offset, val, m, offset);
2933       }
2934
2935       bytes_written += size * 4;
2936    }
2937 }
2938
2939 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2940                Temp address, unsigned base_offset, unsigned align)
2941 {
2942    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2943
2944    Operand m = load_lds_size_m0(ctx);
2945
2946    /* we need at most two stores for 32bit variables */
2947    int start[2], count[2];
2948    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2949    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2950    assert(wrmask == 0);
2951
2952    /* one combined store is sufficient */
2953    if (count[0] == count[1]) {
2954       Builder bld(ctx->program, ctx->block);
2955
2956       Temp address_offset = address;
2957       if ((base_offset >> 2) + start[1] > 255) {
2958          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2959          base_offset = 0;
2960       }
2961
2962       assert(count[0] == 1);
2963       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2964       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2965       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2966       base_offset = base_offset / elem_size_bytes;
2967       bld.ds(op, address_offset, val0, val1, m,
2968              base_offset + start[0], base_offset + start[1]);
2969       return;
2970    }
2971
2972    for (unsigned i = 0; i < 2; i++) {
2973       if (count[i] == 0)
2974          continue;
2975
2976       unsigned elem_size_words = elem_size_bytes / 4;
2977       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2978                       base_offset, start[i] * elem_size_bytes, align);
2979    }
2980    return;
2981 }
2982
2983 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2984 {
2985    if (ctx->stage == vertex_vs) {
2986       visit_store_vs_output(ctx, instr);
2987    } else if (ctx->stage == fragment_fs) {
2988       visit_store_fs_output(ctx, instr);
2989    } else {
2990       unreachable("Shader stage not implemented");
2991    }
2992 }
2993
2994 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2995 {
2996    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2997    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2998
2999    Builder bld(ctx->program, ctx->block);
3000    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3001    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3002 }
3003
3004 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3005 {
3006    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3007    for (unsigned i = 0; i < num_components; i++)
3008       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3009    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3010       assert(num_components == 4);
3011       Builder bld(ctx->program, ctx->block);
3012       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3013    }
3014
3015    for (Operand& op : vec->operands)
3016       op = op.isUndefined() ? Operand(0u) : op;
3017
3018    vec->definitions[0] = Definition(dst);
3019    ctx->block->instructions.emplace_back(std::move(vec));
3020    emit_split_vector(ctx, dst, num_components);
3021    return;
3022 }
3023
3024 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3025 {
3026    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3027    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3028    unsigned idx = nir_intrinsic_base(instr);
3029    unsigned component = nir_intrinsic_component(instr);
3030    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3031
3032    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3033    if (offset) {
3034       assert(offset->u32 == 0);
3035    } else {
3036       /* the lower 15bit of the prim_mask contain the offset into LDS
3037        * while the upper bits contain the number of prims */
3038       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3039       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3040       Builder bld(ctx->program, ctx->block);
3041       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3042       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3043       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3044       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3045       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3046    }
3047
3048    if (instr->dest.ssa.num_components == 1) {
3049       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3050    } else {
3051       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3052       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3053       {
3054          Temp tmp = {ctx->program->allocateId(), v1};
3055          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3056          vec->operands[i] = Operand(tmp);
3057       }
3058       vec->definitions[0] = Definition(dst);
3059       ctx->block->instructions.emplace_back(std::move(vec));
3060    }
3061 }
3062
3063 unsigned get_num_channels_from_data_format(unsigned data_format)
3064 {
3065    switch (data_format) {
3066    case V_008F0C_BUF_DATA_FORMAT_8:
3067    case V_008F0C_BUF_DATA_FORMAT_16:
3068    case V_008F0C_BUF_DATA_FORMAT_32:
3069       return 1;
3070    case V_008F0C_BUF_DATA_FORMAT_8_8:
3071    case V_008F0C_BUF_DATA_FORMAT_16_16:
3072    case V_008F0C_BUF_DATA_FORMAT_32_32:
3073       return 2;
3074    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3075    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3076    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3077       return 3;
3078    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3079    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3080    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3081    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3082    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3083       return 4;
3084    default:
3085       break;
3086    }
3087
3088    return 4;
3089 }
3090
3091 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3092  * so we may need to fix it up. */
3093 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3094 {
3095    Builder bld(ctx->program, ctx->block);
3096
3097    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3098       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3099
3100    /* For the integer-like cases, do a natural sign extension.
3101     *
3102     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3103     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3104     * exponent.
3105     */
3106    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3107    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3108
3109    /* Convert back to the right type. */
3110    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3111       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3112       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3113       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3114    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3115       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3116    }
3117
3118    return alpha;
3119 }
3120
3121 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3122 {
3123    Builder bld(ctx->program, ctx->block);
3124    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3125    if (ctx->stage & sw_vs) {
3126
3127       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3128       if (off_instr->type != nir_instr_type_load_const) {
3129          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3130          nir_print_instr(off_instr, stderr);
3131          fprintf(stderr, "\n");
3132       }
3133       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3134
3135       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3136
3137       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3138       unsigned component = nir_intrinsic_component(instr);
3139       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3140       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3141       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3142       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3143
3144       unsigned dfmt = attrib_format & 0xf;
3145
3146       unsigned nfmt = (attrib_format >> 4) & 0x7;
3147       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3148       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3149       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3150       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3151       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3152       if (post_shuffle)
3153          num_channels = MAX2(num_channels, 3);
3154
3155       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3156
3157       Temp index;
3158       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3159          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3160          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3161          if (divisor) {
3162             ctx->needs_instance_id = true;
3163             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3164             if (divisor != 1) {
3165                Temp divided = bld.tmp(v1);
3166                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3167                index = bld.vadd32(bld.def(v1), start_instance, divided);
3168             } else {
3169                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3170             }
3171          } else {
3172             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3173          }
3174       } else {
3175          index = bld.vadd32(bld.def(v1),
3176                             get_arg(ctx, ctx->args->ac.base_vertex),
3177                             get_arg(ctx, ctx->args->ac.vertex_id));
3178       }
3179
3180       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3181          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3182          attrib_offset = attrib_offset % attrib_stride;
3183       }
3184
3185       Operand soffset(0u);
3186       if (attrib_offset >= 4096) {
3187          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3188          attrib_offset = 0;
3189       }
3190
3191       aco_opcode opcode;
3192       switch (num_channels) {
3193       case 1:
3194          opcode = aco_opcode::tbuffer_load_format_x;
3195          break;
3196       case 2:
3197          opcode = aco_opcode::tbuffer_load_format_xy;
3198          break;
3199       case 3:
3200          opcode = aco_opcode::tbuffer_load_format_xyz;
3201          break;
3202       case 4:
3203          opcode = aco_opcode::tbuffer_load_format_xyzw;
3204          break;
3205       default:
3206          unreachable("Unimplemented load_input vector size");
3207       }
3208
3209       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3210
3211       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3212       mubuf->operands[0] = Operand(index);
3213       mubuf->operands[1] = Operand(list);
3214       mubuf->operands[2] = soffset;
3215       mubuf->definitions[0] = Definition(tmp);
3216       mubuf->idxen = true;
3217       mubuf->can_reorder = true;
3218       mubuf->dfmt = dfmt;
3219       mubuf->nfmt = nfmt;
3220       assert(attrib_offset < 4096);
3221       mubuf->offset = attrib_offset;
3222       ctx->block->instructions.emplace_back(std::move(mubuf));
3223
3224       emit_split_vector(ctx, tmp, tmp.size());
3225
3226       if (tmp.id() != dst.id()) {
3227          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3228                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3229
3230          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3231          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3232          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3233
3234          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3235          for (unsigned i = 0; i < dst.size(); i++) {
3236             unsigned idx = i + component;
3237             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3238                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3239                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3240             } else if (idx < num_channels) {
3241                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3242             } else if (is_float && idx == 3) {
3243                vec->operands[i] = Operand(0x3f800000u);
3244             } else if (!is_float && idx == 3) {
3245                vec->operands[i] = Operand(1u);
3246             } else {
3247                vec->operands[i] = Operand(0u);
3248             }
3249          }
3250          vec->definitions[0] = Definition(dst);
3251          ctx->block->instructions.emplace_back(std::move(vec));
3252          emit_split_vector(ctx, dst, dst.size());
3253       }
3254
3255    } else if (ctx->stage == fragment_fs) {
3256       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3257       if (off_instr->type != nir_instr_type_load_const ||
3258           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3259          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3260          nir_print_instr(off_instr, stderr);
3261          fprintf(stderr, "\n");
3262       }
3263
3264       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3265       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3266       if (offset) {
3267          assert(offset->u32 == 0);
3268       } else {
3269          /* the lower 15bit of the prim_mask contain the offset into LDS
3270           * while the upper bits contain the number of prims */
3271          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3272          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3273          Builder bld(ctx->program, ctx->block);
3274          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3275          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3276          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3277          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3278          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3279       }
3280
3281       unsigned idx = nir_intrinsic_base(instr);
3282       unsigned component = nir_intrinsic_component(instr);
3283
3284       if (dst.size() == 1) {
3285          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3286       } else {
3287          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3288          for (unsigned i = 0; i < dst.size(); i++)
3289             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3290          vec->definitions[0] = Definition(dst);
3291          bld.insert(std::move(vec));
3292       }
3293
3294    } else {
3295       unreachable("Shader stage not implemented");
3296    }
3297 }
3298
3299 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3300 {
3301    if (ctx->program->info->need_indirect_descriptor_sets) {
3302       Builder bld(ctx->program, ctx->block);
3303       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3304       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3305    }
3306
3307    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3308 }
3309
3310
3311 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3312 {
3313    Builder bld(ctx->program, ctx->block);
3314    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3315    if (!ctx->divergent_vals[instr->dest.ssa.index])
3316       index = bld.as_uniform(index);
3317    unsigned desc_set = nir_intrinsic_desc_set(instr);
3318    unsigned binding = nir_intrinsic_binding(instr);
3319
3320    Temp desc_ptr;
3321    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3322    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3323    unsigned offset = layout->binding[binding].offset;
3324    unsigned stride;
3325    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3326        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3327       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3328       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3329       offset = pipeline_layout->push_constant_size + 16 * idx;
3330       stride = 16;
3331    } else {
3332       desc_ptr = load_desc_ptr(ctx, desc_set);
3333       stride = layout->binding[binding].size;
3334    }
3335
3336    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3337    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3338    if (stride != 1) {
3339       if (nir_const_index) {
3340          const_index = const_index * stride;
3341       } else if (index.type() == RegType::vgpr) {
3342          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3343          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3344       } else {
3345          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3346       }
3347    }
3348    if (offset) {
3349       if (nir_const_index) {
3350          const_index = const_index + offset;
3351       } else if (index.type() == RegType::vgpr) {
3352          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3353       } else {
3354          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3355       }
3356    }
3357
3358    if (nir_const_index && const_index == 0) {
3359       index = desc_ptr;
3360    } else if (index.type() == RegType::vgpr) {
3361       index = bld.vadd32(bld.def(v1),
3362                          nir_const_index ? Operand(const_index) : Operand(index),
3363                          Operand(desc_ptr));
3364    } else {
3365       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3366                        nir_const_index ? Operand(const_index) : Operand(index),
3367                        Operand(desc_ptr));
3368    }
3369
3370    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3371 }
3372
3373 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3374                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3375 {
3376    Builder bld(ctx->program, ctx->block);
3377
3378    unsigned num_bytes = dst.size() * 4;
3379    bool dlc = glc && ctx->options->chip_class >= GFX10;
3380
3381    aco_opcode op;
3382    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3383       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3384       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3385       unsigned const_offset = 0;
3386
3387       Temp lower = Temp();
3388       if (num_bytes > 16) {
3389          assert(num_components == 3 || num_components == 4);
3390          op = aco_opcode::buffer_load_dwordx4;
3391          lower = bld.tmp(v4);
3392          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3393          mubuf->definitions[0] = Definition(lower);
3394          mubuf->operands[0] = vaddr;
3395          mubuf->operands[1] = Operand(rsrc);
3396          mubuf->operands[2] = soffset;
3397          mubuf->offen = (offset.type() == RegType::vgpr);
3398          mubuf->glc = glc;
3399          mubuf->dlc = dlc;
3400          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3401          mubuf->can_reorder = readonly;
3402          bld.insert(std::move(mubuf));
3403          emit_split_vector(ctx, lower, 2);
3404          num_bytes -= 16;
3405          const_offset = 16;
3406       }
3407
3408       switch (num_bytes) {
3409          case 4:
3410             op = aco_opcode::buffer_load_dword;
3411             break;
3412          case 8:
3413             op = aco_opcode::buffer_load_dwordx2;
3414             break;
3415          case 12:
3416             op = aco_opcode::buffer_load_dwordx3;
3417             break;
3418          case 16:
3419             op = aco_opcode::buffer_load_dwordx4;
3420             break;
3421          default:
3422             unreachable("Load SSBO not implemented for this size.");
3423       }
3424       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3425       mubuf->operands[0] = vaddr;
3426       mubuf->operands[1] = Operand(rsrc);
3427       mubuf->operands[2] = soffset;
3428       mubuf->offen = (offset.type() == RegType::vgpr);
3429       mubuf->glc = glc;
3430       mubuf->dlc = dlc;
3431       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3432       mubuf->can_reorder = readonly;
3433       mubuf->offset = const_offset;
3434       aco_ptr<Instruction> instr = std::move(mubuf);
3435
3436       if (dst.size() > 4) {
3437          assert(lower != Temp());
3438          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3439          instr->definitions[0] = Definition(upper);
3440          bld.insert(std::move(instr));
3441          if (dst.size() == 8)
3442             emit_split_vector(ctx, upper, 2);
3443          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3444          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3445          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3446          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3447          if (dst.size() == 8)
3448             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3449       }
3450
3451       if (dst.type() == RegType::sgpr) {
3452          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3453          instr->definitions[0] = Definition(vec);
3454          bld.insert(std::move(instr));
3455          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3456       } else {
3457          instr->definitions[0] = Definition(dst);
3458          bld.insert(std::move(instr));
3459          emit_split_vector(ctx, dst, num_components);
3460       }
3461    } else {
3462       switch (num_bytes) {
3463          case 4:
3464             op = aco_opcode::s_buffer_load_dword;
3465             break;
3466          case 8:
3467             op = aco_opcode::s_buffer_load_dwordx2;
3468             break;
3469          case 12:
3470          case 16:
3471             op = aco_opcode::s_buffer_load_dwordx4;
3472             break;
3473          case 24:
3474          case 32:
3475             op = aco_opcode::s_buffer_load_dwordx8;
3476             break;
3477          default:
3478             unreachable("Load SSBO not implemented for this size.");
3479       }
3480       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3481       load->operands[0] = Operand(rsrc);
3482       load->operands[1] = Operand(bld.as_uniform(offset));
3483       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3484       load->definitions[0] = Definition(dst);
3485       load->glc = glc;
3486       load->dlc = dlc;
3487       load->barrier = readonly ? barrier_none : barrier_buffer;
3488       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3489       assert(ctx->options->chip_class >= GFX8 || !glc);
3490
3491       /* trim vector */
3492       if (dst.size() == 3) {
3493          Temp vec = bld.tmp(s4);
3494          load->definitions[0] = Definition(vec);
3495          bld.insert(std::move(load));
3496          emit_split_vector(ctx, vec, 4);
3497
3498          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3499                     emit_extract_vector(ctx, vec, 0, s1),
3500                     emit_extract_vector(ctx, vec, 1, s1),
3501                     emit_extract_vector(ctx, vec, 2, s1));
3502       } else if (dst.size() == 6) {
3503          Temp vec = bld.tmp(s8);
3504          load->definitions[0] = Definition(vec);
3505          bld.insert(std::move(load));
3506          emit_split_vector(ctx, vec, 4);
3507
3508          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3509                     emit_extract_vector(ctx, vec, 0, s2),
3510                     emit_extract_vector(ctx, vec, 1, s2),
3511                     emit_extract_vector(ctx, vec, 2, s2));
3512       } else {
3513          bld.insert(std::move(load));
3514       }
3515       emit_split_vector(ctx, dst, num_components);
3516    }
3517 }
3518
3519 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3520 {
3521    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3522    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3523
3524    Builder bld(ctx->program, ctx->block);
3525
3526    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3527    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3528    unsigned binding = nir_intrinsic_binding(idx_instr);
3529    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3530
3531    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3532       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3533                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3534                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3535                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3536       if (ctx->options->chip_class >= GFX10) {
3537          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3538                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3539                       S_008F0C_RESOURCE_LEVEL(1);
3540       } else {
3541          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3542                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3543       }
3544       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3545                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3546                                      Operand(0xFFFFFFFFu),
3547                                      Operand(desc_type));
3548       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3549                         rsrc, upper_dwords);
3550    } else {
3551       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3552       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3553    }
3554
3555    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3556 }
3557
3558 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3559 {
3560    Builder bld(ctx->program, ctx->block);
3561    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3562
3563    unsigned offset = nir_intrinsic_base(instr);
3564    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3565    if (index_cv && instr->dest.ssa.bit_size == 32) {
3566
3567       unsigned count = instr->dest.ssa.num_components;
3568       unsigned start = (offset + index_cv->u32) / 4u;
3569       start -= ctx->args->ac.base_inline_push_consts;
3570       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3571          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3572          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3573          for (unsigned i = 0; i < count; ++i) {
3574             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3575             vec->operands[i] = Operand{elems[i]};
3576          }
3577          vec->definitions[0] = Definition(dst);
3578          ctx->block->instructions.emplace_back(std::move(vec));
3579          ctx->allocated_vec.emplace(dst.id(), elems);
3580          return;
3581       }
3582    }
3583
3584    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3585    if (offset != 0) // TODO check if index != 0 as well
3586       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3587    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3588    Temp vec = dst;
3589    bool trim = false;
3590    aco_opcode op;
3591
3592    switch (dst.size()) {
3593    case 1:
3594       op = aco_opcode::s_load_dword;
3595       break;
3596    case 2:
3597       op = aco_opcode::s_load_dwordx2;
3598       break;
3599    case 3:
3600       vec = bld.tmp(s4);
3601       trim = true;
3602    case 4:
3603       op = aco_opcode::s_load_dwordx4;
3604       break;
3605    case 6:
3606       vec = bld.tmp(s8);
3607       trim = true;
3608    case 8:
3609       op = aco_opcode::s_load_dwordx8;
3610       break;
3611    default:
3612       unreachable("unimplemented or forbidden load_push_constant.");
3613    }
3614
3615    bld.smem(op, Definition(vec), ptr, index);
3616
3617    if (trim) {
3618       emit_split_vector(ctx, vec, 4);
3619       RegClass rc = dst.size() == 3 ? s1 : s2;
3620       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3621                  emit_extract_vector(ctx, vec, 0, rc),
3622                  emit_extract_vector(ctx, vec, 1, rc),
3623                  emit_extract_vector(ctx, vec, 2, rc));
3624
3625    }
3626    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3627 }
3628
3629 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3630 {
3631    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3632
3633    Builder bld(ctx->program, ctx->block);
3634
3635    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3636                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3637                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3638                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3639    if (ctx->options->chip_class >= GFX10) {
3640       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3641                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3642                    S_008F0C_RESOURCE_LEVEL(1);
3643    } else {
3644       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3645                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3646    }
3647
3648    unsigned base = nir_intrinsic_base(instr);
3649    unsigned range = nir_intrinsic_range(instr);
3650
3651    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3652    if (base && offset.type() == RegType::sgpr)
3653       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3654    else if (base && offset.type() == RegType::vgpr)
3655       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3656
3657    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3658                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3659                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3660                           Operand(desc_type));
3661
3662    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3663 }
3664
3665 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3666 {
3667    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3668       ctx->cf_info.exec_potentially_empty = true;
3669
3670    ctx->program->needs_exact = true;
3671
3672    // TODO: optimize uniform conditions
3673    Builder bld(ctx->program, ctx->block);
3674    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3675    assert(src.regClass() == bld.lm);
3676    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3677    bld.pseudo(aco_opcode::p_discard_if, src);
3678    ctx->block->kind |= block_kind_uses_discard_if;
3679    return;
3680 }
3681
3682 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3683 {
3684    Builder bld(ctx->program, ctx->block);
3685
3686    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3687       ctx->cf_info.exec_potentially_empty = true;
3688
3689    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3690                     ctx->cf_info.parent_loop.has_divergent_continue;
3691
3692    if (ctx->block->loop_nest_depth &&
3693        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3694       /* we handle discards the same way as jump instructions */
3695       append_logical_end(ctx->block);
3696
3697       /* in loops, discard behaves like break */
3698       Block *linear_target = ctx->cf_info.parent_loop.exit;
3699       ctx->block->kind |= block_kind_discard;
3700
3701       if (!divergent) {
3702          /* uniform discard - loop ends here */
3703          assert(nir_instr_is_last(&instr->instr));
3704          ctx->block->kind |= block_kind_uniform;
3705          ctx->cf_info.has_branch = true;
3706          bld.branch(aco_opcode::p_branch);
3707          add_linear_edge(ctx->block->index, linear_target);
3708          return;
3709       }
3710
3711       /* we add a break right behind the discard() instructions */
3712       ctx->block->kind |= block_kind_break;
3713       unsigned idx = ctx->block->index;
3714
3715       /* remove critical edges from linear CFG */
3716       bld.branch(aco_opcode::p_branch);
3717       Block* break_block = ctx->program->create_and_insert_block();
3718       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3719       break_block->kind |= block_kind_uniform;
3720       add_linear_edge(idx, break_block);
3721       add_linear_edge(break_block->index, linear_target);
3722       bld.reset(break_block);
3723       bld.branch(aco_opcode::p_branch);
3724
3725       Block* continue_block = ctx->program->create_and_insert_block();
3726       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3727       add_linear_edge(idx, continue_block);
3728       append_logical_start(continue_block);
3729       ctx->block = continue_block;
3730
3731       return;
3732    }
3733
3734    /* it can currently happen that NIR doesn't remove the unreachable code */
3735    if (!nir_instr_is_last(&instr->instr)) {
3736       ctx->program->needs_exact = true;
3737       /* save exec somewhere temporarily so that it doesn't get
3738        * overwritten before the discard from outer exec masks */
3739       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3740       bld.pseudo(aco_opcode::p_discard_if, cond);
3741       ctx->block->kind |= block_kind_uses_discard_if;
3742       return;
3743    }
3744
3745    /* This condition is incorrect for uniformly branched discards in a loop
3746     * predicated by a divergent condition, but the above code catches that case
3747     * and the discard would end up turning into a discard_if.
3748     * For example:
3749     * if (divergent) {
3750     *    while (...) {
3751     *       if (uniform) {
3752     *          discard;
3753     *       }
3754     *    }
3755     * }
3756     */
3757    if (!ctx->cf_info.parent_if.is_divergent) {
3758       /* program just ends here */
3759       ctx->block->kind |= block_kind_uniform;
3760       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3761               0 /* enabled mask */, 9 /* dest */,
3762               false /* compressed */, true/* done */, true /* valid mask */);
3763       bld.sopp(aco_opcode::s_endpgm);
3764       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3765    } else {
3766       ctx->block->kind |= block_kind_discard;
3767       /* branch and linear edge is added by visit_if() */
3768    }
3769 }
3770
3771 enum aco_descriptor_type {
3772    ACO_DESC_IMAGE,
3773    ACO_DESC_FMASK,
3774    ACO_DESC_SAMPLER,
3775    ACO_DESC_BUFFER,
3776    ACO_DESC_PLANE_0,
3777    ACO_DESC_PLANE_1,
3778    ACO_DESC_PLANE_2,
3779 };
3780
3781 static bool
3782 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3783    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3784       return false;
3785    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3786    return dim == ac_image_cube ||
3787           dim == ac_image_1darray ||
3788           dim == ac_image_2darray ||
3789           dim == ac_image_2darraymsaa;
3790 }
3791
3792 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3793                       enum aco_descriptor_type desc_type,
3794                       const nir_tex_instr *tex_instr, bool image, bool write)
3795 {
3796 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3797    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3798    if (it != ctx->tex_desc.end())
3799       return it->second;
3800 */
3801    Temp index = Temp();
3802    bool index_set = false;
3803    unsigned constant_index = 0;
3804    unsigned descriptor_set;
3805    unsigned base_index;
3806    Builder bld(ctx->program, ctx->block);
3807
3808    if (!deref_instr) {
3809       assert(tex_instr && !image);
3810       descriptor_set = 0;
3811       base_index = tex_instr->sampler_index;
3812    } else {
3813       while(deref_instr->deref_type != nir_deref_type_var) {
3814          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3815          if (!array_size)
3816             array_size = 1;
3817
3818          assert(deref_instr->deref_type == nir_deref_type_array);
3819          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3820          if (const_value) {
3821             constant_index += array_size * const_value->u32;
3822          } else {
3823             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3824             if (indirect.type() == RegType::vgpr)
3825                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3826
3827             if (array_size != 1)
3828                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3829
3830             if (!index_set) {
3831                index = indirect;
3832                index_set = true;
3833             } else {
3834                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3835             }
3836          }
3837
3838          deref_instr = nir_src_as_deref(deref_instr->parent);
3839       }
3840       descriptor_set = deref_instr->var->data.descriptor_set;
3841       base_index = deref_instr->var->data.binding;
3842    }
3843
3844    Temp list = load_desc_ptr(ctx, descriptor_set);
3845    list = convert_pointer_to_64_bit(ctx, list);
3846
3847    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3848    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3849    unsigned offset = binding->offset;
3850    unsigned stride = binding->size;
3851    aco_opcode opcode;
3852    RegClass type;
3853
3854    assert(base_index < layout->binding_count);
3855
3856    switch (desc_type) {
3857    case ACO_DESC_IMAGE:
3858       type = s8;
3859       opcode = aco_opcode::s_load_dwordx8;
3860       break;
3861    case ACO_DESC_FMASK:
3862       type = s8;
3863       opcode = aco_opcode::s_load_dwordx8;
3864       offset += 32;
3865       break;
3866    case ACO_DESC_SAMPLER:
3867       type = s4;
3868       opcode = aco_opcode::s_load_dwordx4;
3869       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3870          offset += radv_combined_image_descriptor_sampler_offset(binding);
3871       break;
3872    case ACO_DESC_BUFFER:
3873       type = s4;
3874       opcode = aco_opcode::s_load_dwordx4;
3875       break;
3876    case ACO_DESC_PLANE_0:
3877    case ACO_DESC_PLANE_1:
3878       type = s8;
3879       opcode = aco_opcode::s_load_dwordx8;
3880       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3881       break;
3882    case ACO_DESC_PLANE_2:
3883       type = s4;
3884       opcode = aco_opcode::s_load_dwordx4;
3885       offset += 64;
3886       break;
3887    default:
3888       unreachable("invalid desc_type\n");
3889    }
3890
3891    offset += constant_index * stride;
3892
3893    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3894       (!index_set || binding->immutable_samplers_equal)) {
3895       if (binding->immutable_samplers_equal)
3896          constant_index = 0;
3897
3898       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3899       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3900                         Operand(samplers[constant_index * 4 + 0]),
3901                         Operand(samplers[constant_index * 4 + 1]),
3902                         Operand(samplers[constant_index * 4 + 2]),
3903                         Operand(samplers[constant_index * 4 + 3]));
3904    }
3905
3906    Operand off;
3907    if (!index_set) {
3908       off = Operand(offset);
3909    } else {
3910       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3911                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3912    }
3913
3914    Temp res = bld.smem(opcode, bld.def(type), list, off);
3915
3916    if (desc_type == ACO_DESC_PLANE_2) {
3917       Temp components[8];
3918       for (unsigned i = 0; i < 8; i++)
3919          components[i] = bld.tmp(s1);
3920       bld.pseudo(aco_opcode::p_split_vector,
3921                  Definition(components[0]),
3922                  Definition(components[1]),
3923                  Definition(components[2]),
3924                  Definition(components[3]),
3925                  res);
3926
3927       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3928       bld.pseudo(aco_opcode::p_split_vector,
3929                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3930                  Definition(components[4]),
3931                  Definition(components[5]),
3932                  Definition(components[6]),
3933                  Definition(components[7]),
3934                  desc2);
3935
3936       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3937                        components[0], components[1], components[2], components[3],
3938                        components[4], components[5], components[6], components[7]);
3939    }
3940
3941    return res;
3942 }
3943
3944 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3945 {
3946    switch (dim) {
3947    case GLSL_SAMPLER_DIM_BUF:
3948       return 1;
3949    case GLSL_SAMPLER_DIM_1D:
3950       return array ? 2 : 1;
3951    case GLSL_SAMPLER_DIM_2D:
3952       return array ? 3 : 2;
3953    case GLSL_SAMPLER_DIM_MS:
3954       return array ? 4 : 3;
3955    case GLSL_SAMPLER_DIM_3D:
3956    case GLSL_SAMPLER_DIM_CUBE:
3957       return 3;
3958    case GLSL_SAMPLER_DIM_RECT:
3959    case GLSL_SAMPLER_DIM_SUBPASS:
3960       return 2;
3961    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3962       return 3;
3963    default:
3964       break;
3965    }
3966    return 0;
3967 }
3968
3969
3970 /* Adjust the sample index according to FMASK.
3971  *
3972  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3973  * which is the identity mapping. Each nibble says which physical sample
3974  * should be fetched to get that sample.
3975  *
3976  * For example, 0x11111100 means there are only 2 samples stored and
3977  * the second sample covers 3/4 of the pixel. When reading samples 0
3978  * and 1, return physical sample 0 (determined by the first two 0s
3979  * in FMASK), otherwise return physical sample 1.
3980  *
3981  * The sample index should be adjusted as follows:
3982  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3983  */
3984 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3985 {
3986    Builder bld(ctx->program, ctx->block);
3987    Temp fmask = bld.tmp(v1);
3988    unsigned dim = ctx->options->chip_class >= GFX10
3989                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3990                   : 0;
3991
3992    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3993    load->operands[0] = Operand(coords);
3994    load->operands[1] = Operand(fmask_desc_ptr);
3995    load->definitions[0] = Definition(fmask);
3996    load->glc = false;
3997    load->dlc = false;
3998    load->dmask = 0x1;
3999    load->unrm = true;
4000    load->da = da;
4001    load->dim = dim;
4002    load->can_reorder = true; /* fmask images shouldn't be modified */
4003    ctx->block->instructions.emplace_back(std::move(load));
4004
4005    Operand sample_index4;
4006    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4007       sample_index4 = Operand(sample_index.constantValue() << 2);
4008    } else if (sample_index.regClass() == s1) {
4009       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4010    } else {
4011       assert(sample_index.regClass() == v1);
4012       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4013    }
4014
4015    Temp final_sample;
4016    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4017       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4018    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4019       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4020    else
4021       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4022
4023    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4024     * resource descriptor is 0 (invalid),
4025     */
4026    Temp compare = bld.tmp(bld.lm);
4027    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4028                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4029
4030    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4031
4032    /* Replace the MSAA sample index. */
4033    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4034 }
4035
4036 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4037 {
4038
4039    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4040    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4041    bool is_array = glsl_sampler_type_is_array(type);
4042    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4043    assert(!add_frag_pos && "Input attachments should be lowered.");
4044    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4045    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4046    int count = image_type_to_components_count(dim, is_array);
4047    std::vector<Operand> coords(count);
4048
4049    if (is_ms) {
4050       Operand sample_index;
4051       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4052       if (sample_cv)
4053          sample_index = Operand(sample_cv->u32);
4054       else
4055          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4056
4057       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4058          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4059          for (unsigned i = 0; i < vec->operands.size(); i++)
4060             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4061          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4062          vec->definitions[0] = Definition(fmask_load_address);
4063          ctx->block->instructions.emplace_back(std::move(vec));
4064
4065          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4066          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4067       }
4068       count--;
4069       coords[count] = sample_index;
4070    }
4071
4072    if (count == 1 && !gfx9_1d)
4073       return emit_extract_vector(ctx, src0, 0, v1);
4074
4075    if (gfx9_1d) {
4076       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4077       coords.resize(coords.size() + 1);
4078       coords[1] = Operand((uint32_t) 0);
4079       if (is_array)
4080          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4081    } else {
4082       for (int i = 0; i < count; i++)
4083          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4084    }
4085
4086    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4087        instr->intrinsic == nir_intrinsic_image_deref_store) {
4088       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4089       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4090
4091       if (!level_zero)
4092          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4093    }
4094
4095    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4096    for (unsigned i = 0; i < coords.size(); i++)
4097       vec->operands[i] = coords[i];
4098    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4099    vec->definitions[0] = Definition(res);
4100    ctx->block->instructions.emplace_back(std::move(vec));
4101    return res;
4102 }
4103
4104
4105 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4106 {
4107    Builder bld(ctx->program, ctx->block);
4108    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4109    const struct glsl_type *type = glsl_without_array(var->type);
4110    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4111    bool is_array = glsl_sampler_type_is_array(type);
4112    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4113
4114    if (dim == GLSL_SAMPLER_DIM_BUF) {
4115       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4116       unsigned num_channels = util_last_bit(mask);
4117       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4118       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4119
4120       aco_opcode opcode;
4121       switch (num_channels) {
4122       case 1:
4123          opcode = aco_opcode::buffer_load_format_x;
4124          break;
4125       case 2:
4126          opcode = aco_opcode::buffer_load_format_xy;
4127          break;
4128       case 3:
4129          opcode = aco_opcode::buffer_load_format_xyz;
4130          break;
4131       case 4:
4132          opcode = aco_opcode::buffer_load_format_xyzw;
4133          break;
4134       default:
4135          unreachable(">4 channel buffer image load");
4136       }
4137       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4138       load->operands[0] = Operand(vindex);
4139       load->operands[1] = Operand(rsrc);
4140       load->operands[2] = Operand((uint32_t) 0);
4141       Temp tmp;
4142       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4143          tmp = dst;
4144       else
4145          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4146       load->definitions[0] = Definition(tmp);
4147       load->idxen = true;
4148       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4149       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4150       load->barrier = barrier_image;
4151       ctx->block->instructions.emplace_back(std::move(load));
4152
4153       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4154       return;
4155    }
4156
4157    Temp coords = get_image_coords(ctx, instr, type);
4158    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4159
4160    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4161    unsigned num_components = util_bitcount(dmask);
4162    Temp tmp;
4163    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4164       tmp = dst;
4165    else
4166       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4167
4168    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4169    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4170
4171    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4172    load->operands[0] = Operand(coords);
4173    load->operands[1] = Operand(resource);
4174    load->definitions[0] = Definition(tmp);
4175    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4176    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4177    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4178    load->dmask = dmask;
4179    load->unrm = true;
4180    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4181    load->barrier = barrier_image;
4182    ctx->block->instructions.emplace_back(std::move(load));
4183
4184    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4185    return;
4186 }
4187
4188 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4189 {
4190    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4191    const struct glsl_type *type = glsl_without_array(var->type);
4192    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4193    bool is_array = glsl_sampler_type_is_array(type);
4194    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4195
4196    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4197
4198    if (dim == GLSL_SAMPLER_DIM_BUF) {
4199       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4200       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4201       aco_opcode opcode;
4202       switch (data.size()) {
4203       case 1:
4204          opcode = aco_opcode::buffer_store_format_x;
4205          break;
4206       case 2:
4207          opcode = aco_opcode::buffer_store_format_xy;
4208          break;
4209       case 3:
4210          opcode = aco_opcode::buffer_store_format_xyz;
4211          break;
4212       case 4:
4213          opcode = aco_opcode::buffer_store_format_xyzw;
4214          break;
4215       default:
4216          unreachable(">4 channel buffer image store");
4217       }
4218       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4219       store->operands[0] = Operand(vindex);
4220       store->operands[1] = Operand(rsrc);
4221       store->operands[2] = Operand((uint32_t) 0);
4222       store->operands[3] = Operand(data);
4223       store->idxen = true;
4224       store->glc = glc;
4225       store->dlc = false;
4226       store->disable_wqm = true;
4227       store->barrier = barrier_image;
4228       ctx->program->needs_exact = true;
4229       ctx->block->instructions.emplace_back(std::move(store));
4230       return;
4231    }
4232
4233    assert(data.type() == RegType::vgpr);
4234    Temp coords = get_image_coords(ctx, instr, type);
4235    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4236
4237    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4238    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4239
4240    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4241    store->operands[0] = Operand(coords);
4242    store->operands[1] = Operand(resource);
4243    store->operands[2] = Operand(s4);
4244    store->operands[3] = Operand(data);
4245    store->glc = glc;
4246    store->dlc = false;
4247    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4248    store->dmask = (1 << data.size()) - 1;
4249    store->unrm = true;
4250    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4251    store->disable_wqm = true;
4252    store->barrier = barrier_image;
4253    ctx->program->needs_exact = true;
4254    ctx->block->instructions.emplace_back(std::move(store));
4255    return;
4256 }
4257
4258 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4259 {
4260    /* return the previous value if dest is ever used */
4261    bool return_previous = false;
4262    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4263       return_previous = true;
4264       break;
4265    }
4266    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4267       return_previous = true;
4268       break;
4269    }
4270
4271    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4272    const struct glsl_type *type = glsl_without_array(var->type);
4273    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4274    bool is_array = glsl_sampler_type_is_array(type);
4275    Builder bld(ctx->program, ctx->block);
4276
4277    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4278    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4279
4280    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4281       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4282
4283    aco_opcode buf_op, image_op;
4284    switch (instr->intrinsic) {
4285       case nir_intrinsic_image_deref_atomic_add:
4286          buf_op = aco_opcode::buffer_atomic_add;
4287          image_op = aco_opcode::image_atomic_add;
4288          break;
4289       case nir_intrinsic_image_deref_atomic_umin:
4290          buf_op = aco_opcode::buffer_atomic_umin;
4291          image_op = aco_opcode::image_atomic_umin;
4292          break;
4293       case nir_intrinsic_image_deref_atomic_imin:
4294          buf_op = aco_opcode::buffer_atomic_smin;
4295          image_op = aco_opcode::image_atomic_smin;
4296          break;
4297       case nir_intrinsic_image_deref_atomic_umax:
4298          buf_op = aco_opcode::buffer_atomic_umax;
4299          image_op = aco_opcode::image_atomic_umax;
4300          break;
4301       case nir_intrinsic_image_deref_atomic_imax:
4302          buf_op = aco_opcode::buffer_atomic_smax;
4303          image_op = aco_opcode::image_atomic_smax;
4304          break;
4305       case nir_intrinsic_image_deref_atomic_and:
4306          buf_op = aco_opcode::buffer_atomic_and;
4307          image_op = aco_opcode::image_atomic_and;
4308          break;
4309       case nir_intrinsic_image_deref_atomic_or:
4310          buf_op = aco_opcode::buffer_atomic_or;
4311          image_op = aco_opcode::image_atomic_or;
4312          break;
4313       case nir_intrinsic_image_deref_atomic_xor:
4314          buf_op = aco_opcode::buffer_atomic_xor;
4315          image_op = aco_opcode::image_atomic_xor;
4316          break;
4317       case nir_intrinsic_image_deref_atomic_exchange:
4318          buf_op = aco_opcode::buffer_atomic_swap;
4319          image_op = aco_opcode::image_atomic_swap;
4320          break;
4321       case nir_intrinsic_image_deref_atomic_comp_swap:
4322          buf_op = aco_opcode::buffer_atomic_cmpswap;
4323          image_op = aco_opcode::image_atomic_cmpswap;
4324          break;
4325       default:
4326          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4327    }
4328
4329    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4330
4331    if (dim == GLSL_SAMPLER_DIM_BUF) {
4332       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4333       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4334       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4335       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4336       mubuf->operands[0] = Operand(vindex);
4337       mubuf->operands[1] = Operand(resource);
4338       mubuf->operands[2] = Operand((uint32_t)0);
4339       mubuf->operands[3] = Operand(data);
4340       if (return_previous)
4341          mubuf->definitions[0] = Definition(dst);
4342       mubuf->offset = 0;
4343       mubuf->idxen = true;
4344       mubuf->glc = return_previous;
4345       mubuf->dlc = false; /* Not needed for atomics */
4346       mubuf->disable_wqm = true;
4347       mubuf->barrier = barrier_image;
4348       ctx->program->needs_exact = true;
4349       ctx->block->instructions.emplace_back(std::move(mubuf));
4350       return;
4351    }
4352
4353    Temp coords = get_image_coords(ctx, instr, type);
4354    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4355    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4356    mimg->operands[0] = Operand(coords);
4357    mimg->operands[1] = Operand(resource);
4358    mimg->operands[2] = Operand(s4); /* no sampler */
4359    mimg->operands[3] = Operand(data);
4360    if (return_previous)
4361       mimg->definitions[0] = Definition(dst);
4362    mimg->glc = return_previous;
4363    mimg->dlc = false; /* Not needed for atomics */
4364    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4365    mimg->dmask = (1 << data.size()) - 1;
4366    mimg->unrm = true;
4367    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4368    mimg->disable_wqm = true;
4369    mimg->barrier = barrier_image;
4370    ctx->program->needs_exact = true;
4371    ctx->block->instructions.emplace_back(std::move(mimg));
4372    return;
4373 }
4374
4375 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4376 {
4377    if (in_elements && ctx->options->chip_class == GFX8) {
4378       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4379       Builder bld(ctx->program, ctx->block);
4380
4381       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4382
4383       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4384       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4385
4386       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4387       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4388
4389       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4390       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4391
4392       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4393       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4394                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4395       if (dst.type() == RegType::vgpr)
4396          bld.copy(Definition(dst), shr_dst);
4397
4398       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4399    } else {
4400       emit_extract_vector(ctx, desc, 2, dst);
4401    }
4402 }
4403
4404 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4405 {
4406    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4407    const struct glsl_type *type = glsl_without_array(var->type);
4408    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4409    bool is_array = glsl_sampler_type_is_array(type);
4410    Builder bld(ctx->program, ctx->block);
4411
4412    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4413       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4414       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4415    }
4416
4417    /* LOD */
4418    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4419
4420    /* Resource */
4421    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4422
4423    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4424
4425    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4426    mimg->operands[0] = Operand(lod);
4427    mimg->operands[1] = Operand(resource);
4428    uint8_t& dmask = mimg->dmask;
4429    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4430    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4431    mimg->da = glsl_sampler_type_is_array(type);
4432    mimg->can_reorder = true;
4433    Definition& def = mimg->definitions[0];
4434    ctx->block->instructions.emplace_back(std::move(mimg));
4435
4436    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4437        glsl_sampler_type_is_array(type)) {
4438
4439       assert(instr->dest.ssa.num_components == 3);
4440       Temp tmp = {ctx->program->allocateId(), v3};
4441       def = Definition(tmp);
4442       emit_split_vector(ctx, tmp, 3);
4443
4444       /* divide 3rd value by 6 by multiplying with magic number */
4445       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4446       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4447
4448       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4449                  emit_extract_vector(ctx, tmp, 0, v1),
4450                  emit_extract_vector(ctx, tmp, 1, v1),
4451                  by_6);
4452
4453    } else if (ctx->options->chip_class == GFX9 &&
4454               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4455               glsl_sampler_type_is_array(type)) {
4456       assert(instr->dest.ssa.num_components == 2);
4457       def = Definition(dst);
4458       dmask = 0x5;
4459    } else {
4460       def = Definition(dst);
4461    }
4462
4463    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4464 }
4465
4466 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4467 {
4468    Builder bld(ctx->program, ctx->block);
4469    unsigned num_components = instr->num_components;
4470
4471    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4472    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4473    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4474
4475    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4476    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4477 }
4478
4479 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4480 {
4481    Builder bld(ctx->program, ctx->block);
4482    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4483    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4484    unsigned writemask = nir_intrinsic_write_mask(instr);
4485    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4486
4487    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4488    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4489
4490    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4491                ctx->options->chip_class >= GFX8;
4492    if (smem)
4493       offset = bld.as_uniform(offset);
4494    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4495
4496    while (writemask) {
4497       int start, count;
4498       u_bit_scan_consecutive_range(&writemask, &start, &count);
4499       if (count == 3 && smem) {
4500          writemask |= 1u << (start + 2);
4501          count = 2;
4502       }
4503       int num_bytes = count * elem_size_bytes;
4504
4505       if (num_bytes > 16) {
4506          assert(elem_size_bytes == 8);
4507          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4508          count = 2;
4509          num_bytes = 16;
4510       }
4511
4512       // TODO: check alignment of sub-dword stores
4513       // TODO: split 3 bytes. there is no store instruction for that
4514
4515       Temp write_data;
4516       if (count != instr->num_components) {
4517          emit_split_vector(ctx, data, instr->num_components);
4518          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4519          for (int i = 0; i < count; i++) {
4520             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4521             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4522          }
4523          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4524          vec->definitions[0] = Definition(write_data);
4525          ctx->block->instructions.emplace_back(std::move(vec));
4526       } else if (!smem && data.type() != RegType::vgpr) {
4527          assert(num_bytes % 4 == 0);
4528          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4529       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4530          assert(num_bytes % 4 == 0);
4531          write_data = bld.as_uniform(data);
4532       } else {
4533          write_data = data;
4534       }
4535
4536       aco_opcode vmem_op, smem_op;
4537       switch (num_bytes) {
4538          case 4:
4539             vmem_op = aco_opcode::buffer_store_dword;
4540             smem_op = aco_opcode::s_buffer_store_dword;
4541             break;
4542          case 8:
4543             vmem_op = aco_opcode::buffer_store_dwordx2;
4544             smem_op = aco_opcode::s_buffer_store_dwordx2;
4545             break;
4546          case 12:
4547             vmem_op = aco_opcode::buffer_store_dwordx3;
4548             smem_op = aco_opcode::last_opcode;
4549             assert(!smem);
4550             break;
4551          case 16:
4552             vmem_op = aco_opcode::buffer_store_dwordx4;
4553             smem_op = aco_opcode::s_buffer_store_dwordx4;
4554             break;
4555          default:
4556             unreachable("Store SSBO not implemented for this size.");
4557       }
4558       if (ctx->stage == fragment_fs)
4559          smem_op = aco_opcode::p_fs_buffer_store_smem;
4560
4561       if (smem) {
4562          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4563          store->operands[0] = Operand(rsrc);
4564          if (start) {
4565             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4566                                 offset, Operand(start * elem_size_bytes));
4567             store->operands[1] = Operand(off);
4568          } else {
4569             store->operands[1] = Operand(offset);
4570          }
4571          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4572             store->operands[1].setFixed(m0);
4573          store->operands[2] = Operand(write_data);
4574          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4575          store->dlc = false;
4576          store->disable_wqm = true;
4577          store->barrier = barrier_buffer;
4578          ctx->block->instructions.emplace_back(std::move(store));
4579          ctx->program->wb_smem_l1_on_end = true;
4580          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4581             ctx->block->kind |= block_kind_needs_lowering;
4582             ctx->program->needs_exact = true;
4583          }
4584       } else {
4585          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4586          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4587          store->operands[1] = Operand(rsrc);
4588          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4589          store->operands[3] = Operand(write_data);
4590          store->offset = start * elem_size_bytes;
4591          store->offen = (offset.type() == RegType::vgpr);
4592          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4593          store->dlc = false;
4594          store->disable_wqm = true;
4595          store->barrier = barrier_buffer;
4596          ctx->program->needs_exact = true;
4597          ctx->block->instructions.emplace_back(std::move(store));
4598       }
4599    }
4600 }
4601
4602 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4603 {
4604    /* return the previous value if dest is ever used */
4605    bool return_previous = false;
4606    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4607       return_previous = true;
4608       break;
4609    }
4610    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4611       return_previous = true;
4612       break;
4613    }
4614
4615    Builder bld(ctx->program, ctx->block);
4616    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4617
4618    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4619       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4620                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4621
4622    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4623    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4624    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4625
4626    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4627
4628    aco_opcode op32, op64;
4629    switch (instr->intrinsic) {
4630       case nir_intrinsic_ssbo_atomic_add:
4631          op32 = aco_opcode::buffer_atomic_add;
4632          op64 = aco_opcode::buffer_atomic_add_x2;
4633          break;
4634       case nir_intrinsic_ssbo_atomic_imin:
4635          op32 = aco_opcode::buffer_atomic_smin;
4636          op64 = aco_opcode::buffer_atomic_smin_x2;
4637          break;
4638       case nir_intrinsic_ssbo_atomic_umin:
4639          op32 = aco_opcode::buffer_atomic_umin;
4640          op64 = aco_opcode::buffer_atomic_umin_x2;
4641          break;
4642       case nir_intrinsic_ssbo_atomic_imax:
4643          op32 = aco_opcode::buffer_atomic_smax;
4644          op64 = aco_opcode::buffer_atomic_smax_x2;
4645          break;
4646       case nir_intrinsic_ssbo_atomic_umax:
4647          op32 = aco_opcode::buffer_atomic_umax;
4648          op64 = aco_opcode::buffer_atomic_umax_x2;
4649          break;
4650       case nir_intrinsic_ssbo_atomic_and:
4651          op32 = aco_opcode::buffer_atomic_and;
4652          op64 = aco_opcode::buffer_atomic_and_x2;
4653          break;
4654       case nir_intrinsic_ssbo_atomic_or:
4655          op32 = aco_opcode::buffer_atomic_or;
4656          op64 = aco_opcode::buffer_atomic_or_x2;
4657          break;
4658       case nir_intrinsic_ssbo_atomic_xor:
4659          op32 = aco_opcode::buffer_atomic_xor;
4660          op64 = aco_opcode::buffer_atomic_xor_x2;
4661          break;
4662       case nir_intrinsic_ssbo_atomic_exchange:
4663          op32 = aco_opcode::buffer_atomic_swap;
4664          op64 = aco_opcode::buffer_atomic_swap_x2;
4665          break;
4666       case nir_intrinsic_ssbo_atomic_comp_swap:
4667          op32 = aco_opcode::buffer_atomic_cmpswap;
4668          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4669          break;
4670       default:
4671          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4672    }
4673    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4674    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4675    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4676    mubuf->operands[1] = Operand(rsrc);
4677    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4678    mubuf->operands[3] = Operand(data);
4679    if (return_previous)
4680       mubuf->definitions[0] = Definition(dst);
4681    mubuf->offset = 0;
4682    mubuf->offen = (offset.type() == RegType::vgpr);
4683    mubuf->glc = return_previous;
4684    mubuf->dlc = false; /* Not needed for atomics */
4685    mubuf->disable_wqm = true;
4686    mubuf->barrier = barrier_buffer;
4687    ctx->program->needs_exact = true;
4688    ctx->block->instructions.emplace_back(std::move(mubuf));
4689 }
4690
4691 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4692
4693    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4694    Builder bld(ctx->program, ctx->block);
4695    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4696    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4697 }
4698
4699 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4700 {
4701    Builder bld(ctx->program, ctx->block);
4702    unsigned num_components = instr->num_components;
4703    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4704
4705    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4706    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4707
4708    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4709    bool dlc = glc && ctx->options->chip_class >= GFX10;
4710    aco_opcode op;
4711    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4712       bool global = ctx->options->chip_class >= GFX9;
4713       aco_opcode op;
4714       switch (num_bytes) {
4715       case 4:
4716          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4717          break;
4718       case 8:
4719          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4720          break;
4721       case 12:
4722          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4723          break;
4724       case 16:
4725          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4726          break;
4727       default:
4728          unreachable("load_global not implemented for this size.");
4729       }
4730       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4731       flat->operands[0] = Operand(addr);
4732       flat->operands[1] = Operand(s1);
4733       flat->glc = glc;
4734       flat->dlc = dlc;
4735       flat->barrier = barrier_buffer;
4736
4737       if (dst.type() == RegType::sgpr) {
4738          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4739          flat->definitions[0] = Definition(vec);
4740          ctx->block->instructions.emplace_back(std::move(flat));
4741          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4742       } else {
4743          flat->definitions[0] = Definition(dst);
4744          ctx->block->instructions.emplace_back(std::move(flat));
4745       }
4746       emit_split_vector(ctx, dst, num_components);
4747    } else {
4748       switch (num_bytes) {
4749          case 4:
4750             op = aco_opcode::s_load_dword;
4751             break;
4752          case 8:
4753             op = aco_opcode::s_load_dwordx2;
4754             break;
4755          case 12:
4756          case 16:
4757             op = aco_opcode::s_load_dwordx4;
4758             break;
4759          default:
4760             unreachable("load_global not implemented for this size.");
4761       }
4762       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4763       load->operands[0] = Operand(addr);
4764       load->operands[1] = Operand(0u);
4765       load->definitions[0] = Definition(dst);
4766       load->glc = glc;
4767       load->dlc = dlc;
4768       load->barrier = barrier_buffer;
4769       assert(ctx->options->chip_class >= GFX8 || !glc);
4770
4771       if (dst.size() == 3) {
4772          /* trim vector */
4773          Temp vec = bld.tmp(s4);
4774          load->definitions[0] = Definition(vec);
4775          ctx->block->instructions.emplace_back(std::move(load));
4776          emit_split_vector(ctx, vec, 4);
4777
4778          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4779                     emit_extract_vector(ctx, vec, 0, s1),
4780                     emit_extract_vector(ctx, vec, 1, s1),
4781                     emit_extract_vector(ctx, vec, 2, s1));
4782       } else {
4783          ctx->block->instructions.emplace_back(std::move(load));
4784       }
4785    }
4786 }
4787
4788 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4789 {
4790    Builder bld(ctx->program, ctx->block);
4791    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4792
4793    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4794    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4795
4796    unsigned writemask = nir_intrinsic_write_mask(instr);
4797    while (writemask) {
4798       int start, count;
4799       u_bit_scan_consecutive_range(&writemask, &start, &count);
4800       unsigned num_bytes = count * elem_size_bytes;
4801
4802       Temp write_data = data;
4803       if (count != instr->num_components) {
4804          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4805          for (int i = 0; i < count; i++)
4806             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4807          write_data = bld.tmp(RegType::vgpr, count);
4808          vec->definitions[0] = Definition(write_data);
4809          ctx->block->instructions.emplace_back(std::move(vec));
4810       }
4811
4812       unsigned offset = start * elem_size_bytes;
4813       if (offset > 0 && ctx->options->chip_class < GFX9) {
4814          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4815          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4816          Temp carry = bld.tmp(bld.lm);
4817          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4818
4819          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4820                   Operand(offset), addr0);
4821          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4822                   Operand(0u), addr1,
4823                   carry).def(1).setHint(vcc);
4824
4825          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4826
4827          offset = 0;
4828       }
4829
4830       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4831       bool global = ctx->options->chip_class >= GFX9;
4832       aco_opcode op;
4833       switch (num_bytes) {
4834       case 4:
4835          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4836          break;
4837       case 8:
4838          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4839          break;
4840       case 12:
4841          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4842          break;
4843       case 16:
4844          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4845          break;
4846       default:
4847          unreachable("store_global not implemented for this size.");
4848       }
4849       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4850       flat->operands[0] = Operand(addr);
4851       flat->operands[1] = Operand(s1);
4852       flat->operands[2] = Operand(data);
4853       flat->glc = glc;
4854       flat->dlc = false;
4855       flat->offset = offset;
4856       flat->disable_wqm = true;
4857       flat->barrier = barrier_buffer;
4858       ctx->program->needs_exact = true;
4859       ctx->block->instructions.emplace_back(std::move(flat));
4860    }
4861 }
4862
4863 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4864 {
4865    /* return the previous value if dest is ever used */
4866    bool return_previous = false;
4867    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4868       return_previous = true;
4869       break;
4870    }
4871    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4872       return_previous = true;
4873       break;
4874    }
4875
4876    Builder bld(ctx->program, ctx->block);
4877    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4878    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4879
4880    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4881       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4882                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4883
4884    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4885
4886    bool global = ctx->options->chip_class >= GFX9;
4887    aco_opcode op32, op64;
4888    switch (instr->intrinsic) {
4889       case nir_intrinsic_global_atomic_add:
4890          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4891          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4892          break;
4893       case nir_intrinsic_global_atomic_imin:
4894          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4895          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4896          break;
4897       case nir_intrinsic_global_atomic_umin:
4898          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4899          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4900          break;
4901       case nir_intrinsic_global_atomic_imax:
4902          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4903          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4904          break;
4905       case nir_intrinsic_global_atomic_umax:
4906          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4907          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4908          break;
4909       case nir_intrinsic_global_atomic_and:
4910          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4911          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4912          break;
4913       case nir_intrinsic_global_atomic_or:
4914          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4915          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4916          break;
4917       case nir_intrinsic_global_atomic_xor:
4918          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4919          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4920          break;
4921       case nir_intrinsic_global_atomic_exchange:
4922          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4923          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4924          break;
4925       case nir_intrinsic_global_atomic_comp_swap:
4926          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4927          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4928          break;
4929       default:
4930          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4931    }
4932    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4933    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4934    flat->operands[0] = Operand(addr);
4935    flat->operands[1] = Operand(s1);
4936    flat->operands[2] = Operand(data);
4937    if (return_previous)
4938       flat->definitions[0] = Definition(dst);
4939    flat->glc = return_previous;
4940    flat->dlc = false; /* Not needed for atomics */
4941    flat->offset = 0;
4942    flat->disable_wqm = true;
4943    flat->barrier = barrier_buffer;
4944    ctx->program->needs_exact = true;
4945    ctx->block->instructions.emplace_back(std::move(flat));
4946 }
4947
4948 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4949    Builder bld(ctx->program, ctx->block);
4950    switch(instr->intrinsic) {
4951       case nir_intrinsic_group_memory_barrier:
4952       case nir_intrinsic_memory_barrier:
4953          bld.barrier(aco_opcode::p_memory_barrier_all);
4954          break;
4955       case nir_intrinsic_memory_barrier_buffer:
4956          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4957          break;
4958       case nir_intrinsic_memory_barrier_image:
4959          bld.barrier(aco_opcode::p_memory_barrier_image);
4960          break;
4961       case nir_intrinsic_memory_barrier_shared:
4962          bld.barrier(aco_opcode::p_memory_barrier_shared);
4963          break;
4964       default:
4965          unreachable("Unimplemented memory barrier intrinsic");
4966          break;
4967    }
4968 }
4969
4970 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4971 {
4972    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4973    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4974    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4975    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4976    Builder bld(ctx->program, ctx->block);
4977
4978    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4979    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4980    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4981 }
4982
4983 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4984 {
4985    unsigned writemask = nir_intrinsic_write_mask(instr);
4986    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4987    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4988    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4989    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4990
4991    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4992    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4993 }
4994
4995 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4996 {
4997    unsigned offset = nir_intrinsic_base(instr);
4998    Operand m = load_lds_size_m0(ctx);
4999    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5000    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5001
5002    unsigned num_operands = 3;
5003    aco_opcode op32, op64, op32_rtn, op64_rtn;
5004    switch(instr->intrinsic) {
5005       case nir_intrinsic_shared_atomic_add:
5006          op32 = aco_opcode::ds_add_u32;
5007          op64 = aco_opcode::ds_add_u64;
5008          op32_rtn = aco_opcode::ds_add_rtn_u32;
5009          op64_rtn = aco_opcode::ds_add_rtn_u64;
5010          break;
5011       case nir_intrinsic_shared_atomic_imin:
5012          op32 = aco_opcode::ds_min_i32;
5013          op64 = aco_opcode::ds_min_i64;
5014          op32_rtn = aco_opcode::ds_min_rtn_i32;
5015          op64_rtn = aco_opcode::ds_min_rtn_i64;
5016          break;
5017       case nir_intrinsic_shared_atomic_umin:
5018          op32 = aco_opcode::ds_min_u32;
5019          op64 = aco_opcode::ds_min_u64;
5020          op32_rtn = aco_opcode::ds_min_rtn_u32;
5021          op64_rtn = aco_opcode::ds_min_rtn_u64;
5022          break;
5023       case nir_intrinsic_shared_atomic_imax:
5024          op32 = aco_opcode::ds_max_i32;
5025          op64 = aco_opcode::ds_max_i64;
5026          op32_rtn = aco_opcode::ds_max_rtn_i32;
5027          op64_rtn = aco_opcode::ds_max_rtn_i64;
5028          break;
5029       case nir_intrinsic_shared_atomic_umax:
5030          op32 = aco_opcode::ds_max_u32;
5031          op64 = aco_opcode::ds_max_u64;
5032          op32_rtn = aco_opcode::ds_max_rtn_u32;
5033          op64_rtn = aco_opcode::ds_max_rtn_u64;
5034          break;
5035       case nir_intrinsic_shared_atomic_and:
5036          op32 = aco_opcode::ds_and_b32;
5037          op64 = aco_opcode::ds_and_b64;
5038          op32_rtn = aco_opcode::ds_and_rtn_b32;
5039          op64_rtn = aco_opcode::ds_and_rtn_b64;
5040          break;
5041       case nir_intrinsic_shared_atomic_or:
5042          op32 = aco_opcode::ds_or_b32;
5043          op64 = aco_opcode::ds_or_b64;
5044          op32_rtn = aco_opcode::ds_or_rtn_b32;
5045          op64_rtn = aco_opcode::ds_or_rtn_b64;
5046          break;
5047       case nir_intrinsic_shared_atomic_xor:
5048          op32 = aco_opcode::ds_xor_b32;
5049          op64 = aco_opcode::ds_xor_b64;
5050          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5051          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5052          break;
5053       case nir_intrinsic_shared_atomic_exchange:
5054          op32 = aco_opcode::ds_write_b32;
5055          op64 = aco_opcode::ds_write_b64;
5056          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5057          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5058          break;
5059       case nir_intrinsic_shared_atomic_comp_swap:
5060          op32 = aco_opcode::ds_cmpst_b32;
5061          op64 = aco_opcode::ds_cmpst_b64;
5062          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5063          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5064          num_operands = 4;
5065          break;
5066       default:
5067          unreachable("Unhandled shared atomic intrinsic");
5068    }
5069
5070    /* return the previous value if dest is ever used */
5071    bool return_previous = false;
5072    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5073       return_previous = true;
5074       break;
5075    }
5076    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5077       return_previous = true;
5078       break;
5079    }
5080
5081    aco_opcode op;
5082    if (data.size() == 1) {
5083       assert(instr->dest.ssa.bit_size == 32);
5084       op = return_previous ? op32_rtn : op32;
5085    } else {
5086       assert(instr->dest.ssa.bit_size == 64);
5087       op = return_previous ? op64_rtn : op64;
5088    }
5089
5090    if (offset > 65535) {
5091       Builder bld(ctx->program, ctx->block);
5092       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5093       offset = 0;
5094    }
5095
5096    aco_ptr<DS_instruction> ds;
5097    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5098    ds->operands[0] = Operand(address);
5099    ds->operands[1] = Operand(data);
5100    if (num_operands == 4)
5101       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5102    ds->operands[num_operands - 1] = m;
5103    ds->offset0 = offset;
5104    if (return_previous)
5105       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5106    ctx->block->instructions.emplace_back(std::move(ds));
5107 }
5108
5109 Temp get_scratch_resource(isel_context *ctx)
5110 {
5111    Builder bld(ctx->program, ctx->block);
5112    Temp scratch_addr = ctx->program->private_segment_buffer;
5113    if (ctx->stage != compute_cs)
5114       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5115
5116    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5117                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5118
5119    if (ctx->program->chip_class >= GFX10) {
5120       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5121                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5122                    S_008F0C_RESOURCE_LEVEL(1);
5123    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5124       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5125                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5126    }
5127
5128    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5129    if (ctx->program->chip_class <= GFX8)
5130       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5131
5132    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5133 }
5134
5135 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5136    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5137    Builder bld(ctx->program, ctx->block);
5138    Temp rsrc = get_scratch_resource(ctx);
5139    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5140    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5141
5142    aco_opcode op;
5143    switch (dst.size()) {
5144       case 1:
5145          op = aco_opcode::buffer_load_dword;
5146          break;
5147       case 2:
5148          op = aco_opcode::buffer_load_dwordx2;
5149          break;
5150       case 3:
5151          op = aco_opcode::buffer_load_dwordx3;
5152          break;
5153       case 4:
5154          op = aco_opcode::buffer_load_dwordx4;
5155          break;
5156       case 6:
5157       case 8: {
5158          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5159          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5160                                 bld.def(v4), offset, rsrc,
5161                                 ctx->program->scratch_offset, 0, true);
5162          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5163                                                   aco_opcode::buffer_load_dwordx4,
5164                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5165                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5166          emit_split_vector(ctx, lower, 2);
5167          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5168          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5169          if (dst.size() == 8) {
5170             emit_split_vector(ctx, upper, 2);
5171             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5172             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5173          } else {
5174             elems[2] = upper;
5175          }
5176
5177          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5178                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5179          for (unsigned i = 0; i < dst.size() / 2; i++)
5180             vec->operands[i] = Operand(elems[i]);
5181          vec->definitions[0] = Definition(dst);
5182          bld.insert(std::move(vec));
5183          ctx->allocated_vec.emplace(dst.id(), elems);
5184          return;
5185       }
5186       default:
5187          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5188    }
5189
5190    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5191    emit_split_vector(ctx, dst, instr->num_components);
5192 }
5193
5194 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5195    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5196    Builder bld(ctx->program, ctx->block);
5197    Temp rsrc = get_scratch_resource(ctx);
5198    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5199    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5200
5201    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5202    unsigned writemask = nir_intrinsic_write_mask(instr);
5203
5204    while (writemask) {
5205       int start, count;
5206       u_bit_scan_consecutive_range(&writemask, &start, &count);
5207       int num_bytes = count * elem_size_bytes;
5208
5209       if (num_bytes > 16) {
5210          assert(elem_size_bytes == 8);
5211          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5212          count = 2;
5213          num_bytes = 16;
5214       }
5215
5216       // TODO: check alignment of sub-dword stores
5217       // TODO: split 3 bytes. there is no store instruction for that
5218
5219       Temp write_data;
5220       if (count != instr->num_components) {
5221          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5222          for (int i = 0; i < count; i++) {
5223             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5224             vec->operands[i] = Operand(elem);
5225          }
5226          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5227          vec->definitions[0] = Definition(write_data);
5228          ctx->block->instructions.emplace_back(std::move(vec));
5229       } else {
5230          write_data = data;
5231       }
5232
5233       aco_opcode op;
5234       switch (num_bytes) {
5235          case 4:
5236             op = aco_opcode::buffer_store_dword;
5237             break;
5238          case 8:
5239             op = aco_opcode::buffer_store_dwordx2;
5240             break;
5241          case 12:
5242             op = aco_opcode::buffer_store_dwordx3;
5243             break;
5244          case 16:
5245             op = aco_opcode::buffer_store_dwordx4;
5246             break;
5247          default:
5248             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5249       }
5250
5251       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5252    }
5253 }
5254
5255 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5256    uint8_t log2_ps_iter_samples;
5257    if (ctx->program->info->ps.force_persample) {
5258       log2_ps_iter_samples =
5259          util_logbase2(ctx->options->key.fs.num_samples);
5260    } else {
5261       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5262    }
5263
5264    /* The bit pattern matches that used by fixed function fragment
5265     * processing. */
5266    static const unsigned ps_iter_masks[] = {
5267       0xffff, /* not used */
5268       0x5555,
5269       0x1111,
5270       0x0101,
5271       0x0001,
5272    };
5273    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5274
5275    Builder bld(ctx->program, ctx->block);
5276
5277    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5278                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5279    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5280    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5281    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5282    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5283 }
5284
5285 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5286 {
5287    Builder bld(ctx->program, ctx->block);
5288
5289    if (cluster_size == 1) {
5290       return src;
5291    } if (op == nir_op_iand && cluster_size == 4) {
5292       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5293       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5294       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5295                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5296    } else if (op == nir_op_ior && cluster_size == 4) {
5297       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5298       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5299                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5300    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5301       //subgroupAnd(val) -> (exec & ~val) == 0
5302       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5303       Temp all = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), bld.scc(tmp), Operand(0u));
5304       return bool_to_vector_condition(ctx, all);
5305    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5306       //subgroupOr(val) -> (val & exec) != 0
5307       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5308       return bool_to_vector_condition(ctx, tmp);
5309    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5310       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5311       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5312       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5313       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5314       return bool_to_vector_condition(ctx, tmp);
5315    } else {
5316       //subgroupClustered{And,Or,Xor}(val, n) ->
5317       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5318       //cluster_offset = ~(n - 1) & lane_id
5319       //cluster_mask = ((1 << n) - 1)
5320       //subgroupClusteredAnd():
5321       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5322       //subgroupClusteredOr():
5323       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5324       //subgroupClusteredXor():
5325       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5326       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5327       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5328
5329       Temp tmp;
5330       if (op == nir_op_iand)
5331          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5332       else
5333          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5334
5335       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5336
5337       if (ctx->program->chip_class <= GFX7)
5338          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5339       else if (ctx->program->wave_size == 64)
5340          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5341       else
5342          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5343       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5344       if (cluster_mask != 0xffffffff)
5345          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5346
5347       Definition cmp_def = Definition();
5348       if (op == nir_op_iand) {
5349          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5350       } else if (op == nir_op_ior) {
5351          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5352       } else if (op == nir_op_ixor) {
5353          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5354                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5355          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5356       }
5357       cmp_def.setHint(vcc);
5358       return cmp_def.getTemp();
5359    }
5360 }
5361
5362 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5363 {
5364    Builder bld(ctx->program, ctx->block);
5365
5366    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5367    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5368    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5369    Temp tmp;
5370    if (op == nir_op_iand)
5371       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5372    else
5373       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5374
5375    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5376    Temp lo = lohi.def(0).getTemp();
5377    Temp hi = lohi.def(1).getTemp();
5378    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5379
5380    Definition cmp_def = Definition();
5381    if (op == nir_op_iand)
5382       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5383    else if (op == nir_op_ior)
5384       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5385    else if (op == nir_op_ixor)
5386       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5387                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5388    cmp_def.setHint(vcc);
5389    return cmp_def.getTemp();
5390 }
5391
5392 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5393 {
5394    Builder bld(ctx->program, ctx->block);
5395
5396    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5397    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5398    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5399    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5400    if (op == nir_op_iand)
5401       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5402    else if (op == nir_op_ior)
5403       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5404    else if (op == nir_op_ixor)
5405       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5406
5407    assert(false);
5408    return Temp();
5409 }
5410
5411 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5412 {
5413    Builder bld(ctx->program, ctx->block);
5414    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5415    if (src.regClass().type() == RegType::vgpr) {
5416       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5417    } else if (src.regClass() == s1) {
5418       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5419    } else if (src.regClass() == s2) {
5420       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5421    } else {
5422       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5423       nir_print_instr(&instr->instr, stderr);
5424       fprintf(stderr, "\n");
5425    }
5426 }
5427
5428 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5429 {
5430    Builder bld(ctx->program, ctx->block);
5431    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5432    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5433    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5434
5435    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5436    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5437    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5438    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5439
5440    /* Build DD X/Y */
5441    if (ctx->program->chip_class >= GFX8) {
5442       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5443       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5444       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5445       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5446       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5447       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5448    } else {
5449       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5450       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5451       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5452       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5453       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5454       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5455       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5456       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5457       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5458       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5459    }
5460
5461    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5462    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5463    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5464    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5465    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5466    Temp wqm1 = bld.tmp(v1);
5467    emit_wqm(ctx, tmp1, wqm1, true);
5468    Temp wqm2 = bld.tmp(v1);
5469    emit_wqm(ctx, tmp2, wqm2, true);
5470    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5471    return;
5472 }
5473
5474 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5475 {
5476    Builder bld(ctx->program, ctx->block);
5477    switch(instr->intrinsic) {
5478    case nir_intrinsic_load_barycentric_sample:
5479    case nir_intrinsic_load_barycentric_pixel:
5480    case nir_intrinsic_load_barycentric_centroid: {
5481       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5482       Temp bary = Temp(0, s2);
5483       switch (mode) {
5484       case INTERP_MODE_SMOOTH:
5485       case INTERP_MODE_NONE:
5486          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5487             bary = get_arg(ctx, ctx->args->ac.persp_center);
5488          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5489             bary = ctx->persp_centroid;
5490          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5491             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5492          break;
5493       case INTERP_MODE_NOPERSPECTIVE:
5494          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5495             bary = get_arg(ctx, ctx->args->ac.linear_center);
5496          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5497             bary = ctx->linear_centroid;
5498          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5499             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5500          break;
5501       default:
5502          break;
5503       }
5504       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5505       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5506       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5507       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5508                  Operand(p1), Operand(p2));
5509       emit_split_vector(ctx, dst, 2);
5510       break;
5511    }
5512    case nir_intrinsic_load_barycentric_at_sample: {
5513       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5514       switch (ctx->options->key.fs.num_samples) {
5515          case 2: sample_pos_offset += 1 << 3; break;
5516          case 4: sample_pos_offset += 3 << 3; break;
5517          case 8: sample_pos_offset += 7 << 3; break;
5518          default: break;
5519       }
5520       Temp sample_pos;
5521       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5522       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5523       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5524       if (addr.type() == RegType::sgpr) {
5525          Operand offset;
5526          if (const_addr) {
5527             sample_pos_offset += const_addr->u32 << 3;
5528             offset = Operand(sample_pos_offset);
5529          } else if (ctx->options->chip_class >= GFX9) {
5530             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5531          } else {
5532             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5533             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5534          }
5535          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5536
5537       } else if (ctx->options->chip_class >= GFX9) {
5538          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5539          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5540       } else {
5541          /* addr += private_segment_buffer + sample_pos_offset */
5542          Temp tmp0 = bld.tmp(s1);
5543          Temp tmp1 = bld.tmp(s1);
5544          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5545          Definition scc_tmp = bld.def(s1, scc);
5546          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5547          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5548          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5549          Temp pck0 = bld.tmp(v1);
5550          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5551          tmp1 = as_vgpr(ctx, tmp1);
5552          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5553          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5554
5555          /* sample_pos = flat_load_dwordx2 addr */
5556          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5557       }
5558
5559       /* sample_pos -= 0.5 */
5560       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5561       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5562       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5563       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5564       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5565
5566       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5567       break;
5568    }
5569    case nir_intrinsic_load_barycentric_at_offset: {
5570       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5571       RegClass rc = RegClass(offset.type(), 1);
5572       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5573       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5574       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5575       break;
5576    }
5577    case nir_intrinsic_load_front_face: {
5578       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5579                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5580       break;
5581    }
5582    case nir_intrinsic_load_view_index:
5583    case nir_intrinsic_load_layer_id: {
5584       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5585          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5586          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5587          break;
5588       }
5589
5590       unsigned idx = nir_intrinsic_base(instr);
5591       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5592                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5593       break;
5594    }
5595    case nir_intrinsic_load_frag_coord: {
5596       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5597       break;
5598    }
5599    case nir_intrinsic_load_sample_pos: {
5600       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5601       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5602       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5603                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5604                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5605       break;
5606    }
5607    case nir_intrinsic_load_interpolated_input:
5608       visit_load_interpolated_input(ctx, instr);
5609       break;
5610    case nir_intrinsic_store_output:
5611       visit_store_output(ctx, instr);
5612       break;
5613    case nir_intrinsic_load_input:
5614       visit_load_input(ctx, instr);
5615       break;
5616    case nir_intrinsic_load_ubo:
5617       visit_load_ubo(ctx, instr);
5618       break;
5619    case nir_intrinsic_load_push_constant:
5620       visit_load_push_constant(ctx, instr);
5621       break;
5622    case nir_intrinsic_load_constant:
5623       visit_load_constant(ctx, instr);
5624       break;
5625    case nir_intrinsic_vulkan_resource_index:
5626       visit_load_resource(ctx, instr);
5627       break;
5628    case nir_intrinsic_discard:
5629       visit_discard(ctx, instr);
5630       break;
5631    case nir_intrinsic_discard_if:
5632       visit_discard_if(ctx, instr);
5633       break;
5634    case nir_intrinsic_load_shared:
5635       visit_load_shared(ctx, instr);
5636       break;
5637    case nir_intrinsic_store_shared:
5638       visit_store_shared(ctx, instr);
5639       break;
5640    case nir_intrinsic_shared_atomic_add:
5641    case nir_intrinsic_shared_atomic_imin:
5642    case nir_intrinsic_shared_atomic_umin:
5643    case nir_intrinsic_shared_atomic_imax:
5644    case nir_intrinsic_shared_atomic_umax:
5645    case nir_intrinsic_shared_atomic_and:
5646    case nir_intrinsic_shared_atomic_or:
5647    case nir_intrinsic_shared_atomic_xor:
5648    case nir_intrinsic_shared_atomic_exchange:
5649    case nir_intrinsic_shared_atomic_comp_swap:
5650       visit_shared_atomic(ctx, instr);
5651       break;
5652    case nir_intrinsic_image_deref_load:
5653       visit_image_load(ctx, instr);
5654       break;
5655    case nir_intrinsic_image_deref_store:
5656       visit_image_store(ctx, instr);
5657       break;
5658    case nir_intrinsic_image_deref_atomic_add:
5659    case nir_intrinsic_image_deref_atomic_umin:
5660    case nir_intrinsic_image_deref_atomic_imin:
5661    case nir_intrinsic_image_deref_atomic_umax:
5662    case nir_intrinsic_image_deref_atomic_imax:
5663    case nir_intrinsic_image_deref_atomic_and:
5664    case nir_intrinsic_image_deref_atomic_or:
5665    case nir_intrinsic_image_deref_atomic_xor:
5666    case nir_intrinsic_image_deref_atomic_exchange:
5667    case nir_intrinsic_image_deref_atomic_comp_swap:
5668       visit_image_atomic(ctx, instr);
5669       break;
5670    case nir_intrinsic_image_deref_size:
5671       visit_image_size(ctx, instr);
5672       break;
5673    case nir_intrinsic_load_ssbo:
5674       visit_load_ssbo(ctx, instr);
5675       break;
5676    case nir_intrinsic_store_ssbo:
5677       visit_store_ssbo(ctx, instr);
5678       break;
5679    case nir_intrinsic_load_global:
5680       visit_load_global(ctx, instr);
5681       break;
5682    case nir_intrinsic_store_global:
5683       visit_store_global(ctx, instr);
5684       break;
5685    case nir_intrinsic_global_atomic_add:
5686    case nir_intrinsic_global_atomic_imin:
5687    case nir_intrinsic_global_atomic_umin:
5688    case nir_intrinsic_global_atomic_imax:
5689    case nir_intrinsic_global_atomic_umax:
5690    case nir_intrinsic_global_atomic_and:
5691    case nir_intrinsic_global_atomic_or:
5692    case nir_intrinsic_global_atomic_xor:
5693    case nir_intrinsic_global_atomic_exchange:
5694    case nir_intrinsic_global_atomic_comp_swap:
5695       visit_global_atomic(ctx, instr);
5696       break;
5697    case nir_intrinsic_ssbo_atomic_add:
5698    case nir_intrinsic_ssbo_atomic_imin:
5699    case nir_intrinsic_ssbo_atomic_umin:
5700    case nir_intrinsic_ssbo_atomic_imax:
5701    case nir_intrinsic_ssbo_atomic_umax:
5702    case nir_intrinsic_ssbo_atomic_and:
5703    case nir_intrinsic_ssbo_atomic_or:
5704    case nir_intrinsic_ssbo_atomic_xor:
5705    case nir_intrinsic_ssbo_atomic_exchange:
5706    case nir_intrinsic_ssbo_atomic_comp_swap:
5707       visit_atomic_ssbo(ctx, instr);
5708       break;
5709    case nir_intrinsic_load_scratch:
5710       visit_load_scratch(ctx, instr);
5711       break;
5712    case nir_intrinsic_store_scratch:
5713       visit_store_scratch(ctx, instr);
5714       break;
5715    case nir_intrinsic_get_buffer_size:
5716       visit_get_buffer_size(ctx, instr);
5717       break;
5718    case nir_intrinsic_control_barrier: {
5719       unsigned* bsize = ctx->program->info->cs.block_size;
5720       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5721       if (workgroup_size > ctx->program->wave_size)
5722          bld.sopp(aco_opcode::s_barrier);
5723       break;
5724    }
5725    case nir_intrinsic_group_memory_barrier:
5726    case nir_intrinsic_memory_barrier:
5727    case nir_intrinsic_memory_barrier_buffer:
5728    case nir_intrinsic_memory_barrier_image:
5729    case nir_intrinsic_memory_barrier_shared:
5730       emit_memory_barrier(ctx, instr);
5731       break;
5732    case nir_intrinsic_memory_barrier_tcs_patch:
5733       break;
5734    case nir_intrinsic_load_num_work_groups: {
5735       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5736       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5737       emit_split_vector(ctx, dst, 3);
5738       break;
5739    }
5740    case nir_intrinsic_load_local_invocation_id: {
5741       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5742       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5743       emit_split_vector(ctx, dst, 3);
5744       break;
5745    }
5746    case nir_intrinsic_load_work_group_id: {
5747       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5748       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5749       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5750                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5751                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5752                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5753       emit_split_vector(ctx, dst, 3);
5754       break;
5755    }
5756    case nir_intrinsic_load_local_invocation_index: {
5757       Temp id = emit_mbcnt(ctx, bld.def(v1));
5758
5759       /* The tg_size bits [6:11] contain the subgroup id,
5760        * we need this multiplied by the wave size, and then OR the thread id to it.
5761        */
5762       if (ctx->program->wave_size == 64) {
5763          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5764          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5765                                 get_arg(ctx, ctx->args->ac.tg_size));
5766          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5767       } else {
5768          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5769          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5770                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5771          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5772       }
5773       break;
5774    }
5775    case nir_intrinsic_load_subgroup_id: {
5776       if (ctx->stage == compute_cs) {
5777          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5778                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5779       } else {
5780          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5781       }
5782       break;
5783    }
5784    case nir_intrinsic_load_subgroup_invocation: {
5785       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5786       break;
5787    }
5788    case nir_intrinsic_load_num_subgroups: {
5789       if (ctx->stage == compute_cs)
5790          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5791                   get_arg(ctx, ctx->args->ac.tg_size));
5792       else
5793          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5794       break;
5795    }
5796    case nir_intrinsic_ballot: {
5797       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5798       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5799       Definition tmp = bld.def(dst.regClass());
5800       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5801       if (instr->src[0].ssa->bit_size == 1) {
5802          assert(src.regClass() == bld.lm);
5803          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5804       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5805          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5806       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5807          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5808       } else {
5809          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5810          nir_print_instr(&instr->instr, stderr);
5811          fprintf(stderr, "\n");
5812       }
5813       if (dst.size() != bld.lm.size()) {
5814          /* Wave32 with ballot size set to 64 */
5815          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5816       }
5817       emit_wqm(ctx, tmp.getTemp(), dst);
5818       break;
5819    }
5820    case nir_intrinsic_shuffle:
5821    case nir_intrinsic_read_invocation: {
5822       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5823       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5824          emit_uniform_subgroup(ctx, instr, src);
5825       } else {
5826          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5827          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5828             tid = bld.as_uniform(tid);
5829          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5830          if (src.regClass() == v1) {
5831             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5832          } else if (src.regClass() == v2) {
5833             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5834             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5835             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5836             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5837             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5838             emit_split_vector(ctx, dst, 2);
5839          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5840             assert(src.regClass() == bld.lm);
5841             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5842             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5843          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5844             assert(src.regClass() == bld.lm);
5845             Temp tmp;
5846             if (ctx->program->chip_class <= GFX7)
5847                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
5848             else if (ctx->program->wave_size == 64)
5849                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5850             else
5851                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5852             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5853             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5854             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5855          } else {
5856             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5857             nir_print_instr(&instr->instr, stderr);
5858             fprintf(stderr, "\n");
5859          }
5860       }
5861       break;
5862    }
5863    case nir_intrinsic_load_sample_id: {
5864       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5865                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5866       break;
5867    }
5868    case nir_intrinsic_load_sample_mask_in: {
5869       visit_load_sample_mask_in(ctx, instr);
5870       break;
5871    }
5872    case nir_intrinsic_read_first_invocation: {
5873       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5874       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5875       if (src.regClass() == v1) {
5876          emit_wqm(ctx,
5877                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5878                   dst);
5879       } else if (src.regClass() == v2) {
5880          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5881          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5882          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5883          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5884          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5885          emit_split_vector(ctx, dst, 2);
5886       } else if (instr->dest.ssa.bit_size == 1) {
5887          assert(src.regClass() == bld.lm);
5888          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5889                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5890          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5891       } else if (src.regClass() == s1) {
5892          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5893       } else if (src.regClass() == s2) {
5894          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5895       } else {
5896          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5897          nir_print_instr(&instr->instr, stderr);
5898          fprintf(stderr, "\n");
5899       }
5900       break;
5901    }
5902    case nir_intrinsic_vote_all: {
5903       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5904       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5905       assert(src.regClass() == bld.lm);
5906       assert(dst.regClass() == bld.lm);
5907
5908       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5909       Temp all = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), bld.scc(tmp), Operand(0u));
5910       bool_to_vector_condition(ctx, emit_wqm(ctx, all), dst);
5911       break;
5912    }
5913    case nir_intrinsic_vote_any: {
5914       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5915       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5916       assert(src.regClass() == bld.lm);
5917       assert(dst.regClass() == bld.lm);
5918
5919       Temp tmp = bool_to_scalar_condition(ctx, src);
5920       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5921       break;
5922    }
5923    case nir_intrinsic_reduce:
5924    case nir_intrinsic_inclusive_scan:
5925    case nir_intrinsic_exclusive_scan: {
5926       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5927       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5928       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5929       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5930          nir_intrinsic_cluster_size(instr) : 0;
5931       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5932
5933       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5934          emit_uniform_subgroup(ctx, instr, src);
5935       } else if (instr->dest.ssa.bit_size == 1) {
5936          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5937             op = nir_op_iand;
5938          else if (op == nir_op_iadd)
5939             op = nir_op_ixor;
5940          else if (op == nir_op_umax || op == nir_op_imax)
5941             op = nir_op_ior;
5942          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5943
5944          switch (instr->intrinsic) {
5945          case nir_intrinsic_reduce:
5946             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5947             break;
5948          case nir_intrinsic_exclusive_scan:
5949             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5950             break;
5951          case nir_intrinsic_inclusive_scan:
5952             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5953             break;
5954          default:
5955             assert(false);
5956          }
5957       } else if (cluster_size == 1) {
5958          bld.copy(Definition(dst), src);
5959       } else {
5960          src = as_vgpr(ctx, src);
5961
5962          ReduceOp reduce_op;
5963          switch (op) {
5964          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5965             CASE(iadd)
5966             CASE(imul)
5967             CASE(fadd)
5968             CASE(fmul)
5969             CASE(imin)
5970             CASE(umin)
5971             CASE(fmin)
5972             CASE(imax)
5973             CASE(umax)
5974             CASE(fmax)
5975             CASE(iand)
5976             CASE(ior)
5977             CASE(ixor)
5978             default:
5979                unreachable("unknown reduction op");
5980          #undef CASE
5981          }
5982
5983          aco_opcode aco_op;
5984          switch (instr->intrinsic) {
5985             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5986             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5987             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5988             default:
5989                unreachable("unknown reduce intrinsic");
5990          }
5991
5992          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5993          reduce->operands[0] = Operand(src);
5994          // filled in by aco_reduce_assign.cpp, used internally as part of the
5995          // reduce sequence
5996          assert(dst.size() == 1 || dst.size() == 2);
5997          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5998          reduce->operands[2] = Operand(v1.as_linear());
5999
6000          Temp tmp_dst = bld.tmp(dst.regClass());
6001          reduce->definitions[0] = Definition(tmp_dst);
6002          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6003          reduce->definitions[2] = Definition();
6004          reduce->definitions[3] = Definition(scc, s1);
6005          reduce->definitions[4] = Definition();
6006          reduce->reduce_op = reduce_op;
6007          reduce->cluster_size = cluster_size;
6008          ctx->block->instructions.emplace_back(std::move(reduce));
6009
6010          emit_wqm(ctx, tmp_dst, dst);
6011       }
6012       break;
6013    }
6014    case nir_intrinsic_quad_broadcast: {
6015       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6016       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6017          emit_uniform_subgroup(ctx, instr, src);
6018       } else {
6019          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6020          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6021          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6022
6023          if (instr->dest.ssa.bit_size == 1) {
6024             assert(src.regClass() == bld.lm);
6025             assert(dst.regClass() == bld.lm);
6026             uint32_t half_mask = 0x11111111u << lane;
6027             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6028             Temp tmp = bld.tmp(bld.lm);
6029             bld.sop1(Builder::s_wqm, Definition(tmp),
6030                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6031                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6032             emit_wqm(ctx, tmp, dst);
6033          } else if (instr->dest.ssa.bit_size == 32) {
6034             if (ctx->program->chip_class >= GFX8)
6035                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6036             else
6037                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6038          } else if (instr->dest.ssa.bit_size == 64) {
6039             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6040             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6041             if (ctx->program->chip_class >= GFX8) {
6042                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6043                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6044             } else {
6045                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6046                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6047             }
6048             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6049             emit_split_vector(ctx, dst, 2);
6050          } else {
6051             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6052             nir_print_instr(&instr->instr, stderr);
6053             fprintf(stderr, "\n");
6054          }
6055       }
6056       break;
6057    }
6058    case nir_intrinsic_quad_swap_horizontal:
6059    case nir_intrinsic_quad_swap_vertical:
6060    case nir_intrinsic_quad_swap_diagonal:
6061    case nir_intrinsic_quad_swizzle_amd: {
6062       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6063       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6064          emit_uniform_subgroup(ctx, instr, src);
6065          break;
6066       }
6067       uint16_t dpp_ctrl = 0;
6068       switch (instr->intrinsic) {
6069       case nir_intrinsic_quad_swap_horizontal:
6070          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6071          break;
6072       case nir_intrinsic_quad_swap_vertical:
6073          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6074          break;
6075       case nir_intrinsic_quad_swap_diagonal:
6076          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6077          break;
6078       case nir_intrinsic_quad_swizzle_amd:
6079          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6080          break;
6081       default:
6082          break;
6083       }
6084       if (ctx->program->chip_class < GFX8)
6085          dpp_ctrl |= (1 << 15);
6086
6087       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6088       if (instr->dest.ssa.bit_size == 1) {
6089          assert(src.regClass() == bld.lm);
6090          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6091          if (ctx->program->chip_class >= GFX8)
6092             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6093          else
6094             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6095          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6096          emit_wqm(ctx, tmp, dst);
6097       } else if (instr->dest.ssa.bit_size == 32) {
6098          Temp tmp;
6099          if (ctx->program->chip_class >= GFX8)
6100             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6101          else
6102             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6103          emit_wqm(ctx, tmp, dst);
6104       } else if (instr->dest.ssa.bit_size == 64) {
6105          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6106          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6107          if (ctx->program->chip_class >= GFX8) {
6108             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6109             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6110          } else {
6111             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6112             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6113          }
6114          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6115          emit_split_vector(ctx, dst, 2);
6116       } else {
6117          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6118          nir_print_instr(&instr->instr, stderr);
6119          fprintf(stderr, "\n");
6120       }
6121       break;
6122    }
6123    case nir_intrinsic_masked_swizzle_amd: {
6124       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6125       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6126          emit_uniform_subgroup(ctx, instr, src);
6127          break;
6128       }
6129       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6130       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6131       if (dst.regClass() == v1) {
6132          emit_wqm(ctx,
6133                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6134                   dst);
6135       } else if (dst.regClass() == v2) {
6136          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6137          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6138          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6139          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6140          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6141          emit_split_vector(ctx, dst, 2);
6142       } else {
6143          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6144          nir_print_instr(&instr->instr, stderr);
6145          fprintf(stderr, "\n");
6146       }
6147       break;
6148    }
6149    case nir_intrinsic_write_invocation_amd: {
6150       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6151       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6152       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6153       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6154       if (dst.regClass() == v1) {
6155          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6156          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6157       } else if (dst.regClass() == v2) {
6158          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6159          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6160          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6161          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6162          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6163          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6164          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6165          emit_split_vector(ctx, dst, 2);
6166       } else {
6167          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6168          nir_print_instr(&instr->instr, stderr);
6169          fprintf(stderr, "\n");
6170       }
6171       break;
6172    }
6173    case nir_intrinsic_mbcnt_amd: {
6174       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6175       RegClass rc = RegClass(src.type(), 1);
6176       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6177       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6178       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6179       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6180       emit_wqm(ctx, wqm_tmp, dst);
6181       break;
6182    }
6183    case nir_intrinsic_load_helper_invocation: {
6184       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6185       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6186       ctx->block->kind |= block_kind_needs_lowering;
6187       ctx->program->needs_exact = true;
6188       break;
6189    }
6190    case nir_intrinsic_is_helper_invocation: {
6191       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6192       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6193       ctx->block->kind |= block_kind_needs_lowering;
6194       ctx->program->needs_exact = true;
6195       break;
6196    }
6197    case nir_intrinsic_demote:
6198       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6199
6200       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6201          ctx->cf_info.exec_potentially_empty = true;
6202       ctx->block->kind |= block_kind_uses_demote;
6203       ctx->program->needs_exact = true;
6204       break;
6205    case nir_intrinsic_demote_if: {
6206       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6207       assert(src.regClass() == bld.lm);
6208       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6209       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6210
6211       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6212          ctx->cf_info.exec_potentially_empty = true;
6213       ctx->block->kind |= block_kind_uses_demote;
6214       ctx->program->needs_exact = true;
6215       break;
6216    }
6217    case nir_intrinsic_first_invocation: {
6218       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6219                get_ssa_temp(ctx, &instr->dest.ssa));
6220       break;
6221    }
6222    case nir_intrinsic_shader_clock:
6223       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6224       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6225       break;
6226    case nir_intrinsic_load_vertex_id_zero_base: {
6227       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6228       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6229       break;
6230    }
6231    case nir_intrinsic_load_first_vertex: {
6232       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6233       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6234       break;
6235    }
6236    case nir_intrinsic_load_base_instance: {
6237       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6238       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6239       break;
6240    }
6241    case nir_intrinsic_load_instance_id: {
6242       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6243       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6244       break;
6245    }
6246    case nir_intrinsic_load_draw_id: {
6247       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6248       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6249       break;
6250    }
6251    default:
6252       fprintf(stderr, "Unimplemented intrinsic instr: ");
6253       nir_print_instr(&instr->instr, stderr);
6254       fprintf(stderr, "\n");
6255       abort();
6256
6257       break;
6258    }
6259 }
6260
6261
6262 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6263                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6264                     enum glsl_base_type *stype)
6265 {
6266    nir_deref_instr *texture_deref_instr = NULL;
6267    nir_deref_instr *sampler_deref_instr = NULL;
6268    int plane = -1;
6269
6270    for (unsigned i = 0; i < instr->num_srcs; i++) {
6271       switch (instr->src[i].src_type) {
6272       case nir_tex_src_texture_deref:
6273          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6274          break;
6275       case nir_tex_src_sampler_deref:
6276          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6277          break;
6278       case nir_tex_src_plane:
6279          plane = nir_src_as_int(instr->src[i].src);
6280          break;
6281       default:
6282          break;
6283       }
6284    }
6285
6286    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6287
6288    if (!sampler_deref_instr)
6289       sampler_deref_instr = texture_deref_instr;
6290
6291    if (plane >= 0) {
6292       assert(instr->op != nir_texop_txf_ms &&
6293              instr->op != nir_texop_samples_identical);
6294       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6295       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6296    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6297       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6298    } else {
6299       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6300    }
6301    if (samp_ptr) {
6302       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6303
6304       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6305          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6306          Builder bld(ctx->program, ctx->block);
6307
6308          /* to avoid unnecessary moves, we split and recombine sampler and image */
6309          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6310                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6311          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6312          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6313                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6314                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6315          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6316                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6317
6318          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6319          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6320                                img[0], img[1], img[2], img[3],
6321                                img[4], img[5], img[6], img[7]);
6322          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6323                                 samp[0], samp[1], samp[2], samp[3]);
6324       }
6325    }
6326    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6327                      instr->op == nir_texop_samples_identical))
6328       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6329 }
6330
6331 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6332                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6333 {
6334    Builder bld(ctx->program, ctx->block);
6335
6336    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6337    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6338    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6339
6340    Operand neg_one(0xbf800000u);
6341    Operand one(0x3f800000u);
6342    Operand two(0x40000000u);
6343    Operand four(0x40800000u);
6344
6345    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6346    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6347    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6348
6349    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6350    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6351    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6352    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6353
6354    // select sc
6355    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6356    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6357                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6358                        one, is_ma_y);
6359    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6360
6361    // select tc
6362    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6363    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6364    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6365
6366    // select ma
6367    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6368                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6369                   deriv_z, is_ma_z);
6370    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6371    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6372 }
6373
6374 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6375 {
6376    Builder bld(ctx->program, ctx->block);
6377    Temp coord_args[4], ma, tc, sc, id;
6378    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6379       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6380
6381    if (is_array) {
6382       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6383
6384       // see comment in ac_prepare_cube_coords()
6385       if (ctx->options->chip_class <= GFX8)
6386          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6387    }
6388
6389    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6390
6391    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6392    vop3a->operands[0] = Operand(ma);
6393    vop3a->abs[0] = true;
6394    Temp invma = bld.tmp(v1);
6395    vop3a->definitions[0] = Definition(invma);
6396    ctx->block->instructions.emplace_back(std::move(vop3a));
6397
6398    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6399    if (!is_deriv)
6400       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6401
6402    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6403    if (!is_deriv)
6404       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6405
6406    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6407
6408    if (is_deriv) {
6409       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6410       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6411
6412       for (unsigned i = 0; i < 2; i++) {
6413          // see comment in ac_prepare_cube_coords()
6414          Temp deriv_ma;
6415          Temp deriv_sc, deriv_tc;
6416          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6417                            &deriv_ma, &deriv_sc, &deriv_tc);
6418
6419          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6420
6421          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6422                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6423                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6424          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6425                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6426                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6427          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6428       }
6429
6430       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6431       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6432    }
6433
6434    if (is_array)
6435       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6436    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6437
6438 }
6439
6440 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6441 {
6442    Temp coord_vec[3];
6443    for (unsigned i = 0; i < coords.size(); i++)
6444       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6445
6446    Builder bld(ctx->program, ctx->block);
6447    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6448
6449    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6450    for (unsigned i = 0; i < coords.size(); i++)
6451       vec->operands[i] = Operand(coord_vec[i]);
6452    Temp res = bld.tmp(RegType::vgpr, coords.size());
6453    vec->definitions[0] = Definition(res);
6454    ctx->block->instructions.emplace_back(std::move(vec));
6455    return res;
6456 }
6457
6458 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6459 {
6460    if (vec->parent_instr->type != nir_instr_type_alu)
6461       return;
6462    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6463    if (vec_instr->op != nir_op_vec(vec->num_components))
6464       return;
6465
6466    for (unsigned i = 0; i < vec->num_components; i++) {
6467       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6468               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6469    }
6470 }
6471
6472 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6473 {
6474    Builder bld(ctx->program, ctx->block);
6475    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6476         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6477    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6478         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6479    nir_const_value *sample_index_cv = NULL;
6480    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6481    enum glsl_base_type stype;
6482    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6483
6484    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6485                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6486    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6487                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6488
6489    for (unsigned i = 0; i < instr->num_srcs; i++) {
6490       switch (instr->src[i].src_type) {
6491       case nir_tex_src_coord:
6492          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6493          break;
6494       case nir_tex_src_bias:
6495          if (instr->op == nir_texop_txb) {
6496             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6497             has_bias = true;
6498          }
6499          break;
6500       case nir_tex_src_lod: {
6501          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6502
6503          if (val && val->f32 <= 0.0) {
6504             level_zero = true;
6505          } else {
6506             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6507             has_lod = true;
6508          }
6509          break;
6510       }
6511       case nir_tex_src_comparator:
6512          if (instr->is_shadow) {
6513             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6514             has_compare = true;
6515          }
6516          break;
6517       case nir_tex_src_offset:
6518          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6519          get_const_vec(instr->src[i].src.ssa, const_offset);
6520          has_offset = true;
6521          break;
6522       case nir_tex_src_ddx:
6523          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6524          has_ddx = true;
6525          break;
6526       case nir_tex_src_ddy:
6527          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6528          has_ddy = true;
6529          break;
6530       case nir_tex_src_ms_index:
6531          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6532          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6533          has_sample_index = true;
6534          break;
6535       case nir_tex_src_texture_offset:
6536       case nir_tex_src_sampler_offset:
6537       default:
6538          break;
6539       }
6540    }
6541 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6542    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6543       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6544
6545    if (instr->op == nir_texop_texture_samples) {
6546       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6547
6548       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6549       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6550       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6551       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6552
6553       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6554                samples, Operand(1u), bld.scc(is_msaa));
6555       return;
6556    }
6557
6558    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6559       aco_ptr<Instruction> tmp_instr;
6560       Temp acc, pack = Temp();
6561
6562       uint32_t pack_const = 0;
6563       for (unsigned i = 0; i < offset.size(); i++) {
6564          if (!const_offset[i])
6565             continue;
6566          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6567       }
6568
6569       if (offset.type() == RegType::sgpr) {
6570          for (unsigned i = 0; i < offset.size(); i++) {
6571             if (const_offset[i])
6572                continue;
6573
6574             acc = emit_extract_vector(ctx, offset, i, s1);
6575             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6576
6577             if (i) {
6578                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6579             }
6580
6581             if (pack == Temp()) {
6582                pack = acc;
6583             } else {
6584                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6585             }
6586          }
6587
6588          if (pack_const && pack != Temp())
6589             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6590       } else {
6591          for (unsigned i = 0; i < offset.size(); i++) {
6592             if (const_offset[i])
6593                continue;
6594
6595             acc = emit_extract_vector(ctx, offset, i, v1);
6596             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6597
6598             if (i) {
6599                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6600             }
6601
6602             if (pack == Temp()) {
6603                pack = acc;
6604             } else {
6605                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6606             }
6607          }
6608
6609          if (pack_const && pack != Temp())
6610             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6611       }
6612       if (pack_const && pack == Temp())
6613          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6614       else if (pack == Temp())
6615          has_offset = false;
6616       else
6617          offset = pack;
6618    }
6619
6620    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6621       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6622
6623    /* pack derivatives */
6624    if (has_ddx || has_ddy) {
6625       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6626          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6627                              ddx, Operand(0u), ddy, Operand(0u));
6628       } else {
6629          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6630       }
6631       has_derivs = true;
6632    }
6633
6634    if (instr->coord_components > 1 &&
6635        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6636        instr->is_array &&
6637        instr->op != nir_texop_txf)
6638       coords = apply_round_slice(ctx, coords, 1);
6639
6640    if (instr->coord_components > 2 &&
6641       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6642        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6643        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6644        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6645        instr->is_array &&
6646        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6647       coords = apply_round_slice(ctx, coords, 2);
6648
6649    if (ctx->options->chip_class == GFX9 &&
6650        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6651        instr->op != nir_texop_lod && instr->coord_components) {
6652       assert(coords.size() > 0 && coords.size() < 3);
6653
6654       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6655       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6656       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6657       if (coords.size() > 1)
6658          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6659       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6660       vec->definitions[0] = Definition(coords);
6661       ctx->block->instructions.emplace_back(std::move(vec));
6662    }
6663
6664    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6665
6666    if (instr->op == nir_texop_samples_identical)
6667       resource = fmask_ptr;
6668
6669    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6670              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6671             instr->op != nir_texop_txs) {
6672       assert(has_sample_index);
6673       Operand op(sample_index);
6674       if (sample_index_cv)
6675          op = Operand(sample_index_cv->u32);
6676       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6677    }
6678
6679    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6680       Temp split_coords[coords.size()];
6681       emit_split_vector(ctx, coords, coords.size());
6682       for (unsigned i = 0; i < coords.size(); i++)
6683          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6684
6685       unsigned i = 0;
6686       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6687          Temp off = emit_extract_vector(ctx, offset, i, v1);
6688          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6689       }
6690
6691       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6692       for (unsigned i = 0; i < coords.size(); i++)
6693          vec->operands[i] = Operand(split_coords[i]);
6694       coords = bld.tmp(coords.regClass());
6695       vec->definitions[0] = Definition(coords);
6696       ctx->block->instructions.emplace_back(std::move(vec));
6697
6698       has_offset = false;
6699    }
6700
6701    /* Build tex instruction */
6702    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6703    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6704                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6705                   : 0;
6706    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6707    Temp tmp_dst = dst;
6708
6709    /* gather4 selects the component by dmask and always returns vec4 */
6710    if (instr->op == nir_texop_tg4) {
6711       assert(instr->dest.ssa.num_components == 4);
6712       if (instr->is_shadow)
6713          dmask = 1;
6714       else
6715          dmask = 1 << instr->component;
6716       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6717          tmp_dst = bld.tmp(v4);
6718    } else if (instr->op == nir_texop_samples_identical) {
6719       tmp_dst = bld.tmp(v1);
6720    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6721       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6722    }
6723
6724    aco_ptr<MIMG_instruction> tex;
6725    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6726       if (!has_lod)
6727          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6728
6729       bool div_by_6 = instr->op == nir_texop_txs &&
6730                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6731                       instr->is_array &&
6732                       (dmask & (1 << 2));
6733       if (tmp_dst.id() == dst.id() && div_by_6)
6734          tmp_dst = bld.tmp(tmp_dst.regClass());
6735
6736       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6737       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6738       tex->operands[1] = Operand(resource);
6739       if (ctx->options->chip_class == GFX9 &&
6740           instr->op == nir_texop_txs &&
6741           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6742           instr->is_array) {
6743          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6744       } else if (instr->op == nir_texop_query_levels) {
6745          tex->dmask = 1 << 3;
6746       } else {
6747          tex->dmask = dmask;
6748       }
6749       tex->da = da;
6750       tex->definitions[0] = Definition(tmp_dst);
6751       tex->dim = dim;
6752       tex->can_reorder = true;
6753       ctx->block->instructions.emplace_back(std::move(tex));
6754
6755       if (div_by_6) {
6756          /* divide 3rd value by 6 by multiplying with magic number */
6757          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6758          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6759          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6760          assert(instr->dest.ssa.num_components == 3);
6761          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6762          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6763                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6764                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6765                               by_6);
6766
6767       }
6768
6769       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6770       return;
6771    }
6772
6773    Temp tg4_compare_cube_wa64 = Temp();
6774
6775    if (tg4_integer_workarounds) {
6776       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6777       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6778       tex->operands[1] = Operand(resource);
6779       tex->dim = dim;
6780       tex->dmask = 0x3;
6781       tex->da = da;
6782       Temp size = bld.tmp(v2);
6783       tex->definitions[0] = Definition(size);
6784       tex->can_reorder = true;
6785       ctx->block->instructions.emplace_back(std::move(tex));
6786       emit_split_vector(ctx, size, size.size());
6787
6788       Temp half_texel[2];
6789       for (unsigned i = 0; i < 2; i++) {
6790          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6791          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6792          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6793          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6794       }
6795
6796       Temp orig_coords[2] = {
6797          emit_extract_vector(ctx, coords, 0, v1),
6798          emit_extract_vector(ctx, coords, 1, v1)};
6799       Temp new_coords[2] = {
6800          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6801          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6802       };
6803
6804       if (tg4_integer_cube_workaround) {
6805          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6806          Temp desc[resource.size()];
6807          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6808                                                                            Format::PSEUDO, 1, resource.size())};
6809          split->operands[0] = Operand(resource);
6810          for (unsigned i = 0; i < resource.size(); i++) {
6811             desc[i] = bld.tmp(s1);
6812             split->definitions[i] = Definition(desc[i]);
6813          }
6814          ctx->block->instructions.emplace_back(std::move(split));
6815
6816          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6817          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6818                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6819
6820          Temp nfmt;
6821          if (stype == GLSL_TYPE_UINT) {
6822             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6823                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6824                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6825                             bld.scc(compare_cube_wa));
6826          } else {
6827             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6828                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6829                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6830                             bld.scc(compare_cube_wa));
6831          }
6832          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6833          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6834
6835          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6836
6837          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6838                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6839          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6840
6841          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6842                                                                          Format::PSEUDO, resource.size(), 1)};
6843          for (unsigned i = 0; i < resource.size(); i++)
6844             vec->operands[i] = Operand(desc[i]);
6845          resource = bld.tmp(resource.regClass());
6846          vec->definitions[0] = Definition(resource);
6847          ctx->block->instructions.emplace_back(std::move(vec));
6848
6849          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6850                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6851          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6852                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6853       }
6854
6855       if (coords.size() == 3) {
6856          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6857                              new_coords[0], new_coords[1],
6858                              emit_extract_vector(ctx, coords, 2, v1));
6859       } else {
6860          assert(coords.size() == 2);
6861          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6862                              new_coords[0], new_coords[1]);
6863       }
6864    }
6865
6866    std::vector<Operand> args;
6867    if (has_offset)
6868       args.emplace_back(Operand(offset));
6869    if (has_bias)
6870       args.emplace_back(Operand(bias));
6871    if (has_compare)
6872       args.emplace_back(Operand(compare));
6873    if (has_derivs)
6874       args.emplace_back(Operand(derivs));
6875    args.emplace_back(Operand(coords));
6876    if (has_sample_index)
6877       args.emplace_back(Operand(sample_index));
6878    if (has_lod)
6879       args.emplace_back(lod);
6880
6881    Temp arg;
6882    if (args.size() > 1) {
6883       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6884       unsigned size = 0;
6885       for (unsigned i = 0; i < args.size(); i++) {
6886          size += args[i].size();
6887          vec->operands[i] = args[i];
6888       }
6889       RegClass rc = RegClass(RegType::vgpr, size);
6890       Temp tmp = bld.tmp(rc);
6891       vec->definitions[0] = Definition(tmp);
6892       ctx->block->instructions.emplace_back(std::move(vec));
6893       arg = tmp;
6894    } else {
6895       assert(args[0].isTemp());
6896       arg = as_vgpr(ctx, args[0].getTemp());
6897    }
6898
6899    /* we don't need the bias, sample index, compare value or offset to be
6900     * computed in WQM but if the p_create_vector copies the coordinates, then it
6901     * needs to be in WQM */
6902    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6903        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6904        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6905       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6906
6907    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6908       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6909
6910       assert(coords.size() == 1);
6911       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6912       aco_opcode op;
6913       switch (last_bit) {
6914       case 1:
6915          op = aco_opcode::buffer_load_format_x; break;
6916       case 2:
6917          op = aco_opcode::buffer_load_format_xy; break;
6918       case 3:
6919          op = aco_opcode::buffer_load_format_xyz; break;
6920       case 4:
6921          op = aco_opcode::buffer_load_format_xyzw; break;
6922       default:
6923          unreachable("Tex instruction loads more than 4 components.");
6924       }
6925
6926       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6927       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6928          tmp_dst = dst;
6929       else
6930          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6931
6932       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6933       mubuf->operands[0] = Operand(coords);
6934       mubuf->operands[1] = Operand(resource);
6935       mubuf->operands[2] = Operand((uint32_t) 0);
6936       mubuf->definitions[0] = Definition(tmp_dst);
6937       mubuf->idxen = true;
6938       mubuf->can_reorder = true;
6939       ctx->block->instructions.emplace_back(std::move(mubuf));
6940
6941       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6942       return;
6943    }
6944
6945
6946    if (instr->op == nir_texop_txf ||
6947        instr->op == nir_texop_txf_ms ||
6948        instr->op == nir_texop_samples_identical) {
6949       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6950       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6951       tex->operands[0] = Operand(arg);
6952       tex->operands[1] = Operand(resource);
6953       tex->dim = dim;
6954       tex->dmask = dmask;
6955       tex->unrm = true;
6956       tex->da = da;
6957       tex->definitions[0] = Definition(tmp_dst);
6958       tex->can_reorder = true;
6959       ctx->block->instructions.emplace_back(std::move(tex));
6960
6961       if (instr->op == nir_texop_samples_identical) {
6962          assert(dmask == 1 && dst.regClass() == v1);
6963          assert(dst.id() != tmp_dst.id());
6964
6965          Temp tmp = bld.tmp(bld.lm);
6966          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6967          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6968
6969       } else {
6970          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6971       }
6972       return;
6973    }
6974
6975    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6976    aco_opcode opcode = aco_opcode::image_sample;
6977    if (has_offset) { /* image_sample_*_o */
6978       if (has_compare) {
6979          opcode = aco_opcode::image_sample_c_o;
6980          if (has_derivs)
6981             opcode = aco_opcode::image_sample_c_d_o;
6982          if (has_bias)
6983             opcode = aco_opcode::image_sample_c_b_o;
6984          if (level_zero)
6985             opcode = aco_opcode::image_sample_c_lz_o;
6986          if (has_lod)
6987             opcode = aco_opcode::image_sample_c_l_o;
6988       } else {
6989          opcode = aco_opcode::image_sample_o;
6990          if (has_derivs)
6991             opcode = aco_opcode::image_sample_d_o;
6992          if (has_bias)
6993             opcode = aco_opcode::image_sample_b_o;
6994          if (level_zero)
6995             opcode = aco_opcode::image_sample_lz_o;
6996          if (has_lod)
6997             opcode = aco_opcode::image_sample_l_o;
6998       }
6999    } else { /* no offset */
7000       if (has_compare) {
7001          opcode = aco_opcode::image_sample_c;
7002          if (has_derivs)
7003             opcode = aco_opcode::image_sample_c_d;
7004          if (has_bias)
7005             opcode = aco_opcode::image_sample_c_b;
7006          if (level_zero)
7007             opcode = aco_opcode::image_sample_c_lz;
7008          if (has_lod)
7009             opcode = aco_opcode::image_sample_c_l;
7010       } else {
7011          opcode = aco_opcode::image_sample;
7012          if (has_derivs)
7013             opcode = aco_opcode::image_sample_d;
7014          if (has_bias)
7015             opcode = aco_opcode::image_sample_b;
7016          if (level_zero)
7017             opcode = aco_opcode::image_sample_lz;
7018          if (has_lod)
7019             opcode = aco_opcode::image_sample_l;
7020       }
7021    }
7022
7023    if (instr->op == nir_texop_tg4) {
7024       if (has_offset) {
7025          opcode = aco_opcode::image_gather4_lz_o;
7026          if (has_compare)
7027             opcode = aco_opcode::image_gather4_c_lz_o;
7028       } else {
7029          opcode = aco_opcode::image_gather4_lz;
7030          if (has_compare)
7031             opcode = aco_opcode::image_gather4_c_lz;
7032       }
7033    } else if (instr->op == nir_texop_lod) {
7034       opcode = aco_opcode::image_get_lod;
7035    }
7036
7037    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7038    tex->operands[0] = Operand(arg);
7039    tex->operands[1] = Operand(resource);
7040    tex->operands[2] = Operand(sampler);
7041    tex->dim = dim;
7042    tex->dmask = dmask;
7043    tex->da = da;
7044    tex->definitions[0] = Definition(tmp_dst);
7045    tex->can_reorder = true;
7046    ctx->block->instructions.emplace_back(std::move(tex));
7047
7048    if (tg4_integer_cube_workaround) {
7049       assert(tmp_dst.id() != dst.id());
7050       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7051
7052       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7053       Temp val[4];
7054       for (unsigned i = 0; i < dst.size(); i++) {
7055          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7056          Temp cvt_val;
7057          if (stype == GLSL_TYPE_UINT)
7058             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7059          else
7060             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7061          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7062       }
7063       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7064       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7065                            val[0], val[1], val[2], val[3]);
7066    }
7067    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7068    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7069
7070 }
7071
7072
7073 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7074 {
7075    Temp tmp = get_ssa_temp(ctx, ssa);
7076    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7077       return Operand(tmp.regClass());
7078    else
7079       return Operand(tmp);
7080 }
7081
7082 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7083 {
7084    aco_ptr<Pseudo_instruction> phi;
7085    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7086    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7087
7088    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7089    logical |= ctx->block->kind & block_kind_merge;
7090    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7091
7092    /* we want a sorted list of sources, since the predecessor list is also sorted */
7093    std::map<unsigned, nir_ssa_def*> phi_src;
7094    nir_foreach_phi_src(src, instr)
7095       phi_src[src->pred->index] = src->src.ssa;
7096
7097    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7098    unsigned num_operands = 0;
7099    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7100    unsigned num_defined = 0;
7101    unsigned cur_pred_idx = 0;
7102    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7103       if (cur_pred_idx < preds.size()) {
7104          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7105          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7106          unsigned skipped = 0;
7107          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7108             skipped++;
7109          if (cur_pred_idx + skipped < preds.size()) {
7110             for (unsigned i = 0; i < skipped; i++)
7111                operands[num_operands++] = Operand(dst.regClass());
7112             cur_pred_idx += skipped;
7113          } else {
7114             continue;
7115          }
7116       }
7117       cur_pred_idx++;
7118       Operand op = get_phi_operand(ctx, src.second);
7119       operands[num_operands++] = op;
7120       num_defined += !op.isUndefined();
7121    }
7122    /* handle block_kind_continue_or_break at loop exit blocks */
7123    while (cur_pred_idx++ < preds.size())
7124       operands[num_operands++] = Operand(dst.regClass());
7125
7126    if (num_defined == 0) {
7127       Builder bld(ctx->program, ctx->block);
7128       if (dst.regClass() == s1) {
7129          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7130       } else if (dst.regClass() == v1) {
7131          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7132       } else {
7133          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7134          for (unsigned i = 0; i < dst.size(); i++)
7135             vec->operands[i] = Operand(0u);
7136          vec->definitions[0] = Definition(dst);
7137          ctx->block->instructions.emplace_back(std::move(vec));
7138       }
7139       return;
7140    }
7141
7142    /* we can use a linear phi in some cases if one src is undef */
7143    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7144       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7145
7146       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7147       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7148       assert(invert->kind & block_kind_invert);
7149
7150       unsigned then_block = invert->linear_preds[0];
7151
7152       Block* insert_block = NULL;
7153       for (unsigned i = 0; i < num_operands; i++) {
7154          Operand op = operands[i];
7155          if (op.isUndefined())
7156             continue;
7157          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7158          phi->operands[0] = op;
7159          break;
7160       }
7161       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7162       phi->operands[1] = Operand(dst.regClass());
7163       phi->definitions[0] = Definition(dst);
7164       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7165       return;
7166    }
7167
7168    /* try to scalarize vector phis */
7169    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7170       // TODO: scalarize linear phis on divergent ifs
7171       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7172       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7173       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7174          Operand src = operands[i];
7175          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7176             can_scalarize = false;
7177       }
7178       if (can_scalarize) {
7179          unsigned num_components = instr->dest.ssa.num_components;
7180          assert(dst.size() % num_components == 0);
7181          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7182
7183          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7184          for (unsigned k = 0; k < num_components; k++) {
7185             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7186             for (unsigned i = 0; i < num_operands; i++) {
7187                Operand src = operands[i];
7188                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7189             }
7190             Temp phi_dst = {ctx->program->allocateId(), rc};
7191             phi->definitions[0] = Definition(phi_dst);
7192             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7193             new_vec[k] = phi_dst;
7194             vec->operands[k] = Operand(phi_dst);
7195          }
7196          vec->definitions[0] = Definition(dst);
7197          ctx->block->instructions.emplace_back(std::move(vec));
7198          ctx->allocated_vec.emplace(dst.id(), new_vec);
7199          return;
7200       }
7201    }
7202
7203    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7204    for (unsigned i = 0; i < num_operands; i++)
7205       phi->operands[i] = operands[i];
7206    phi->definitions[0] = Definition(dst);
7207    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7208 }
7209
7210
7211 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7212 {
7213    Temp dst = get_ssa_temp(ctx, &instr->def);
7214
7215    assert(dst.type() == RegType::sgpr);
7216
7217    if (dst.size() == 1) {
7218       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7219    } else {
7220       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7221       for (unsigned i = 0; i < dst.size(); i++)
7222          vec->operands[i] = Operand(0u);
7223       vec->definitions[0] = Definition(dst);
7224       ctx->block->instructions.emplace_back(std::move(vec));
7225    }
7226 }
7227
7228 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7229 {
7230    Builder bld(ctx->program, ctx->block);
7231    Block *logical_target;
7232    append_logical_end(ctx->block);
7233    unsigned idx = ctx->block->index;
7234
7235    switch (instr->type) {
7236    case nir_jump_break:
7237       logical_target = ctx->cf_info.parent_loop.exit;
7238       add_logical_edge(idx, logical_target);
7239       ctx->block->kind |= block_kind_break;
7240
7241       if (!ctx->cf_info.parent_if.is_divergent &&
7242           !ctx->cf_info.parent_loop.has_divergent_continue) {
7243          /* uniform break - directly jump out of the loop */
7244          ctx->block->kind |= block_kind_uniform;
7245          ctx->cf_info.has_branch = true;
7246          bld.branch(aco_opcode::p_branch);
7247          add_linear_edge(idx, logical_target);
7248          return;
7249       }
7250       ctx->cf_info.parent_loop.has_divergent_branch = true;
7251       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7252       break;
7253    case nir_jump_continue:
7254       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7255       add_logical_edge(idx, logical_target);
7256       ctx->block->kind |= block_kind_continue;
7257
7258       if (ctx->cf_info.parent_if.is_divergent) {
7259          /* for potential uniform breaks after this continue,
7260             we must ensure that they are handled correctly */
7261          ctx->cf_info.parent_loop.has_divergent_continue = true;
7262          ctx->cf_info.parent_loop.has_divergent_branch = true;
7263          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7264       } else {
7265          /* uniform continue - directly jump to the loop header */
7266          ctx->block->kind |= block_kind_uniform;
7267          ctx->cf_info.has_branch = true;
7268          bld.branch(aco_opcode::p_branch);
7269          add_linear_edge(idx, logical_target);
7270          return;
7271       }
7272       break;
7273    default:
7274       fprintf(stderr, "Unknown NIR jump instr: ");
7275       nir_print_instr(&instr->instr, stderr);
7276       fprintf(stderr, "\n");
7277       abort();
7278    }
7279
7280    /* remove critical edges from linear CFG */
7281    bld.branch(aco_opcode::p_branch);
7282    Block* break_block = ctx->program->create_and_insert_block();
7283    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7284    break_block->kind |= block_kind_uniform;
7285    add_linear_edge(idx, break_block);
7286    /* the loop_header pointer might be invalidated by this point */
7287    if (instr->type == nir_jump_continue)
7288       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7289    add_linear_edge(break_block->index, logical_target);
7290    bld.reset(break_block);
7291    bld.branch(aco_opcode::p_branch);
7292
7293    Block* continue_block = ctx->program->create_and_insert_block();
7294    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7295    add_linear_edge(idx, continue_block);
7296    append_logical_start(continue_block);
7297    ctx->block = continue_block;
7298    return;
7299 }
7300
7301 void visit_block(isel_context *ctx, nir_block *block)
7302 {
7303    nir_foreach_instr(instr, block) {
7304       switch (instr->type) {
7305       case nir_instr_type_alu:
7306          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7307          break;
7308       case nir_instr_type_load_const:
7309          visit_load_const(ctx, nir_instr_as_load_const(instr));
7310          break;
7311       case nir_instr_type_intrinsic:
7312          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7313          break;
7314       case nir_instr_type_tex:
7315          visit_tex(ctx, nir_instr_as_tex(instr));
7316          break;
7317       case nir_instr_type_phi:
7318          visit_phi(ctx, nir_instr_as_phi(instr));
7319          break;
7320       case nir_instr_type_ssa_undef:
7321          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7322          break;
7323       case nir_instr_type_deref:
7324          break;
7325       case nir_instr_type_jump:
7326          visit_jump(ctx, nir_instr_as_jump(instr));
7327          break;
7328       default:
7329          fprintf(stderr, "Unknown NIR instr type: ");
7330          nir_print_instr(instr, stderr);
7331          fprintf(stderr, "\n");
7332          //abort();
7333       }
7334    }
7335
7336    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7337       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7338 }
7339
7340
7341
7342 static void visit_loop(isel_context *ctx, nir_loop *loop)
7343 {
7344    append_logical_end(ctx->block);
7345    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7346    Builder bld(ctx->program, ctx->block);
7347    bld.branch(aco_opcode::p_branch);
7348    unsigned loop_preheader_idx = ctx->block->index;
7349
7350    Block loop_exit = Block();
7351    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7352    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7353
7354    Block* loop_header = ctx->program->create_and_insert_block();
7355    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7356    loop_header->kind |= block_kind_loop_header;
7357    add_edge(loop_preheader_idx, loop_header);
7358    ctx->block = loop_header;
7359
7360    /* emit loop body */
7361    unsigned loop_header_idx = loop_header->index;
7362    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7363    append_logical_start(ctx->block);
7364    visit_cf_list(ctx, &loop->body);
7365
7366    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7367    if (!ctx->cf_info.has_branch) {
7368       append_logical_end(ctx->block);
7369       if (ctx->cf_info.exec_potentially_empty) {
7370          /* Discards can result in code running with an empty exec mask.
7371           * This would result in divergent breaks not ever being taken. As a
7372           * workaround, break the loop when the loop mask is empty instead of
7373           * always continuing. */
7374          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7375          unsigned block_idx = ctx->block->index;
7376
7377          /* create helper blocks to avoid critical edges */
7378          Block *break_block = ctx->program->create_and_insert_block();
7379          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7380          break_block->kind = block_kind_uniform;
7381          bld.reset(break_block);
7382          bld.branch(aco_opcode::p_branch);
7383          add_linear_edge(block_idx, break_block);
7384          add_linear_edge(break_block->index, &loop_exit);
7385
7386          Block *continue_block = ctx->program->create_and_insert_block();
7387          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7388          continue_block->kind = block_kind_uniform;
7389          bld.reset(continue_block);
7390          bld.branch(aco_opcode::p_branch);
7391          add_linear_edge(block_idx, continue_block);
7392          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7393
7394          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7395          ctx->block = &ctx->program->blocks[block_idx];
7396       } else {
7397          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7398          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7399             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7400          else
7401             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7402       }
7403
7404       bld.reset(ctx->block);
7405       bld.branch(aco_opcode::p_branch);
7406    }
7407
7408    /* fixup phis in loop header from unreachable blocks */
7409    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7410       bool linear = ctx->cf_info.has_branch;
7411       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7412       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7413          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7414              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7415             /* the last operand should be the one that needs to be removed */
7416             instr->operands.pop_back();
7417          } else if (!is_phi(instr)) {
7418             break;
7419          }
7420       }
7421    }
7422
7423    ctx->cf_info.has_branch = false;
7424
7425    // TODO: if the loop has not a single exit, we must add one °°
7426    /* emit loop successor block */
7427    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7428    append_logical_start(ctx->block);
7429
7430    #if 0
7431    // TODO: check if it is beneficial to not branch on continues
7432    /* trim linear phis in loop header */
7433    for (auto&& instr : loop_entry->instructions) {
7434       if (instr->opcode == aco_opcode::p_linear_phi) {
7435          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7436          new_phi->definitions[0] = instr->definitions[0];
7437          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7438             new_phi->operands[i] = instr->operands[i];
7439          /* check that the remaining operands are all the same */
7440          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7441             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7442          instr.swap(new_phi);
7443       } else if (instr->opcode == aco_opcode::p_phi) {
7444          continue;
7445       } else {
7446          break;
7447       }
7448    }
7449    #endif
7450 }
7451
7452 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7453 {
7454    ic->cond = cond;
7455
7456    append_logical_end(ctx->block);
7457    ctx->block->kind |= block_kind_branch;
7458
7459    /* branch to linear then block */
7460    assert(cond.regClass() == ctx->program->lane_mask);
7461    aco_ptr<Pseudo_branch_instruction> branch;
7462    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7463    branch->operands[0] = Operand(cond);
7464    ctx->block->instructions.push_back(std::move(branch));
7465
7466    ic->BB_if_idx = ctx->block->index;
7467    ic->BB_invert = Block();
7468    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7469    /* Invert blocks are intentionally not marked as top level because they
7470     * are not part of the logical cfg. */
7471    ic->BB_invert.kind |= block_kind_invert;
7472    ic->BB_endif = Block();
7473    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7474    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7475
7476    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7477    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7478    ctx->cf_info.parent_if.is_divergent = true;
7479    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7480
7481    /** emit logical then block */
7482    Block* BB_then_logical = ctx->program->create_and_insert_block();
7483    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7484    add_edge(ic->BB_if_idx, BB_then_logical);
7485    ctx->block = BB_then_logical;
7486    append_logical_start(BB_then_logical);
7487 }
7488
7489 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7490 {
7491    Block *BB_then_logical = ctx->block;
7492    append_logical_end(BB_then_logical);
7493     /* branch from logical then block to invert block */
7494    aco_ptr<Pseudo_branch_instruction> branch;
7495    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7496    BB_then_logical->instructions.emplace_back(std::move(branch));
7497    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7498    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7499       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7500    BB_then_logical->kind |= block_kind_uniform;
7501    assert(!ctx->cf_info.has_branch);
7502    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7503    ctx->cf_info.parent_loop.has_divergent_branch = false;
7504
7505    /** emit linear then block */
7506    Block* BB_then_linear = ctx->program->create_and_insert_block();
7507    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7508    BB_then_linear->kind |= block_kind_uniform;
7509    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7510    /* branch from linear then block to invert block */
7511    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7512    BB_then_linear->instructions.emplace_back(std::move(branch));
7513    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7514
7515    /** emit invert merge block */
7516    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7517    ic->invert_idx = ctx->block->index;
7518
7519    /* branch to linear else block (skip else) */
7520    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7521    branch->operands[0] = Operand(ic->cond);
7522    ctx->block->instructions.push_back(std::move(branch));
7523
7524    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7525    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7526
7527    /** emit logical else block */
7528    Block* BB_else_logical = ctx->program->create_and_insert_block();
7529    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7530    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7531    add_linear_edge(ic->invert_idx, BB_else_logical);
7532    ctx->block = BB_else_logical;
7533    append_logical_start(BB_else_logical);
7534 }
7535
7536 static void end_divergent_if(isel_context *ctx, if_context *ic)
7537 {
7538    Block *BB_else_logical = ctx->block;
7539    append_logical_end(BB_else_logical);
7540
7541    /* branch from logical else block to endif block */
7542    aco_ptr<Pseudo_branch_instruction> branch;
7543    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7544    BB_else_logical->instructions.emplace_back(std::move(branch));
7545    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7546    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7547       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7548    BB_else_logical->kind |= block_kind_uniform;
7549
7550    assert(!ctx->cf_info.has_branch);
7551    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7552
7553
7554    /** emit linear else block */
7555    Block* BB_else_linear = ctx->program->create_and_insert_block();
7556    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7557    BB_else_linear->kind |= block_kind_uniform;
7558    add_linear_edge(ic->invert_idx, BB_else_linear);
7559
7560    /* branch from linear else block to endif block */
7561    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7562    BB_else_linear->instructions.emplace_back(std::move(branch));
7563    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7564
7565
7566    /** emit endif merge block */
7567    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7568    append_logical_start(ctx->block);
7569
7570
7571    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7572    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7573    /* uniform control flow never has an empty exec-mask */
7574    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7575       ctx->cf_info.exec_potentially_empty = false;
7576 }
7577
7578 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7579 {
7580    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7581    Builder bld(ctx->program, ctx->block);
7582    aco_ptr<Pseudo_branch_instruction> branch;
7583
7584    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7585       /**
7586        * Uniform conditionals are represented in the following way*) :
7587        *
7588        * The linear and logical CFG:
7589        *                        BB_IF
7590        *                        /    \
7591        *       BB_THEN (logical)      BB_ELSE (logical)
7592        *                        \    /
7593        *                        BB_ENDIF
7594        *
7595        * *) Exceptions may be due to break and continue statements within loops
7596        *    If a break/continue happens within uniform control flow, it branches
7597        *    to the loop exit/entry block. Otherwise, it branches to the next
7598        *    merge block.
7599        **/
7600       append_logical_end(ctx->block);
7601       ctx->block->kind |= block_kind_uniform;
7602
7603       /* emit branch */
7604       assert(cond.regClass() == bld.lm);
7605       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7606       cond = bool_to_scalar_condition(ctx, cond);
7607
7608       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7609       branch->operands[0] = Operand(cond);
7610       branch->operands[0].setFixed(scc);
7611       ctx->block->instructions.emplace_back(std::move(branch));
7612
7613       unsigned BB_if_idx = ctx->block->index;
7614       Block BB_endif = Block();
7615       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7616       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7617
7618       /** emit then block */
7619       Block* BB_then = ctx->program->create_and_insert_block();
7620       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7621       add_edge(BB_if_idx, BB_then);
7622       append_logical_start(BB_then);
7623       ctx->block = BB_then;
7624       visit_cf_list(ctx, &if_stmt->then_list);
7625       BB_then = ctx->block;
7626       bool then_branch = ctx->cf_info.has_branch;
7627       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7628
7629       if (!then_branch) {
7630          append_logical_end(BB_then);
7631          /* branch from then block to endif block */
7632          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7633          BB_then->instructions.emplace_back(std::move(branch));
7634          add_linear_edge(BB_then->index, &BB_endif);
7635          if (!then_branch_divergent)
7636             add_logical_edge(BB_then->index, &BB_endif);
7637          BB_then->kind |= block_kind_uniform;
7638       }
7639
7640       ctx->cf_info.has_branch = false;
7641       ctx->cf_info.parent_loop.has_divergent_branch = false;
7642
7643       /** emit else block */
7644       Block* BB_else = ctx->program->create_and_insert_block();
7645       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7646       add_edge(BB_if_idx, BB_else);
7647       append_logical_start(BB_else);
7648       ctx->block = BB_else;
7649       visit_cf_list(ctx, &if_stmt->else_list);
7650       BB_else = ctx->block;
7651
7652       if (!ctx->cf_info.has_branch) {
7653          append_logical_end(BB_else);
7654          /* branch from then block to endif block */
7655          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7656          BB_else->instructions.emplace_back(std::move(branch));
7657          add_linear_edge(BB_else->index, &BB_endif);
7658          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7659             add_logical_edge(BB_else->index, &BB_endif);
7660          BB_else->kind |= block_kind_uniform;
7661       }
7662
7663       ctx->cf_info.has_branch &= then_branch;
7664       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7665
7666       /** emit endif merge block */
7667       if (!ctx->cf_info.has_branch) {
7668          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7669          append_logical_start(ctx->block);
7670       }
7671    } else { /* non-uniform condition */
7672       /**
7673        * To maintain a logical and linear CFG without critical edges,
7674        * non-uniform conditionals are represented in the following way*) :
7675        *
7676        * The linear CFG:
7677        *                        BB_IF
7678        *                        /    \
7679        *       BB_THEN (logical)      BB_THEN (linear)
7680        *                        \    /
7681        *                        BB_INVERT (linear)
7682        *                        /    \
7683        *       BB_ELSE (logical)      BB_ELSE (linear)
7684        *                        \    /
7685        *                        BB_ENDIF
7686        *
7687        * The logical CFG:
7688        *                        BB_IF
7689        *                        /    \
7690        *       BB_THEN (logical)      BB_ELSE (logical)
7691        *                        \    /
7692        *                        BB_ENDIF
7693        *
7694        * *) Exceptions may be due to break and continue statements within loops
7695        **/
7696
7697       if_context ic;
7698
7699       begin_divergent_if_then(ctx, &ic, cond);
7700       visit_cf_list(ctx, &if_stmt->then_list);
7701
7702       begin_divergent_if_else(ctx, &ic);
7703       visit_cf_list(ctx, &if_stmt->else_list);
7704
7705       end_divergent_if(ctx, &ic);
7706    }
7707 }
7708
7709 static void visit_cf_list(isel_context *ctx,
7710                           struct exec_list *list)
7711 {
7712    foreach_list_typed(nir_cf_node, node, node, list) {
7713       switch (node->type) {
7714       case nir_cf_node_block:
7715          visit_block(ctx, nir_cf_node_as_block(node));
7716          break;
7717       case nir_cf_node_if:
7718          visit_if(ctx, nir_cf_node_as_if(node));
7719          break;
7720       case nir_cf_node_loop:
7721          visit_loop(ctx, nir_cf_node_as_loop(node));
7722          break;
7723       default:
7724          unreachable("unimplemented cf list type");
7725       }
7726    }
7727 }
7728
7729 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7730 {
7731    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7732    uint64_t mask = ctx->vs_output.mask[slot];
7733    if (!is_pos && !mask)
7734       return;
7735    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7736       return;
7737    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7738    exp->enabled_mask = mask;
7739    for (unsigned i = 0; i < 4; ++i) {
7740       if (mask & (1 << i))
7741          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7742       else
7743          exp->operands[i] = Operand(v1);
7744    }
7745    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
7746     * Setting valid_mask=1 prevents it and has no other effect.
7747     */
7748    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
7749    exp->done = false;
7750    exp->compressed = false;
7751    if (is_pos)
7752       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7753    else
7754       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7755    ctx->block->instructions.emplace_back(std::move(exp));
7756 }
7757
7758 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7759 {
7760    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7761    exp->enabled_mask = 0;
7762    for (unsigned i = 0; i < 4; ++i)
7763       exp->operands[i] = Operand(v1);
7764    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7765       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7766       exp->enabled_mask |= 0x1;
7767    }
7768    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7769       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7770       exp->enabled_mask |= 0x4;
7771    }
7772    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7773       if (ctx->options->chip_class < GFX9) {
7774          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7775          exp->enabled_mask |= 0x8;
7776       } else {
7777          Builder bld(ctx->program, ctx->block);
7778
7779          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7780                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7781          if (exp->operands[2].isTemp())
7782             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7783
7784          exp->operands[2] = Operand(out);
7785          exp->enabled_mask |= 0x4;
7786       }
7787    }
7788    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
7789    exp->done = false;
7790    exp->compressed = false;
7791    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7792    ctx->block->instructions.emplace_back(std::move(exp));
7793 }
7794
7795 static void create_vs_exports(isel_context *ctx)
7796 {
7797    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7798
7799    if (outinfo->export_prim_id) {
7800       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7801       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7802    }
7803
7804    if (ctx->options->key.has_multiview_view_index) {
7805       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7806       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7807    }
7808
7809    /* the order these position exports are created is important */
7810    int next_pos = 0;
7811    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7812    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7813       export_vs_psiz_layer_viewport(ctx, &next_pos);
7814    }
7815    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7816       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7817    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7818       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7819
7820    if (ctx->options->key.vs_common_out.export_clip_dists) {
7821       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7822          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7823       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7824          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7825    }
7826
7827    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7828       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7829           i != VARYING_SLOT_PRIMITIVE_ID)
7830          continue;
7831
7832       export_vs_varying(ctx, i, false, NULL);
7833    }
7834 }
7835
7836 static void emit_stream_output(isel_context *ctx,
7837                                Temp const *so_buffers,
7838                                Temp const *so_write_offset,
7839                                const struct radv_stream_output *output)
7840 {
7841    unsigned num_comps = util_bitcount(output->component_mask);
7842    unsigned loc = output->location;
7843    unsigned buf = output->buffer;
7844    unsigned offset = output->offset;
7845
7846    assert(num_comps && num_comps <= 4);
7847    if (!num_comps || num_comps > 4)
7848       return;
7849
7850    unsigned start = ffs(output->component_mask) - 1;
7851
7852    Temp out[4];
7853    bool all_undef = true;
7854    assert(ctx->stage == vertex_vs);
7855    for (unsigned i = 0; i < num_comps; i++) {
7856       out[i] = ctx->vs_output.outputs[loc][start + i];
7857       all_undef = all_undef && !out[i].id();
7858    }
7859    if (all_undef)
7860       return;
7861
7862    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7863    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7864    for (unsigned i = 0; i < num_comps; ++i)
7865       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7866    vec->definitions[0] = Definition(write_data);
7867    ctx->block->instructions.emplace_back(std::move(vec));
7868
7869    aco_opcode opcode;
7870    switch (num_comps) {
7871    case 1:
7872       opcode = aco_opcode::buffer_store_dword;
7873       break;
7874    case 2:
7875       opcode = aco_opcode::buffer_store_dwordx2;
7876       break;
7877    case 3:
7878       opcode = aco_opcode::buffer_store_dwordx3;
7879       break;
7880    case 4:
7881       opcode = aco_opcode::buffer_store_dwordx4;
7882       break;
7883    }
7884
7885    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7886    store->operands[0] = Operand(so_write_offset[buf]);
7887    store->operands[1] = Operand(so_buffers[buf]);
7888    store->operands[2] = Operand((uint32_t) 0);
7889    store->operands[3] = Operand(write_data);
7890    if (offset > 4095) {
7891       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7892       Builder bld(ctx->program, ctx->block);
7893       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7894    } else {
7895       store->offset = offset;
7896    }
7897    store->offen = true;
7898    store->glc = true;
7899    store->dlc = false;
7900    store->slc = true;
7901    store->can_reorder = true;
7902    ctx->block->instructions.emplace_back(std::move(store));
7903 }
7904
7905 static void emit_streamout(isel_context *ctx, unsigned stream)
7906 {
7907    Builder bld(ctx->program, ctx->block);
7908
7909    Temp so_buffers[4];
7910    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7911    for (unsigned i = 0; i < 4; i++) {
7912       unsigned stride = ctx->program->info->so.strides[i];
7913       if (!stride)
7914          continue;
7915
7916       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7917    }
7918
7919    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7920                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7921
7922    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7923
7924    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
7925
7926    if_context ic;
7927    begin_divergent_if_then(ctx, &ic, can_emit);
7928
7929    bld.reset(ctx->block);
7930
7931    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7932
7933    Temp so_write_offset[4];
7934
7935    for (unsigned i = 0; i < 4; i++) {
7936       unsigned stride = ctx->program->info->so.strides[i];
7937       if (!stride)
7938          continue;
7939
7940       if (stride == 1) {
7941          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7942                                 get_arg(ctx, ctx->args->streamout_write_idx),
7943                                 get_arg(ctx, ctx->args->streamout_offset[i]));
7944          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7945
7946          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7947       } else {
7948          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7949          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
7950                                  get_arg(ctx, ctx->args->streamout_offset[i]));
7951          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7952       }
7953    }
7954
7955    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7956       struct radv_stream_output *output =
7957          &ctx->program->info->so.outputs[i];
7958       if (stream != output->stream)
7959          continue;
7960
7961       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7962    }
7963
7964    begin_divergent_if_else(ctx, &ic);
7965    end_divergent_if(ctx, &ic);
7966 }
7967
7968 } /* end namespace */
7969
7970 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
7971 {
7972    /* Split all arguments except for the first (ring_offsets) and the last
7973     * (exec) so that the dead channels don't stay live throughout the program.
7974     */
7975    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
7976       if (startpgm->definitions[i].regClass().size() > 1) {
7977          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
7978                            startpgm->definitions[i].regClass().size());
7979       }
7980    }
7981 }
7982
7983 void handle_bc_optimize(isel_context *ctx)
7984 {
7985    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7986    Builder bld(ctx->program, ctx->block);
7987    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7988    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7989    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7990    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
7991    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
7992    if (uses_center && uses_centroid) {
7993       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
7994                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
7995
7996       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7997          Temp new_coord[2];
7998          for (unsigned i = 0; i < 2; i++) {
7999             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8000             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8001             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8002                                     persp_centroid, persp_center, sel);
8003          }
8004          ctx->persp_centroid = bld.tmp(v2);
8005          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8006                     Operand(new_coord[0]), Operand(new_coord[1]));
8007          emit_split_vector(ctx, ctx->persp_centroid, 2);
8008       }
8009
8010       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8011          Temp new_coord[2];
8012          for (unsigned i = 0; i < 2; i++) {
8013             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8014             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8015             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8016                                     linear_centroid, linear_center, sel);
8017          }
8018          ctx->linear_centroid = bld.tmp(v2);
8019          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8020                     Operand(new_coord[0]), Operand(new_coord[1]));
8021          emit_split_vector(ctx, ctx->linear_centroid, 2);
8022       }
8023    }
8024 }
8025
8026 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8027 {
8028    Program *program = ctx->program;
8029
8030    unsigned float_controls = shader->info.float_controls_execution_mode;
8031
8032    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8033       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8034    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8035       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8036                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8037
8038    program->next_fp_mode.must_flush_denorms32 =
8039       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8040    program->next_fp_mode.must_flush_denorms16_64 =
8041       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8042                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8043
8044    program->next_fp_mode.care_about_round32 =
8045       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8046
8047    program->next_fp_mode.care_about_round16_64 =
8048       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8049                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8050
8051    /* default to preserving fp16 and fp64 denorms, since it's free */
8052    if (program->next_fp_mode.must_flush_denorms16_64)
8053       program->next_fp_mode.denorm16_64 = 0;
8054    else
8055       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8056
8057    /* preserving fp32 denorms is expensive, so only do it if asked */
8058    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8059       program->next_fp_mode.denorm32 = fp_denorm_keep;
8060    else
8061       program->next_fp_mode.denorm32 = 0;
8062
8063    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8064       program->next_fp_mode.round32 = fp_round_tz;
8065    else
8066       program->next_fp_mode.round32 = fp_round_ne;
8067
8068    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8069       program->next_fp_mode.round16_64 = fp_round_tz;
8070    else
8071       program->next_fp_mode.round16_64 = fp_round_ne;
8072
8073    ctx->block->fp_mode = program->next_fp_mode;
8074 }
8075
8076 void select_program(Program *program,
8077                     unsigned shader_count,
8078                     struct nir_shader *const *shaders,
8079                     ac_shader_config* config,
8080                     struct radv_shader_args *args)
8081 {
8082    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
8083
8084    for (unsigned i = 0; i < shader_count; i++) {
8085       nir_shader *nir = shaders[i];
8086       init_context(&ctx, nir);
8087
8088       setup_fp_mode(&ctx, nir);
8089
8090       if (!i) {
8091          /* needs to be after init_context() for FS */
8092          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8093          append_logical_start(ctx.block);
8094          split_arguments(&ctx, startpgm);
8095       }
8096
8097       if_context ic;
8098       if (shader_count >= 2) {
8099          Builder bld(ctx.program, ctx.block);
8100          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
8101          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8102          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8103
8104          begin_divergent_if_then(&ctx, &ic, cond);
8105       }
8106
8107       if (i) {
8108          Builder bld(ctx.program, ctx.block);
8109          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
8110          bld.sopp(aco_opcode::s_barrier);
8111       }
8112
8113       if (ctx.stage == fragment_fs)
8114          handle_bc_optimize(&ctx);
8115
8116       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8117       visit_cf_list(&ctx, &func->body);
8118
8119       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8120          emit_streamout(&ctx, 0);
8121
8122       if (ctx.stage == vertex_vs)
8123          create_vs_exports(&ctx);
8124
8125       if (shader_count >= 2) {
8126          begin_divergent_if_else(&ctx, &ic);
8127          end_divergent_if(&ctx, &ic);
8128       }
8129
8130       ralloc_free(ctx.divergent_vals);
8131    }
8132
8133    program->config->float_mode = program->blocks[0].fp_mode.val;
8134
8135    append_logical_end(ctx.block);
8136    ctx.block->kind |= block_kind_uniform;
8137    Builder bld(ctx.program, ctx.block);
8138    if (ctx.program->wb_smem_l1_on_end)
8139       bld.smem(aco_opcode::s_dcache_wb, false);
8140    bld.sopp(aco_opcode::s_endpgm);
8141
8142    /* cleanup CFG */
8143    for (Block& BB : program->blocks) {
8144       for (unsigned idx : BB.linear_preds)
8145          program->blocks[idx].linear_succs.emplace_back(BB.index);
8146       for (unsigned idx : BB.logical_preds)
8147          program->blocks[idx].logical_succs.emplace_back(BB.index);
8148    }
8149 }
8150 }