src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.readlane(bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 439                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 440 {
 441    Builder bld(ctx->program, ctx->block);
 442    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 443    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 444    if (src1.type() == RegType::sgpr) {
 445       if (commutative && src0.type() == RegType::vgpr) {
 446          Temp t = src0;
 447          src0 = src1;
 448          src1 = t;
 449       } else if (src0.type() == RegType::vgpr &&
 450                  op != aco_opcode::v_madmk_f32 &&
 451                  op != aco_opcode::v_madak_f32 &&
 452                  op != aco_opcode::v_madmk_f16 &&
 453                  op != aco_opcode::v_madak_f16) {
 454          /* If the instruction is not commutative, we emit a VOP3A instruction */
 455          bld.vop2_e64(op, Definition(dst), src0, src1);
 456          return;
 457       } else {
 458          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 459       }
 460    }
 461
 462    if (flush_denorms && ctx->program->chip_class < GFX9) {
 463       assert(dst.size() == 1);
 464       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 465       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 466    } else {
 467       bld.vop2(op, Definition(dst), src0, src1);
 468    }
 469 }
 470
 471 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 472                             bool flush_denorms = false)
 473 {
 474    Temp src0 = get_alu_src(ctx, instr->src[0]);
 475    Temp src1 = get_alu_src(ctx, instr->src[1]);
 476    Temp src2 = get_alu_src(ctx, instr->src[2]);
 477
 478    /* ensure that the instruction has at most 1 sgpr operand
 479     * The optimizer will inline constants for us */
 480    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 481       src0 = as_vgpr(ctx, src0);
 482    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 483       src1 = as_vgpr(ctx, src1);
 484    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 485       src2 = as_vgpr(ctx, src2);
 486
 487    Builder bld(ctx->program, ctx->block);
 488    if (flush_denorms && ctx->program->chip_class < GFX9) {
 489       assert(dst.size() == 1);
 490       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 491       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 492    } else {
 493       bld.vop3(op, Definition(dst), src0, src1, src2);
 494    }
 495 }
 496
 497 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 498 {
 499    Builder bld(ctx->program, ctx->block);
 500    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 501 }
 502
 503 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 504 {
 505    Temp src0 = get_alu_src(ctx, instr->src[0]);
 506    Temp src1 = get_alu_src(ctx, instr->src[1]);
 507    assert(src0.size() == src1.size());
 508
 509    aco_ptr<Instruction> vopc;
 510    if (src1.type() == RegType::sgpr) {
 511       if (src0.type() == RegType::vgpr) {
 512          /* to swap the operands, we might also have to change the opcode */
 513          switch (op) {
 514             case aco_opcode::v_cmp_lt_f32:
 515                op = aco_opcode::v_cmp_gt_f32;
 516                break;
 517             case aco_opcode::v_cmp_ge_f32:
 518                op = aco_opcode::v_cmp_le_f32;
 519                break;
 520             case aco_opcode::v_cmp_lt_i32:
 521                op = aco_opcode::v_cmp_gt_i32;
 522                break;
 523             case aco_opcode::v_cmp_ge_i32:
 524                op = aco_opcode::v_cmp_le_i32;
 525                break;
 526             case aco_opcode::v_cmp_lt_u32:
 527                op = aco_opcode::v_cmp_gt_u32;
 528                break;
 529             case aco_opcode::v_cmp_ge_u32:
 530                op = aco_opcode::v_cmp_le_u32;
 531                break;
 532             case aco_opcode::v_cmp_lt_f64:
 533                op = aco_opcode::v_cmp_gt_f64;
 534                break;
 535             case aco_opcode::v_cmp_ge_f64:
 536                op = aco_opcode::v_cmp_le_f64;
 537                break;
 538             case aco_opcode::v_cmp_lt_i64:
 539                op = aco_opcode::v_cmp_gt_i64;
 540                break;
 541             case aco_opcode::v_cmp_ge_i64:
 542                op = aco_opcode::v_cmp_le_i64;
 543                break;
 544             case aco_opcode::v_cmp_lt_u64:
 545                op = aco_opcode::v_cmp_gt_u64;
 546                break;
 547             case aco_opcode::v_cmp_ge_u64:
 548                op = aco_opcode::v_cmp_le_u64;
 549                break;
 550             default: /* eq and ne are commutative */
 551                break;
 552          }
 553          Temp t = src0;
 554          src0 = src1;
 555          src1 = t;
 556       } else {
 557          src1 = as_vgpr(ctx, src1);
 558       }
 559    }
 560
 561    Builder bld(ctx->program, ctx->block);
 562    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 563 }
 564
 565 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 566 {
 567    Temp src0 = get_alu_src(ctx, instr->src[0]);
 568    Temp src1 = get_alu_src(ctx, instr->src[1]);
 569    Builder bld(ctx->program, ctx->block);
 570
 571    assert(dst.regClass() == bld.lm);
 572    assert(src0.type() == RegType::sgpr);
 573    assert(src1.type() == RegType::sgpr);
 574    assert(src0.regClass() == src1.regClass());
 575
 576    /* Emit the SALU comparison instruction */
 577    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 578    /* Turn the result into a per-lane bool */
 579    bool_to_vector_condition(ctx, cmp, dst);
 580 }
 581
 582 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 583                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 584 {
 585    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 586    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 587    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 588    bool use_valu = s_op == aco_opcode::num_opcodes ||
 589                    divergent_vals ||
 590                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 591                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 592    aco_opcode op = use_valu ? v_op : s_op;
 593    assert(op != aco_opcode::num_opcodes);
 594
 595    if (use_valu)
 596       emit_vopc_instruction(ctx, instr, op, dst);
 597    else
 598       emit_sopc_instruction(ctx, instr, op, dst);
 599 }
 600
 601 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 602 {
 603    Builder bld(ctx->program, ctx->block);
 604    Temp src0 = get_alu_src(ctx, instr->src[0]);
 605    Temp src1 = get_alu_src(ctx, instr->src[1]);
 606
 607    assert(dst.regClass() == bld.lm);
 608    assert(src0.regClass() == bld.lm);
 609    assert(src1.regClass() == bld.lm);
 610
 611    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 612 }
 613
 614 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 615 {
 616    Builder bld(ctx->program, ctx->block);
 617    Temp cond = get_alu_src(ctx, instr->src[0]);
 618    Temp then = get_alu_src(ctx, instr->src[1]);
 619    Temp els = get_alu_src(ctx, instr->src[2]);
 620
 621    assert(cond.regClass() == bld.lm);
 622
 623    if (dst.type() == RegType::vgpr) {
 624       aco_ptr<Instruction> bcsel;
 625       if (dst.size() == 1) {
 626          then = as_vgpr(ctx, then);
 627          els = as_vgpr(ctx, els);
 628
 629          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 630       } else if (dst.size() == 2) {
 631          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 632          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 633          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 634          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 635
 636          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 637          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 638
 639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 640       } else {
 641          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 642          nir_print_instr(&instr->instr, stderr);
 643          fprintf(stderr, "\n");
 644       }
 645       return;
 646    }
 647
 648    if (instr->dest.dest.ssa.bit_size == 1) {
 649       assert(dst.regClass() == bld.lm);
 650       assert(then.regClass() == bld.lm);
 651       assert(els.regClass() == bld.lm);
 652    }
 653
 654    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 655       if (dst.regClass() == s1 || dst.regClass() == s2) {
 656          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 657          assert(dst.size() == then.size());
 658          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 659          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 660       } else {
 661          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 662          nir_print_instr(&instr->instr, stderr);
 663          fprintf(stderr, "\n");
 664       }
 665       return;
 666    }
 667
 668    /* divergent boolean bcsel
 669     * this implements bcsel on bools: dst = s0 ? s1 : s2
 670     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 671    assert(instr->dest.dest.ssa.bit_size == 1);
 672
 673    if (cond.id() != then.id())
 674       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 675
 676    if (cond.id() == els.id())
 677       bld.sop1(Builder::s_mov, Definition(dst), then);
 678    else
 679       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 680                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 681 }
 682
 683 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 684                     aco_opcode op, uint32_t undo)
 685 {
 686    /* multiply by 16777216 to handle denormals */
 687    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 688                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 689    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 690    scaled = bld.vop1(op, bld.def(v1), scaled);
 691    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 692
 693    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 694
 695    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 696 }
 697
 698 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 699 {
 700    if (ctx->block->fp_mode.denorm32 == 0) {
 701       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 702       return;
 703    }
 704
 705    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 706 }
 707
 708 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 709 {
 710    if (ctx->block->fp_mode.denorm32 == 0) {
 711       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 712       return;
 713    }
 714
 715    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 716 }
 717
 718 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 719 {
 720    if (ctx->block->fp_mode.denorm32 == 0) {
 721       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 722       return;
 723    }
 724
 725    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 726 }
 727
 728 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 729 {
 730    if (ctx->block->fp_mode.denorm32 == 0) {
 731       bld.vop1(aco_opcode::v_log_f32, dst, val);
 732       return;
 733    }
 734
 735    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 736 }
 737
 738 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 739 {
 740    if (!instr->dest.dest.is_ssa) {
 741       fprintf(stderr, "nir alu dst not in ssa: ");
 742       nir_print_instr(&instr->instr, stderr);
 743       fprintf(stderr, "\n");
 744       abort();
 745    }
 746    Builder bld(ctx->program, ctx->block);
 747    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 748    switch(instr->op) {
 749    case nir_op_vec2:
 750    case nir_op_vec3:
 751    case nir_op_vec4: {
 752       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 753       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 754       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 755          elems[i] = get_alu_src(ctx, instr->src[i]);
 756          vec->operands[i] = Operand{elems[i]};
 757       }
 758       vec->definitions[0] = Definition(dst);
 759       ctx->block->instructions.emplace_back(std::move(vec));
 760       ctx->allocated_vec.emplace(dst.id(), elems);
 761       break;
 762    }
 763    case nir_op_mov: {
 764       Temp src = get_alu_src(ctx, instr->src[0]);
 765       aco_ptr<Instruction> mov;
 766       if (dst.type() == RegType::sgpr) {
 767          if (src.type() == RegType::vgpr)
 768             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 769          else if (src.regClass() == s1)
 770             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 771          else if (src.regClass() == s2)
 772             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 773          else
 774             unreachable("wrong src register class for nir_op_imov");
 775       } else if (dst.regClass() == v1) {
 776          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 777       } else if (dst.regClass() == v2) {
 778          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 779       } else {
 780          nir_print_instr(&instr->instr, stderr);
 781          unreachable("Should have been lowered to scalar.");
 782       }
 783       break;
 784    }
 785    case nir_op_inot: {
 786       Temp src = get_alu_src(ctx, instr->src[0]);
 787       if (instr->dest.dest.ssa.bit_size == 1) {
 788          assert(src.regClass() == bld.lm);
 789          assert(dst.regClass() == bld.lm);
 790          bld.sop2(Builder::s_andn2, Definition(dst), bld.def(s1, scc), Operand(exec, bld.lm), src);
 791       } else if (dst.regClass() == v1) {
 792          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 793       } else if (dst.type() == RegType::sgpr) {
 794          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 795          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 796       } else {
 797          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 798          nir_print_instr(&instr->instr, stderr);
 799          fprintf(stderr, "\n");
 800       }
 801       break;
 802    }
 803    case nir_op_ineg: {
 804       Temp src = get_alu_src(ctx, instr->src[0]);
 805       if (dst.regClass() == v1) {
 806          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 807       } else if (dst.regClass() == s1) {
 808          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 809       } else if (dst.size() == 2) {
 810          Temp src0 = bld.tmp(dst.type(), 1);
 811          Temp src1 = bld.tmp(dst.type(), 1);
 812          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 813
 814          if (dst.regClass() == s2) {
 815             Temp carry = bld.tmp(s1);
 816             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 817             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 818             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 819          } else {
 820             Temp lower = bld.tmp(v1);
 821             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 822             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 823             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 824          }
 825       } else {
 826          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 827          nir_print_instr(&instr->instr, stderr);
 828          fprintf(stderr, "\n");
 829       }
 830       break;
 831    }
 832    case nir_op_iabs: {
 833       if (dst.regClass() == s1) {
 834          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 835       } else if (dst.regClass() == v1) {
 836          Temp src = get_alu_src(ctx, instr->src[0]);
 837          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 838       } else {
 839          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 840          nir_print_instr(&instr->instr, stderr);
 841          fprintf(stderr, "\n");
 842       }
 843       break;
 844    }
 845    case nir_op_isign: {
 846       Temp src = get_alu_src(ctx, instr->src[0]);
 847       if (dst.regClass() == s1) {
 848          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 849          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 850          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 851       } else if (dst.regClass() == s2) {
 852          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 853          Temp neqz;
 854          if (ctx->program->chip_class >= GFX8)
 855             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 856          else
 857             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 858          /* SCC gets zero-extended to 64 bit */
 859          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 860       } else if (dst.regClass() == v1) {
 861          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 862          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 863          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 864       } else if (dst.regClass() == v2) {
 865          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 866          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 867          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 868          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 869          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 870          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 871       } else {
 872          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 873          nir_print_instr(&instr->instr, stderr);
 874          fprintf(stderr, "\n");
 875       }
 876       break;
 877    }
 878    case nir_op_imax: {
 879       if (dst.regClass() == v1) {
 880          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 881       } else if (dst.regClass() == s1) {
 882          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 883       } else {
 884          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 885          nir_print_instr(&instr->instr, stderr);
 886          fprintf(stderr, "\n");
 887       }
 888       break;
 889    }
 890    case nir_op_umax: {
 891       if (dst.regClass() == v1) {
 892          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 893       } else if (dst.regClass() == s1) {
 894          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 895       } else {
 896          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 897          nir_print_instr(&instr->instr, stderr);
 898          fprintf(stderr, "\n");
 899       }
 900       break;
 901    }
 902    case nir_op_imin: {
 903       if (dst.regClass() == v1) {
 904          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 905       } else if (dst.regClass() == s1) {
 906          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 907       } else {
 908          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 909          nir_print_instr(&instr->instr, stderr);
 910          fprintf(stderr, "\n");
 911       }
 912       break;
 913    }
 914    case nir_op_umin: {
 915       if (dst.regClass() == v1) {
 916          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 917       } else if (dst.regClass() == s1) {
 918          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 919       } else {
 920          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 921          nir_print_instr(&instr->instr, stderr);
 922          fprintf(stderr, "\n");
 923       }
 924       break;
 925    }
 926    case nir_op_ior: {
 927       if (instr->dest.dest.ssa.bit_size == 1) {
 928          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 929       } else if (dst.regClass() == v1) {
 930          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 931       } else if (dst.regClass() == s1) {
 932          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 933       } else if (dst.regClass() == s2) {
 934          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 935       } else {
 936          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 937          nir_print_instr(&instr->instr, stderr);
 938          fprintf(stderr, "\n");
 939       }
 940       break;
 941    }
 942    case nir_op_iand: {
 943       if (instr->dest.dest.ssa.bit_size == 1) {
 944          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 945       } else if (dst.regClass() == v1) {
 946          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 947       } else if (dst.regClass() == s1) {
 948          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 949       } else if (dst.regClass() == s2) {
 950          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 951       } else {
 952          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 953          nir_print_instr(&instr->instr, stderr);
 954          fprintf(stderr, "\n");
 955       }
 956       break;
 957    }
 958    case nir_op_ixor: {
 959       if (instr->dest.dest.ssa.bit_size == 1) {
 960          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 961       } else if (dst.regClass() == v1) {
 962          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 963       } else if (dst.regClass() == s1) {
 964          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 965       } else if (dst.regClass() == s2) {
 966          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 967       } else {
 968          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 969          nir_print_instr(&instr->instr, stderr);
 970          fprintf(stderr, "\n");
 971       }
 972       break;
 973    }
 974    case nir_op_ushr: {
 975       if (dst.regClass() == v1) {
 976          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 977       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 978          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 979                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 980       } else if (dst.regClass() == v2) {
 981          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
 982                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
 983       } else if (dst.regClass() == s2) {
 984          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 985       } else if (dst.regClass() == s1) {
 986          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 987       } else {
 988          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 989          nir_print_instr(&instr->instr, stderr);
 990          fprintf(stderr, "\n");
 991       }
 992       break;
 993    }
 994    case nir_op_ishl: {
 995       if (dst.regClass() == v1) {
 996          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 997       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 998          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 999                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1000       } else if (dst.regClass() == v2) {
1001          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1002                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1003       } else if (dst.regClass() == s1) {
1004          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1005       } else if (dst.regClass() == s2) {
1006          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1007       } else {
1008          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1009          nir_print_instr(&instr->instr, stderr);
1010          fprintf(stderr, "\n");
1011       }
1012       break;
1013    }
1014    case nir_op_ishr: {
1015       if (dst.regClass() == v1) {
1016          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1017       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1018          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1019                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1020       } else if (dst.regClass() == v2) {
1021          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1022                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1023       } else if (dst.regClass() == s1) {
1024          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1025       } else if (dst.regClass() == s2) {
1026          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1027       } else {
1028          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1029          nir_print_instr(&instr->instr, stderr);
1030          fprintf(stderr, "\n");
1031       }
1032       break;
1033    }
1034    case nir_op_find_lsb: {
1035       Temp src = get_alu_src(ctx, instr->src[0]);
1036       if (src.regClass() == s1) {
1037          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1038       } else if (src.regClass() == v1) {
1039          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1040       } else if (src.regClass() == s2) {
1041          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1042       } else {
1043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1044          nir_print_instr(&instr->instr, stderr);
1045          fprintf(stderr, "\n");
1046       }
1047       break;
1048    }
1049    case nir_op_ufind_msb:
1050    case nir_op_ifind_msb: {
1051       Temp src = get_alu_src(ctx, instr->src[0]);
1052       if (src.regClass() == s1 || src.regClass() == s2) {
1053          aco_opcode op = src.regClass() == s2 ?
1054                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1055                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1056          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1057
1058          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1059                                         Operand(src.size() * 32u - 1u), msb_rev);
1060          Temp msb = sub.def(0).getTemp();
1061          Temp carry = sub.def(1).getTemp();
1062
1063          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1064       } else if (src.regClass() == v1) {
1065          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1066          Temp msb_rev = bld.tmp(v1);
1067          emit_vop1_instruction(ctx, instr, op, msb_rev);
1068          Temp msb = bld.tmp(v1);
1069          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1070          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1071       } else {
1072          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1073          nir_print_instr(&instr->instr, stderr);
1074          fprintf(stderr, "\n");
1075       }
1076       break;
1077    }
1078    case nir_op_bitfield_reverse: {
1079       if (dst.regClass() == s1) {
1080          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1081       } else if (dst.regClass() == v1) {
1082          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1083       } else {
1084          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1085          nir_print_instr(&instr->instr, stderr);
1086          fprintf(stderr, "\n");
1087       }
1088       break;
1089    }
1090    case nir_op_iadd: {
1091       if (dst.regClass() == s1) {
1092          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1093          break;
1094       }
1095
1096       Temp src0 = get_alu_src(ctx, instr->src[0]);
1097       Temp src1 = get_alu_src(ctx, instr->src[1]);
1098       if (dst.regClass() == v1) {
1099          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1100          break;
1101       }
1102
1103       assert(src0.size() == 2 && src1.size() == 2);
1104       Temp src00 = bld.tmp(src0.type(), 1);
1105       Temp src01 = bld.tmp(dst.type(), 1);
1106       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1107       Temp src10 = bld.tmp(src1.type(), 1);
1108       Temp src11 = bld.tmp(dst.type(), 1);
1109       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1110
1111       if (dst.regClass() == s2) {
1112          Temp carry = bld.tmp(s1);
1113          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1114          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1116       } else if (dst.regClass() == v2) {
1117          Temp dst0 = bld.tmp(v1);
1118          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1119          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1120          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1121       } else {
1122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1123          nir_print_instr(&instr->instr, stderr);
1124          fprintf(stderr, "\n");
1125       }
1126       break;
1127    }
1128    case nir_op_uadd_sat: {
1129       Temp src0 = get_alu_src(ctx, instr->src[0]);
1130       Temp src1 = get_alu_src(ctx, instr->src[1]);
1131       if (dst.regClass() == s1) {
1132          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1133          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1134                   src0, src1);
1135          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1136       } else if (dst.regClass() == v1) {
1137          if (ctx->options->chip_class >= GFX9) {
1138             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1139             add->operands[0] = Operand(src0);
1140             add->operands[1] = Operand(src1);
1141             add->definitions[0] = Definition(dst);
1142             add->clamp = 1;
1143             ctx->block->instructions.emplace_back(std::move(add));
1144          } else {
1145             if (src1.regClass() != v1)
1146                std::swap(src0, src1);
1147             assert(src1.regClass() == v1);
1148             Temp tmp = bld.tmp(v1);
1149             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1150             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1151          }
1152       } else {
1153          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1154          nir_print_instr(&instr->instr, stderr);
1155          fprintf(stderr, "\n");
1156       }
1157       break;
1158    }
1159    case nir_op_uadd_carry: {
1160       Temp src0 = get_alu_src(ctx, instr->src[0]);
1161       Temp src1 = get_alu_src(ctx, instr->src[1]);
1162       if (dst.regClass() == s1) {
1163          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1164          break;
1165       }
1166       if (dst.regClass() == v1) {
1167          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1168          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1169          break;
1170       }
1171
1172       Temp src00 = bld.tmp(src0.type(), 1);
1173       Temp src01 = bld.tmp(dst.type(), 1);
1174       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1175       Temp src10 = bld.tmp(src1.type(), 1);
1176       Temp src11 = bld.tmp(dst.type(), 1);
1177       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1178       if (dst.regClass() == s2) {
1179          Temp carry = bld.tmp(s1);
1180          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1181          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1182          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1183       } else if (dst.regClass() == v2) {
1184          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1185          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1186          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1187          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1188       } else {
1189          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1190          nir_print_instr(&instr->instr, stderr);
1191          fprintf(stderr, "\n");
1192       }
1193       break;
1194    }
1195    case nir_op_isub: {
1196       if (dst.regClass() == s1) {
1197          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1198          break;
1199       }
1200
1201       Temp src0 = get_alu_src(ctx, instr->src[0]);
1202       Temp src1 = get_alu_src(ctx, instr->src[1]);
1203       if (dst.regClass() == v1) {
1204          bld.vsub32(Definition(dst), src0, src1);
1205          break;
1206       }
1207
1208       Temp src00 = bld.tmp(src0.type(), 1);
1209       Temp src01 = bld.tmp(dst.type(), 1);
1210       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1211       Temp src10 = bld.tmp(src1.type(), 1);
1212       Temp src11 = bld.tmp(dst.type(), 1);
1213       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1214       if (dst.regClass() == s2) {
1215          Temp carry = bld.tmp(s1);
1216          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1217          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1218          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1219       } else if (dst.regClass() == v2) {
1220          Temp lower = bld.tmp(v1);
1221          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1222          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1223          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1224       } else {
1225          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1226          nir_print_instr(&instr->instr, stderr);
1227          fprintf(stderr, "\n");
1228       }
1229       break;
1230    }
1231    case nir_op_usub_borrow: {
1232       Temp src0 = get_alu_src(ctx, instr->src[0]);
1233       Temp src1 = get_alu_src(ctx, instr->src[1]);
1234       if (dst.regClass() == s1) {
1235          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1236          break;
1237       } else if (dst.regClass() == v1) {
1238          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1239          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1240          break;
1241       }
1242
1243       Temp src00 = bld.tmp(src0.type(), 1);
1244       Temp src01 = bld.tmp(dst.type(), 1);
1245       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1246       Temp src10 = bld.tmp(src1.type(), 1);
1247       Temp src11 = bld.tmp(dst.type(), 1);
1248       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1249       if (dst.regClass() == s2) {
1250          Temp borrow = bld.tmp(s1);
1251          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1252          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1253          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1254       } else if (dst.regClass() == v2) {
1255          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1256          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1257          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1258          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1259       } else {
1260          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1261          nir_print_instr(&instr->instr, stderr);
1262          fprintf(stderr, "\n");
1263       }
1264       break;
1265    }
1266    case nir_op_imul: {
1267       if (dst.regClass() == v1) {
1268          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1269                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1270       } else if (dst.regClass() == s1) {
1271          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1272       } else {
1273          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1274          nir_print_instr(&instr->instr, stderr);
1275          fprintf(stderr, "\n");
1276       }
1277       break;
1278    }
1279    case nir_op_umul_high: {
1280       if (dst.regClass() == v1) {
1281          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1282       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1283          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1284       } else if (dst.regClass() == s1) {
1285          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1286                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1287          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1288       } else {
1289          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1290          nir_print_instr(&instr->instr, stderr);
1291          fprintf(stderr, "\n");
1292       }
1293       break;
1294    }
1295    case nir_op_imul_high: {
1296       if (dst.regClass() == v1) {
1297          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1298       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1299          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1300       } else if (dst.regClass() == s1) {
1301          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1302                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1303          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1304       } else {
1305          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1306          nir_print_instr(&instr->instr, stderr);
1307          fprintf(stderr, "\n");
1308       }
1309       break;
1310    }
1311    case nir_op_fmul: {
1312       if (dst.size() == 1) {
1313          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1314       } else if (dst.size() == 2) {
1315          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1316                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1317       } else {
1318          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1319          nir_print_instr(&instr->instr, stderr);
1320          fprintf(stderr, "\n");
1321       }
1322       break;
1323    }
1324    case nir_op_fadd: {
1325       if (dst.size() == 1) {
1326          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1327       } else if (dst.size() == 2) {
1328          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1329                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1330       } else {
1331          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1332          nir_print_instr(&instr->instr, stderr);
1333          fprintf(stderr, "\n");
1334       }
1335       break;
1336    }
1337    case nir_op_fsub: {
1338       Temp src0 = get_alu_src(ctx, instr->src[0]);
1339       Temp src1 = get_alu_src(ctx, instr->src[1]);
1340       if (dst.size() == 1) {
1341          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1342             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1343          else
1344             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1345       } else if (dst.size() == 2) {
1346          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1347                                      get_alu_src(ctx, instr->src[0]),
1348                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1349          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1350          sub->neg[1] = true;
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_fmax: {
1359       if (dst.size() == 1) {
1360          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1361       } else if (dst.size() == 2) {
1362          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1363             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1364                                 get_alu_src(ctx, instr->src[0]),
1365                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1366             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1367          } else {
1368             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1369                      get_alu_src(ctx, instr->src[0]),
1370                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1371          }
1372       } else {
1373          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1374          nir_print_instr(&instr->instr, stderr);
1375          fprintf(stderr, "\n");
1376       }
1377       break;
1378    }
1379    case nir_op_fmin: {
1380       if (dst.size() == 1) {
1381          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1382       } else if (dst.size() == 2) {
1383          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1384             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1385                                 get_alu_src(ctx, instr->src[0]),
1386                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1387             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1388          } else {
1389             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1390                      get_alu_src(ctx, instr->src[0]),
1391                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1392          }
1393       } else {
1394          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1395          nir_print_instr(&instr->instr, stderr);
1396          fprintf(stderr, "\n");
1397       }
1398       break;
1399    }
1400    case nir_op_fmax3: {
1401       if (dst.size() == 1) {
1402          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1403       } else {
1404          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1405          nir_print_instr(&instr->instr, stderr);
1406          fprintf(stderr, "\n");
1407       }
1408       break;
1409    }
1410    case nir_op_fmin3: {
1411       if (dst.size() == 1) {
1412          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1413       } else {
1414          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1415          nir_print_instr(&instr->instr, stderr);
1416          fprintf(stderr, "\n");
1417       }
1418       break;
1419    }
1420    case nir_op_fmed3: {
1421       if (dst.size() == 1) {
1422          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1423       } else {
1424          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1425          nir_print_instr(&instr->instr, stderr);
1426          fprintf(stderr, "\n");
1427       }
1428       break;
1429    }
1430    case nir_op_umax3: {
1431       if (dst.size() == 1) {
1432          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1433       } else {
1434          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1435          nir_print_instr(&instr->instr, stderr);
1436          fprintf(stderr, "\n");
1437       }
1438       break;
1439    }
1440    case nir_op_umin3: {
1441       if (dst.size() == 1) {
1442          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1443       } else {
1444          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1445          nir_print_instr(&instr->instr, stderr);
1446          fprintf(stderr, "\n");
1447       }
1448       break;
1449    }
1450    case nir_op_umed3: {
1451       if (dst.size() == 1) {
1452          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1453       } else {
1454          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1455          nir_print_instr(&instr->instr, stderr);
1456          fprintf(stderr, "\n");
1457       }
1458       break;
1459    }
1460    case nir_op_imax3: {
1461       if (dst.size() == 1) {
1462          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1463       } else {
1464          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1465          nir_print_instr(&instr->instr, stderr);
1466          fprintf(stderr, "\n");
1467       }
1468       break;
1469    }
1470    case nir_op_imin3: {
1471       if (dst.size() == 1) {
1472          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1473       } else {
1474          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1475          nir_print_instr(&instr->instr, stderr);
1476          fprintf(stderr, "\n");
1477       }
1478       break;
1479    }
1480    case nir_op_imed3: {
1481       if (dst.size() == 1) {
1482          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1483       } else {
1484          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1485          nir_print_instr(&instr->instr, stderr);
1486          fprintf(stderr, "\n");
1487       }
1488       break;
1489    }
1490    case nir_op_cube_face_coord: {
1491       Temp in = get_alu_src(ctx, instr->src[0], 3);
1492       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1493                       emit_extract_vector(ctx, in, 1, v1),
1494                       emit_extract_vector(ctx, in, 2, v1) };
1495       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1496       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1497       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1498       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1499       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1500       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1501       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1502       break;
1503    }
1504    case nir_op_cube_face_index: {
1505       Temp in = get_alu_src(ctx, instr->src[0], 3);
1506       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1507                       emit_extract_vector(ctx, in, 1, v1),
1508                       emit_extract_vector(ctx, in, 2, v1) };
1509       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1510       break;
1511    }
1512    case nir_op_bcsel: {
1513       emit_bcsel(ctx, instr, dst);
1514       break;
1515    }
1516    case nir_op_frsq: {
1517       if (dst.size() == 1) {
1518          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1519       } else if (dst.size() == 2) {
1520          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1521       } else {
1522          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1523          nir_print_instr(&instr->instr, stderr);
1524          fprintf(stderr, "\n");
1525       }
1526       break;
1527    }
1528    case nir_op_fneg: {
1529       Temp src = get_alu_src(ctx, instr->src[0]);
1530       if (dst.size() == 1) {
1531          if (ctx->block->fp_mode.must_flush_denorms32)
1532             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1533          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1534       } else if (dst.size() == 2) {
1535          if (ctx->block->fp_mode.must_flush_denorms16_64)
1536             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1537          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1538          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1539          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1540          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1541       } else {
1542          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1543          nir_print_instr(&instr->instr, stderr);
1544          fprintf(stderr, "\n");
1545       }
1546       break;
1547    }
1548    case nir_op_fabs: {
1549       Temp src = get_alu_src(ctx, instr->src[0]);
1550       if (dst.size() == 1) {
1551          if (ctx->block->fp_mode.must_flush_denorms32)
1552             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1553          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1554       } else if (dst.size() == 2) {
1555          if (ctx->block->fp_mode.must_flush_denorms16_64)
1556             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1557          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1558          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1559          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1560          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1561       } else {
1562          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1563          nir_print_instr(&instr->instr, stderr);
1564          fprintf(stderr, "\n");
1565       }
1566       break;
1567    }
1568    case nir_op_fsat: {
1569       Temp src = get_alu_src(ctx, instr->src[0]);
1570       if (dst.size() == 1) {
1571          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1572          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1573          // TODO: confirm that this holds under any circumstances
1574       } else if (dst.size() == 2) {
1575          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1576          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1577          vop3->clamp = true;
1578       } else {
1579          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1580          nir_print_instr(&instr->instr, stderr);
1581          fprintf(stderr, "\n");
1582       }
1583       break;
1584    }
1585    case nir_op_flog2: {
1586       if (dst.size() == 1) {
1587          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1588       } else {
1589          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1590          nir_print_instr(&instr->instr, stderr);
1591          fprintf(stderr, "\n");
1592       }
1593       break;
1594    }
1595    case nir_op_frcp: {
1596       if (dst.size() == 1) {
1597          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1598       } else if (dst.size() == 2) {
1599          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1600       } else {
1601          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1602          nir_print_instr(&instr->instr, stderr);
1603          fprintf(stderr, "\n");
1604       }
1605       break;
1606    }
1607    case nir_op_fexp2: {
1608       if (dst.size() == 1) {
1609          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1610       } else {
1611          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1612          nir_print_instr(&instr->instr, stderr);
1613          fprintf(stderr, "\n");
1614       }
1615       break;
1616    }
1617    case nir_op_fsqrt: {
1618       if (dst.size() == 1) {
1619          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1620       } else if (dst.size() == 2) {
1621          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1622       } else {
1623          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1624          nir_print_instr(&instr->instr, stderr);
1625          fprintf(stderr, "\n");
1626       }
1627       break;
1628    }
1629    case nir_op_ffract: {
1630       if (dst.size() == 1) {
1631          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1632       } else if (dst.size() == 2) {
1633          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1634       } else {
1635          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1636          nir_print_instr(&instr->instr, stderr);
1637          fprintf(stderr, "\n");
1638       }
1639       break;
1640    }
1641    case nir_op_ffloor: {
1642       if (dst.size() == 1) {
1643          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1644       } else if (dst.size() == 2) {
1645          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1646       } else {
1647          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1648          nir_print_instr(&instr->instr, stderr);
1649          fprintf(stderr, "\n");
1650       }
1651       break;
1652    }
1653    case nir_op_fceil: {
1654       if (dst.size() == 1) {
1655          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1656       } else if (dst.size() == 2) {
1657          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1658       } else {
1659          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1660          nir_print_instr(&instr->instr, stderr);
1661          fprintf(stderr, "\n");
1662       }
1663       break;
1664    }
1665    case nir_op_ftrunc: {
1666       if (dst.size() == 1) {
1667          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1668       } else if (dst.size() == 2) {
1669          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1670       } else {
1671          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1672          nir_print_instr(&instr->instr, stderr);
1673          fprintf(stderr, "\n");
1674       }
1675       break;
1676    }
1677    case nir_op_fround_even: {
1678       if (dst.size() == 1) {
1679          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1680       } else if (dst.size() == 2) {
1681          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1682       } else {
1683          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1684          nir_print_instr(&instr->instr, stderr);
1685          fprintf(stderr, "\n");
1686       }
1687       break;
1688    }
1689    case nir_op_fsin:
1690    case nir_op_fcos: {
1691       Temp src = get_alu_src(ctx, instr->src[0]);
1692       aco_ptr<Instruction> norm;
1693       if (dst.size() == 1) {
1694          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1695          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1696
1697          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1698          if (ctx->options->chip_class < GFX9)
1699             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1700
1701          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1702          bld.vop1(opcode, Definition(dst), tmp);
1703       } else {
1704          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1705          nir_print_instr(&instr->instr, stderr);
1706          fprintf(stderr, "\n");
1707       }
1708       break;
1709    }
1710    case nir_op_ldexp: {
1711       if (dst.size() == 1) {
1712          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1713                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1714                   get_alu_src(ctx, instr->src[1]));
1715       } else if (dst.size() == 2) {
1716          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1717                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1718                   get_alu_src(ctx, instr->src[1]));
1719       } else {
1720          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1721          nir_print_instr(&instr->instr, stderr);
1722          fprintf(stderr, "\n");
1723       }
1724       break;
1725    }
1726    case nir_op_frexp_sig: {
1727       if (dst.size() == 1) {
1728          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1729                   get_alu_src(ctx, instr->src[0]));
1730       } else if (dst.size() == 2) {
1731          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1732                   get_alu_src(ctx, instr->src[0]));
1733       } else {
1734          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1735          nir_print_instr(&instr->instr, stderr);
1736          fprintf(stderr, "\n");
1737       }
1738       break;
1739    }
1740    case nir_op_frexp_exp: {
1741       if (instr->src[0].src.ssa->bit_size == 32) {
1742          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1743                   get_alu_src(ctx, instr->src[0]));
1744       } else if (instr->src[0].src.ssa->bit_size == 64) {
1745          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1746                   get_alu_src(ctx, instr->src[0]));
1747       } else {
1748          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1749          nir_print_instr(&instr->instr, stderr);
1750          fprintf(stderr, "\n");
1751       }
1752       break;
1753    }
1754    case nir_op_fsign: {
1755       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1756       if (dst.size() == 1) {
1757          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1758          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1759          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1760          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1761       } else if (dst.size() == 2) {
1762          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1763          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1764          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1765
1766          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1767          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1768          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1769
1770          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1771       } else {
1772          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1773          nir_print_instr(&instr->instr, stderr);
1774          fprintf(stderr, "\n");
1775       }
1776       break;
1777    }
1778    case nir_op_f2f32: {
1779       if (instr->src[0].src.ssa->bit_size == 64) {
1780          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1781       } else {
1782          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1783          nir_print_instr(&instr->instr, stderr);
1784          fprintf(stderr, "\n");
1785       }
1786       break;
1787    }
1788    case nir_op_f2f64: {
1789       if (instr->src[0].src.ssa->bit_size == 32) {
1790          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1791       } else {
1792          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1793          nir_print_instr(&instr->instr, stderr);
1794          fprintf(stderr, "\n");
1795       }
1796       break;
1797    }
1798    case nir_op_i2f32: {
1799       assert(dst.size() == 1);
1800       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1801       break;
1802    }
1803    case nir_op_i2f64: {
1804       if (instr->src[0].src.ssa->bit_size == 32) {
1805          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1806       } else if (instr->src[0].src.ssa->bit_size == 64) {
1807          Temp src = get_alu_src(ctx, instr->src[0]);
1808          RegClass rc = RegClass(src.type(), 1);
1809          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1810          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1811          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1812          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1813          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1814          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1815
1816       } else {
1817          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1818          nir_print_instr(&instr->instr, stderr);
1819          fprintf(stderr, "\n");
1820       }
1821       break;
1822    }
1823    case nir_op_u2f32: {
1824       assert(dst.size() == 1);
1825       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1826       break;
1827    }
1828    case nir_op_u2f64: {
1829       if (instr->src[0].src.ssa->bit_size == 32) {
1830          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1831       } else if (instr->src[0].src.ssa->bit_size == 64) {
1832          Temp src = get_alu_src(ctx, instr->src[0]);
1833          RegClass rc = RegClass(src.type(), 1);
1834          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1835          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1836          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1837          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1838          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1839          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1840       } else {
1841          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1842          nir_print_instr(&instr->instr, stderr);
1843          fprintf(stderr, "\n");
1844       }
1845       break;
1846    }
1847    case nir_op_f2i32: {
1848       Temp src = get_alu_src(ctx, instr->src[0]);
1849       if (instr->src[0].src.ssa->bit_size == 32) {
1850          if (dst.type() == RegType::vgpr)
1851             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1852          else
1853             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1854                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1855
1856       } else if (instr->src[0].src.ssa->bit_size == 64) {
1857          if (dst.type() == RegType::vgpr)
1858             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1859          else
1860             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1861                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1862
1863       } else {
1864          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1865          nir_print_instr(&instr->instr, stderr);
1866          fprintf(stderr, "\n");
1867       }
1868       break;
1869    }
1870    case nir_op_f2u32: {
1871       Temp src = get_alu_src(ctx, instr->src[0]);
1872       if (instr->src[0].src.ssa->bit_size == 32) {
1873          if (dst.type() == RegType::vgpr)
1874             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1875          else
1876             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1877                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1878
1879       } else if (instr->src[0].src.ssa->bit_size == 64) {
1880          if (dst.type() == RegType::vgpr)
1881             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1882          else
1883             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1884                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1885
1886       } else {
1887          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1888          nir_print_instr(&instr->instr, stderr);
1889          fprintf(stderr, "\n");
1890       }
1891       break;
1892    }
1893    case nir_op_f2i64: {
1894       Temp src = get_alu_src(ctx, instr->src[0]);
1895       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1896          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1897          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1898          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1899          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1900          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1901          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1902          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1903          Temp new_exponent = bld.tmp(v1);
1904          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1905          if (ctx->program->chip_class >= GFX8)
1906             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1907          else
1908             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
1909          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1910          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1911          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1912          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1913          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1914          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1915          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1916          Temp new_lower = bld.tmp(v1);
1917          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1918          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1919          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1920
1921       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1922          if (src.type() == RegType::vgpr)
1923             src = bld.as_uniform(src);
1924          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1925          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1926          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1927          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1928          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1929          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1930          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1931          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1932          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1933          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1934          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1935          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1936          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1937          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1938          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1939          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1940          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1941          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1942          Temp borrow = bld.tmp(s1);
1943          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1944          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1945          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1946
1947       } else if (instr->src[0].src.ssa->bit_size == 64) {
1948          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1949          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1950          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1951          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1952          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1953          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1954          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1955          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1956          if (dst.type() == RegType::sgpr) {
1957             lower = bld.as_uniform(lower);
1958             upper = bld.as_uniform(upper);
1959          }
1960          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1961
1962       } else {
1963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1964          nir_print_instr(&instr->instr, stderr);
1965          fprintf(stderr, "\n");
1966       }
1967       break;
1968    }
1969    case nir_op_f2u64: {
1970       Temp src = get_alu_src(ctx, instr->src[0]);
1971       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1972          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1973          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1974          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1975          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1976          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1977          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1978          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1979          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1980          Temp new_exponent = bld.tmp(v1);
1981          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1982          if (ctx->program->chip_class >= GFX8)
1983             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1984          else
1985             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
1986          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1987          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1988          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1989          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1990          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1991          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1992          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1993
1994       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1995          if (src.type() == RegType::vgpr)
1996             src = bld.as_uniform(src);
1997          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1998          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1999          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2000          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2001          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2002          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2003          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2004          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2005          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2006          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2007          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2008          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2009          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2010          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2011          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2012          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2013          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2014          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2015
2016       } else if (instr->src[0].src.ssa->bit_size == 64) {
2017          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2018          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
2019          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2020          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2021          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
2022          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2023          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2024          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2025          if (dst.type() == RegType::sgpr) {
2026             lower = bld.as_uniform(lower);
2027             upper = bld.as_uniform(upper);
2028          }
2029          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2030
2031       } else {
2032          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2033          nir_print_instr(&instr->instr, stderr);
2034          fprintf(stderr, "\n");
2035       }
2036       break;
2037    }
2038    case nir_op_b2f32: {
2039       Temp src = get_alu_src(ctx, instr->src[0]);
2040       assert(src.regClass() == bld.lm);
2041
2042       if (dst.regClass() == s1) {
2043          src = bool_to_scalar_condition(ctx, src);
2044          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2045       } else if (dst.regClass() == v1) {
2046          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2047       } else {
2048          unreachable("Wrong destination register class for nir_op_b2f32.");
2049       }
2050       break;
2051    }
2052    case nir_op_b2f64: {
2053       Temp src = get_alu_src(ctx, instr->src[0]);
2054       assert(src.regClass() == bld.lm);
2055
2056       if (dst.regClass() == s2) {
2057          src = bool_to_scalar_condition(ctx, src);
2058          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2059       } else if (dst.regClass() == v2) {
2060          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2061          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2062          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2063       } else {
2064          unreachable("Wrong destination register class for nir_op_b2f64.");
2065       }
2066       break;
2067    }
2068    case nir_op_i2i32: {
2069       Temp src = get_alu_src(ctx, instr->src[0]);
2070       if (instr->src[0].src.ssa->bit_size == 64) {
2071          /* we can actually just say dst = src, as it would map the lower register */
2072          emit_extract_vector(ctx, src, 0, dst);
2073       } else {
2074          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2075          nir_print_instr(&instr->instr, stderr);
2076          fprintf(stderr, "\n");
2077       }
2078       break;
2079    }
2080    case nir_op_u2u32: {
2081       Temp src = get_alu_src(ctx, instr->src[0]);
2082       if (instr->src[0].src.ssa->bit_size == 16) {
2083          if (dst.regClass() == s1) {
2084             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2085          } else {
2086             // TODO: do better with SDWA
2087             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2088          }
2089       } else if (instr->src[0].src.ssa->bit_size == 64) {
2090          /* we can actually just say dst = src, as it would map the lower register */
2091          emit_extract_vector(ctx, src, 0, dst);
2092       } else {
2093          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2094          nir_print_instr(&instr->instr, stderr);
2095          fprintf(stderr, "\n");
2096       }
2097       break;
2098    }
2099    case nir_op_i2i64: {
2100       Temp src = get_alu_src(ctx, instr->src[0]);
2101       if (src.regClass() == s1) {
2102          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2103          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2104       } else if (src.regClass() == v1) {
2105          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2106          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2107       } else {
2108          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2109          nir_print_instr(&instr->instr, stderr);
2110          fprintf(stderr, "\n");
2111       }
2112       break;
2113    }
2114    case nir_op_u2u64: {
2115       Temp src = get_alu_src(ctx, instr->src[0]);
2116       if (instr->src[0].src.ssa->bit_size == 32) {
2117          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2118       } else {
2119          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2120          nir_print_instr(&instr->instr, stderr);
2121          fprintf(stderr, "\n");
2122       }
2123       break;
2124    }
2125    case nir_op_b2i32: {
2126       Temp src = get_alu_src(ctx, instr->src[0]);
2127       assert(src.regClass() == bld.lm);
2128
2129       if (dst.regClass() == s1) {
2130          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2131          bool_to_scalar_condition(ctx, src, dst);
2132       } else if (dst.regClass() == v1) {
2133          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2134       } else {
2135          unreachable("Invalid register class for b2i32");
2136       }
2137       break;
2138    }
2139    case nir_op_i2b1: {
2140       Temp src = get_alu_src(ctx, instr->src[0]);
2141       assert(dst.regClass() == bld.lm);
2142
2143       if (src.type() == RegType::vgpr) {
2144          assert(src.regClass() == v1 || src.regClass() == v2);
2145          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2146                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2147       } else {
2148          assert(src.regClass() == s1 || src.regClass() == s2);
2149          Temp tmp;
2150          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2151             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2152          } else {
2153             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2154                            bld.scc(bld.def(s1)), Operand(0u), src);
2155          }
2156          bool_to_vector_condition(ctx, tmp, dst);
2157       }
2158       break;
2159    }
2160    case nir_op_pack_64_2x32_split: {
2161       Temp src0 = get_alu_src(ctx, instr->src[0]);
2162       Temp src1 = get_alu_src(ctx, instr->src[1]);
2163
2164       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2165       break;
2166    }
2167    case nir_op_unpack_64_2x32_split_x:
2168       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2169       break;
2170    case nir_op_unpack_64_2x32_split_y:
2171       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2172       break;
2173    case nir_op_pack_half_2x16: {
2174       Temp src = get_alu_src(ctx, instr->src[0], 2);
2175
2176       if (dst.regClass() == v1) {
2177          Temp src0 = bld.tmp(v1);
2178          Temp src1 = bld.tmp(v1);
2179          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2180          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2181             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2182          else
2183             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2184                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2185                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2186       } else {
2187          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2188          nir_print_instr(&instr->instr, stderr);
2189          fprintf(stderr, "\n");
2190       }
2191       break;
2192    }
2193    case nir_op_unpack_half_2x16_split_x: {
2194       if (dst.regClass() == v1) {
2195          Builder bld(ctx->program, ctx->block);
2196          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2197       } else {
2198          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2199          nir_print_instr(&instr->instr, stderr);
2200          fprintf(stderr, "\n");
2201       }
2202       break;
2203    }
2204    case nir_op_unpack_half_2x16_split_y: {
2205       if (dst.regClass() == v1) {
2206          Builder bld(ctx->program, ctx->block);
2207          /* TODO: use SDWA here */
2208          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2209                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2210       } else {
2211          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2212          nir_print_instr(&instr->instr, stderr);
2213          fprintf(stderr, "\n");
2214       }
2215       break;
2216    }
2217    case nir_op_fquantize2f16: {
2218       Temp src = get_alu_src(ctx, instr->src[0]);
2219       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2220       Temp f32, cmp_res;
2221
2222       if (ctx->program->chip_class >= GFX8) {
2223          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2224          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2225          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2226       } else {
2227          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2228           * so compare the result and flush to 0 if it's smaller.
2229           */
2230          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2231          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2232          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), f32, smallest);
2233          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2234          cmp_res = vop3->definitions[0].getTemp();
2235       }
2236
2237       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2238          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2239          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2240       } else {
2241          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2242       }
2243       break;
2244    }
2245    case nir_op_bfm: {
2246       Temp bits = get_alu_src(ctx, instr->src[0]);
2247       Temp offset = get_alu_src(ctx, instr->src[1]);
2248
2249       if (dst.regClass() == s1) {
2250          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2251       } else if (dst.regClass() == v1) {
2252          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2253       } else {
2254          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2255          nir_print_instr(&instr->instr, stderr);
2256          fprintf(stderr, "\n");
2257       }
2258       break;
2259    }
2260    case nir_op_bitfield_select: {
2261       /* (mask & insert) | (~mask & base) */
2262       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2263       Temp insert = get_alu_src(ctx, instr->src[1]);
2264       Temp base = get_alu_src(ctx, instr->src[2]);
2265
2266       /* dst = (insert & bitmask) | (base & ~bitmask) */
2267       if (dst.regClass() == s1) {
2268          aco_ptr<Instruction> sop2;
2269          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2270          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2271          Operand lhs;
2272          if (const_insert && const_bitmask) {
2273             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2274          } else {
2275             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2276             lhs = Operand(insert);
2277          }
2278
2279          Operand rhs;
2280          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2281          if (const_base && const_bitmask) {
2282             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2283          } else {
2284             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2285             rhs = Operand(base);
2286          }
2287
2288          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2289
2290       } else if (dst.regClass() == v1) {
2291          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2292             base = as_vgpr(ctx, base);
2293          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2294             insert = as_vgpr(ctx, insert);
2295
2296          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2297
2298       } else {
2299          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2300          nir_print_instr(&instr->instr, stderr);
2301          fprintf(stderr, "\n");
2302       }
2303       break;
2304    }
2305    case nir_op_ubfe:
2306    case nir_op_ibfe: {
2307       Temp base = get_alu_src(ctx, instr->src[0]);
2308       Temp offset = get_alu_src(ctx, instr->src[1]);
2309       Temp bits = get_alu_src(ctx, instr->src[2]);
2310
2311       if (dst.type() == RegType::sgpr) {
2312          Operand extract;
2313          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2314          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2315          if (const_offset && const_bits) {
2316             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2317             extract = Operand(const_extract);
2318          } else {
2319             Operand width;
2320             if (const_bits) {
2321                width = Operand(const_bits->u32 << 16);
2322             } else {
2323                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2324             }
2325             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2326          }
2327
2328          aco_opcode opcode;
2329          if (dst.regClass() == s1) {
2330             if (instr->op == nir_op_ubfe)
2331                opcode = aco_opcode::s_bfe_u32;
2332             else
2333                opcode = aco_opcode::s_bfe_i32;
2334          } else if (dst.regClass() == s2) {
2335             if (instr->op == nir_op_ubfe)
2336                opcode = aco_opcode::s_bfe_u64;
2337             else
2338                opcode = aco_opcode::s_bfe_i64;
2339          } else {
2340             unreachable("Unsupported BFE bit size");
2341          }
2342
2343          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2344
2345       } else {
2346          aco_opcode opcode;
2347          if (dst.regClass() == v1) {
2348             if (instr->op == nir_op_ubfe)
2349                opcode = aco_opcode::v_bfe_u32;
2350             else
2351                opcode = aco_opcode::v_bfe_i32;
2352          } else {
2353             unreachable("Unsupported BFE bit size");
2354          }
2355
2356          emit_vop3a_instruction(ctx, instr, opcode, dst);
2357       }
2358       break;
2359    }
2360    case nir_op_bit_count: {
2361       Temp src = get_alu_src(ctx, instr->src[0]);
2362       if (src.regClass() == s1) {
2363          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2364       } else if (src.regClass() == v1) {
2365          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2366       } else if (src.regClass() == v2) {
2367          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2368                   emit_extract_vector(ctx, src, 1, v1),
2369                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2370                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2371       } else if (src.regClass() == s2) {
2372          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2373       } else {
2374          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2375          nir_print_instr(&instr->instr, stderr);
2376          fprintf(stderr, "\n");
2377       }
2378       break;
2379    }
2380    case nir_op_flt: {
2381       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2382       break;
2383    }
2384    case nir_op_fge: {
2385       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2386       break;
2387    }
2388    case nir_op_feq: {
2389       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2390       break;
2391    }
2392    case nir_op_fne: {
2393       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2394       break;
2395    }
2396    case nir_op_ilt: {
2397       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2398       break;
2399    }
2400    case nir_op_ige: {
2401       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2402       break;
2403    }
2404    case nir_op_ieq: {
2405       if (instr->src[0].src.ssa->bit_size == 1)
2406          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2407       else
2408          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2409                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2410       break;
2411    }
2412    case nir_op_ine: {
2413       if (instr->src[0].src.ssa->bit_size == 1)
2414          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2415       else
2416          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2417                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2418       break;
2419    }
2420    case nir_op_ult: {
2421       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2422       break;
2423    }
2424    case nir_op_uge: {
2425       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2426       break;
2427    }
2428    case nir_op_fddx:
2429    case nir_op_fddy:
2430    case nir_op_fddx_fine:
2431    case nir_op_fddy_fine:
2432    case nir_op_fddx_coarse:
2433    case nir_op_fddy_coarse: {
2434       Temp src = get_alu_src(ctx, instr->src[0]);
2435       uint16_t dpp_ctrl1, dpp_ctrl2;
2436       if (instr->op == nir_op_fddx_fine) {
2437          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2438          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2439       } else if (instr->op == nir_op_fddy_fine) {
2440          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2441          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2442       } else {
2443          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2444          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2445             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2446          else
2447             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2448       }
2449
2450       Temp tmp;
2451       if (ctx->program->chip_class >= GFX8) {
2452          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2453          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2454       } else {
2455          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2456          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2457          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2458       }
2459       emit_wqm(ctx, tmp, dst, true);
2460       break;
2461    }
2462    default:
2463       fprintf(stderr, "Unknown NIR ALU instr: ");
2464       nir_print_instr(&instr->instr, stderr);
2465       fprintf(stderr, "\n");
2466    }
2467 }
2468
2469 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2470 {
2471    Temp dst = get_ssa_temp(ctx, &instr->def);
2472
2473    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2474    // which get truncated the lsb if double and msb if int
2475    // for now, we only use s_mov_b64 with 64bit inline constants
2476    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2477    assert(dst.type() == RegType::sgpr);
2478
2479    Builder bld(ctx->program, ctx->block);
2480
2481    if (instr->def.bit_size == 1) {
2482       assert(dst.regClass() == bld.lm);
2483       int val = instr->value[0].b ? -1 : 0;
2484       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2485       bld.sop1(Builder::s_mov, Definition(dst), op);
2486    } else if (dst.size() == 1) {
2487       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2488    } else {
2489       assert(dst.size() != 1);
2490       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2491       if (instr->def.bit_size == 64)
2492          for (unsigned i = 0; i < dst.size(); i++)
2493             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2494       else {
2495          for (unsigned i = 0; i < dst.size(); i++)
2496             vec->operands[i] = Operand{instr->value[i].u32};
2497       }
2498       vec->definitions[0] = Definition(dst);
2499       ctx->block->instructions.emplace_back(std::move(vec));
2500    }
2501 }
2502
2503 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2504 {
2505    uint32_t new_mask = 0;
2506    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2507       if (mask & (1u << i))
2508          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2509    return new_mask;
2510 }
2511
2512 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2513 {
2514    /* This wouldn't work inside control flow or with indirect offsets but
2515     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2516
2517    unsigned write_mask = nir_intrinsic_write_mask(instr);
2518    unsigned component = nir_intrinsic_component(instr);
2519    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2520    unsigned idx = nir_intrinsic_base(instr) + component;
2521
2522    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2523    if (off_instr->type != nir_instr_type_load_const) {
2524       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2525       nir_print_instr(off_instr, stderr);
2526       fprintf(stderr, "\n");
2527    }
2528    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2529
2530    if (instr->src[0].ssa->bit_size == 64)
2531       write_mask = widen_mask(write_mask, 2);
2532
2533    for (unsigned i = 0; i < 8; ++i) {
2534       if (write_mask & (1 << i)) {
2535          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2536          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2537       }
2538       idx++;
2539    }
2540 }
2541
2542 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2543 {
2544    Builder bld(ctx->program, ctx->block);
2545    unsigned write_mask = nir_intrinsic_write_mask(instr);
2546    Operand values[4];
2547    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2548    for (unsigned i = 0; i < 4; ++i) {
2549       if (write_mask & (1 << i)) {
2550          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2551          values[i] = Operand(tmp);
2552       } else {
2553          values[i] = Operand(v1);
2554       }
2555    }
2556
2557    unsigned index = nir_intrinsic_base(instr) / 4;
2558    unsigned target, col_format;
2559    unsigned enabled_channels = 0xF;
2560    aco_opcode compr_op = (aco_opcode)0;
2561
2562    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2563    assert(offset && "Non-const offsets on exports not yet supported");
2564    index += offset->u32;
2565
2566    assert(index != FRAG_RESULT_COLOR);
2567
2568    /* Unlike vertex shader exports, it's fine to use multiple exports to
2569     * export separate channels of one target. So shaders which export both
2570     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2571     * TODO: combine the exports in those cases and create better code
2572     */
2573
2574    if (index == FRAG_RESULT_SAMPLE_MASK) {
2575
2576       if (ctx->program->info->ps.writes_z) {
2577          target = V_008DFC_SQ_EXP_MRTZ;
2578          enabled_channels = 0x4;
2579          col_format = (unsigned) -1;
2580
2581          values[2] = values[0];
2582          values[0] = Operand(v1);
2583       } else {
2584          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2585                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2586          return;
2587       }
2588
2589    } else if (index == FRAG_RESULT_DEPTH) {
2590
2591       target = V_008DFC_SQ_EXP_MRTZ;
2592       enabled_channels = 0x1;
2593       col_format = (unsigned) -1;
2594
2595    } else if (index == FRAG_RESULT_STENCIL) {
2596
2597       if (ctx->program->info->ps.writes_z) {
2598          target = V_008DFC_SQ_EXP_MRTZ;
2599          enabled_channels = 0x2;
2600          col_format = (unsigned) -1;
2601
2602          values[1] = values[0];
2603          values[0] = Operand(v1);
2604       } else {
2605          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2606          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2607                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2608          return;
2609       }
2610
2611    } else {
2612       index -= FRAG_RESULT_DATA0;
2613       target = V_008DFC_SQ_EXP_MRT + index;
2614       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2615    }
2616    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2617    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2618
2619    switch (col_format)
2620    {
2621    case V_028714_SPI_SHADER_ZERO:
2622       enabled_channels = 0; /* writemask */
2623       target = V_008DFC_SQ_EXP_NULL;
2624       break;
2625
2626    case V_028714_SPI_SHADER_32_R:
2627       enabled_channels = 1;
2628       break;
2629
2630    case V_028714_SPI_SHADER_32_GR:
2631       enabled_channels = 0x3;
2632       break;
2633
2634    case V_028714_SPI_SHADER_32_AR:
2635       if (ctx->options->chip_class >= GFX10) {
2636          /* Special case: on GFX10, the outputs are different for 32_AR */
2637          enabled_channels = 0x3;
2638          values[1] = values[3];
2639          values[3] = Operand(v1);
2640       } else {
2641          enabled_channels = 0x9;
2642       }
2643       break;
2644
2645    case V_028714_SPI_SHADER_FP16_ABGR:
2646       enabled_channels = 0x5;
2647       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2648       break;
2649
2650    case V_028714_SPI_SHADER_UNORM16_ABGR:
2651       enabled_channels = 0x5;
2652       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2653       break;
2654
2655    case V_028714_SPI_SHADER_SNORM16_ABGR:
2656       enabled_channels = 0x5;
2657       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2658       break;
2659
2660    case V_028714_SPI_SHADER_UINT16_ABGR: {
2661       enabled_channels = 0x5;
2662       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2663       if (is_int8 || is_int10) {
2664          /* clamp */
2665          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2666          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2667
2668          for (unsigned i = 0; i < 4; i++) {
2669             if ((write_mask >> i) & 1) {
2670                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2671                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2672                                     values[i]);
2673             }
2674          }
2675       }
2676       break;
2677    }
2678
2679    case V_028714_SPI_SHADER_SINT16_ABGR:
2680       enabled_channels = 0x5;
2681       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2682       if (is_int8 || is_int10) {
2683          /* clamp */
2684          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2685          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2686          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2687          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2688
2689          for (unsigned i = 0; i < 4; i++) {
2690             if ((write_mask >> i) & 1) {
2691                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2692                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2693                                     values[i]);
2694                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2695                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2696                                     values[i]);
2697             }
2698          }
2699       }
2700       break;
2701
2702    case V_028714_SPI_SHADER_32_ABGR:
2703       enabled_channels = 0xF;
2704       break;
2705
2706    default:
2707       break;
2708    }
2709
2710    if (target == V_008DFC_SQ_EXP_NULL)
2711       return;
2712
2713    if ((bool) compr_op) {
2714       for (int i = 0; i < 2; i++) {
2715          /* check if at least one of the values to be compressed is enabled */
2716          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2717          if (enabled) {
2718             enabled_channels |= enabled << (i*2);
2719             values[i] = bld.vop3(compr_op, bld.def(v1),
2720                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2721                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2722          } else {
2723             values[i] = Operand(v1);
2724          }
2725       }
2726       values[2] = Operand(v1);
2727       values[3] = Operand(v1);
2728    } else {
2729       for (int i = 0; i < 4; i++)
2730          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2731    }
2732
2733    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2734            enabled_channels, target, (bool) compr_op);
2735 }
2736
2737 Operand load_lds_size_m0(isel_context *ctx)
2738 {
2739    /* TODO: m0 does not need to be initialized on GFX9+ */
2740    Builder bld(ctx->program, ctx->block);
2741    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2742 }
2743
2744 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2745               Temp address, unsigned base_offset, unsigned align)
2746 {
2747    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2748
2749    Builder bld(ctx->program, ctx->block);
2750
2751    Operand m = load_lds_size_m0(ctx);
2752
2753    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2754    unsigned bytes_read = 0;
2755    unsigned result_size = 0;
2756    unsigned total_bytes = num_components * elem_size_bytes;
2757    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2758
2759    while (bytes_read < total_bytes) {
2760       unsigned todo = total_bytes - bytes_read;
2761       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2762       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2763
2764       aco_opcode op = aco_opcode::last_opcode;
2765       bool read2 = false;
2766       if (todo >= 16 && aligned16) {
2767          op = aco_opcode::ds_read_b128;
2768          todo = 16;
2769       } else if (todo >= 16 && aligned8) {
2770          op = aco_opcode::ds_read2_b64;
2771          read2 = true;
2772          todo = 16;
2773       } else if (todo >= 12 && aligned16) {
2774          op = aco_opcode::ds_read_b96;
2775          todo = 12;
2776       } else if (todo >= 8 && aligned8) {
2777          op = aco_opcode::ds_read_b64;
2778          todo = 8;
2779       } else if (todo >= 8) {
2780          op = aco_opcode::ds_read2_b32;
2781          read2 = true;
2782          todo = 8;
2783       } else if (todo >= 4) {
2784          op = aco_opcode::ds_read_b32;
2785          todo = 4;
2786       } else {
2787          assert(false);
2788       }
2789       assert(todo % elem_size_bytes == 0);
2790       unsigned num_elements = todo / elem_size_bytes;
2791       unsigned offset = base_offset + bytes_read;
2792       unsigned max_offset = read2 ? 1019 : 65535;
2793
2794       Temp address_offset = address;
2795       if (offset > max_offset) {
2796          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2797          offset = bytes_read;
2798       }
2799       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2800
2801       Temp res;
2802       if (num_components == 1 && dst.type() == RegType::vgpr)
2803          res = dst;
2804       else
2805          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2806
2807       if (read2)
2808          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2809       else
2810          res = bld.ds(op, Definition(res), address_offset, m, offset);
2811
2812       if (num_components == 1) {
2813          assert(todo == total_bytes);
2814          if (dst.type() == RegType::sgpr)
2815             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2816          return;
2817       }
2818
2819       if (dst.type() == RegType::sgpr)
2820          res = bld.as_uniform(res);
2821
2822       if (num_elements == 1) {
2823          result[result_size++] = res;
2824       } else {
2825          assert(res != dst && res.size() % num_elements == 0);
2826          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2827          split->operands[0] = Operand(res);
2828          for (unsigned i = 0; i < num_elements; i++)
2829             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2830          ctx->block->instructions.emplace_back(std::move(split));
2831       }
2832
2833       bytes_read += todo;
2834    }
2835
2836    assert(result_size == num_components && result_size > 1);
2837    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2838    for (unsigned i = 0; i < result_size; i++)
2839       vec->operands[i] = Operand(result[i]);
2840    vec->definitions[0] = Definition(dst);
2841    ctx->block->instructions.emplace_back(std::move(vec));
2842    ctx->allocated_vec.emplace(dst.id(), result);
2843 }
2844
2845 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2846 {
2847    if (start == 0 && size == data.size())
2848       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2849
2850    unsigned size_hint = 1;
2851    auto it = ctx->allocated_vec.find(data.id());
2852    if (it != ctx->allocated_vec.end())
2853       size_hint = it->second[0].size();
2854    if (size % size_hint || start % size_hint)
2855       size_hint = 1;
2856
2857    start /= size_hint;
2858    size /= size_hint;
2859
2860    Temp elems[size];
2861    for (unsigned i = 0; i < size; i++)
2862       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2863
2864    if (size == 1)
2865       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2866
2867    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2868    for (unsigned i = 0; i < size; i++)
2869       vec->operands[i] = Operand(elems[i]);
2870    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2871    vec->definitions[0] = Definition(res);
2872    ctx->block->instructions.emplace_back(std::move(vec));
2873    return res;
2874 }
2875
2876 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2877 {
2878    Builder bld(ctx->program, ctx->block);
2879    unsigned bytes_written = 0;
2880    while (bytes_written < total_size * 4) {
2881       unsigned todo = total_size * 4 - bytes_written;
2882       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2883       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2884
2885       aco_opcode op = aco_opcode::last_opcode;
2886       bool write2 = false;
2887       unsigned size = 0;
2888       if (todo >= 16 && aligned16) {
2889          op = aco_opcode::ds_write_b128;
2890          size = 4;
2891       } else if (todo >= 16 && aligned8) {
2892          op = aco_opcode::ds_write2_b64;
2893          write2 = true;
2894          size = 4;
2895       } else if (todo >= 12 && aligned16) {
2896          op = aco_opcode::ds_write_b96;
2897          size = 3;
2898       } else if (todo >= 8 && aligned8) {
2899          op = aco_opcode::ds_write_b64;
2900          size = 2;
2901       } else if (todo >= 8) {
2902          op = aco_opcode::ds_write2_b32;
2903          write2 = true;
2904          size = 2;
2905       } else if (todo >= 4) {
2906          op = aco_opcode::ds_write_b32;
2907          size = 1;
2908       } else {
2909          assert(false);
2910       }
2911
2912       unsigned offset = offset0 + offset1 + bytes_written;
2913       unsigned max_offset = write2 ? 1020 : 65535;
2914       Temp address_offset = address;
2915       if (offset > max_offset) {
2916          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2917          offset = offset1 + bytes_written;
2918       }
2919       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2920
2921       if (write2) {
2922          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2923          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2924          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2925       } else {
2926          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2927          bld.ds(op, address_offset, val, m, offset);
2928       }
2929
2930       bytes_written += size * 4;
2931    }
2932 }
2933
2934 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2935                Temp address, unsigned base_offset, unsigned align)
2936 {
2937    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2938
2939    Operand m = load_lds_size_m0(ctx);
2940
2941    /* we need at most two stores for 32bit variables */
2942    int start[2], count[2];
2943    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2944    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2945    assert(wrmask == 0);
2946
2947    /* one combined store is sufficient */
2948    if (count[0] == count[1]) {
2949       Builder bld(ctx->program, ctx->block);
2950
2951       Temp address_offset = address;
2952       if ((base_offset >> 2) + start[1] > 255) {
2953          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2954          base_offset = 0;
2955       }
2956
2957       assert(count[0] == 1);
2958       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2959       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2960       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2961       base_offset = base_offset / elem_size_bytes;
2962       bld.ds(op, address_offset, val0, val1, m,
2963              base_offset + start[0], base_offset + start[1]);
2964       return;
2965    }
2966
2967    for (unsigned i = 0; i < 2; i++) {
2968       if (count[i] == 0)
2969          continue;
2970
2971       unsigned elem_size_words = elem_size_bytes / 4;
2972       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2973                       base_offset, start[i] * elem_size_bytes, align);
2974    }
2975    return;
2976 }
2977
2978 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2979 {
2980    if (ctx->stage == vertex_vs) {
2981       visit_store_vs_output(ctx, instr);
2982    } else if (ctx->stage == fragment_fs) {
2983       visit_store_fs_output(ctx, instr);
2984    } else {
2985       unreachable("Shader stage not implemented");
2986    }
2987 }
2988
2989 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2990 {
2991    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2992    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2993
2994    Builder bld(ctx->program, ctx->block);
2995    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2996    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2997 }
2998
2999 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3000 {
3001    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3002    for (unsigned i = 0; i < num_components; i++)
3003       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3004    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3005       assert(num_components == 4);
3006       Builder bld(ctx->program, ctx->block);
3007       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3008    }
3009
3010    for (Operand& op : vec->operands)
3011       op = op.isUndefined() ? Operand(0u) : op;
3012
3013    vec->definitions[0] = Definition(dst);
3014    ctx->block->instructions.emplace_back(std::move(vec));
3015    emit_split_vector(ctx, dst, num_components);
3016    return;
3017 }
3018
3019 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3020 {
3021    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3022    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3023    unsigned idx = nir_intrinsic_base(instr);
3024    unsigned component = nir_intrinsic_component(instr);
3025    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3026
3027    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3028    if (offset) {
3029       assert(offset->u32 == 0);
3030    } else {
3031       /* the lower 15bit of the prim_mask contain the offset into LDS
3032        * while the upper bits contain the number of prims */
3033       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3034       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3035       Builder bld(ctx->program, ctx->block);
3036       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3037       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3038       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3039       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3040       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3041    }
3042
3043    if (instr->dest.ssa.num_components == 1) {
3044       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3045    } else {
3046       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3047       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3048       {
3049          Temp tmp = {ctx->program->allocateId(), v1};
3050          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3051          vec->operands[i] = Operand(tmp);
3052       }
3053       vec->definitions[0] = Definition(dst);
3054       ctx->block->instructions.emplace_back(std::move(vec));
3055    }
3056 }
3057
3058 unsigned get_num_channels_from_data_format(unsigned data_format)
3059 {
3060    switch (data_format) {
3061    case V_008F0C_BUF_DATA_FORMAT_8:
3062    case V_008F0C_BUF_DATA_FORMAT_16:
3063    case V_008F0C_BUF_DATA_FORMAT_32:
3064       return 1;
3065    case V_008F0C_BUF_DATA_FORMAT_8_8:
3066    case V_008F0C_BUF_DATA_FORMAT_16_16:
3067    case V_008F0C_BUF_DATA_FORMAT_32_32:
3068       return 2;
3069    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3070    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3071    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3072       return 3;
3073    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3074    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3075    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3076    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3077    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3078       return 4;
3079    default:
3080       break;
3081    }
3082
3083    return 4;
3084 }
3085
3086 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3087  * so we may need to fix it up. */
3088 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3089 {
3090    Builder bld(ctx->program, ctx->block);
3091
3092    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3093       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3094
3095    /* For the integer-like cases, do a natural sign extension.
3096     *
3097     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3098     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3099     * exponent.
3100     */
3101    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3102    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3103
3104    /* Convert back to the right type. */
3105    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3106       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3107       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3108       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3109    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3110       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3111    }
3112
3113    return alpha;
3114 }
3115
3116 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3117 {
3118    Builder bld(ctx->program, ctx->block);
3119    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3120    if (ctx->stage & sw_vs) {
3121
3122       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3123       if (off_instr->type != nir_instr_type_load_const) {
3124          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3125          nir_print_instr(off_instr, stderr);
3126          fprintf(stderr, "\n");
3127       }
3128       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3129
3130       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3131
3132       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3133       unsigned component = nir_intrinsic_component(instr);
3134       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3135       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3136       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3137       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3138
3139       unsigned dfmt = attrib_format & 0xf;
3140
3141       unsigned nfmt = (attrib_format >> 4) & 0x7;
3142       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3143       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3144       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3145       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3146       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3147       if (post_shuffle)
3148          num_channels = MAX2(num_channels, 3);
3149
3150       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3151
3152       Temp index;
3153       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3154          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3155          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3156          if (divisor) {
3157             ctx->needs_instance_id = true;
3158             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3159             if (divisor != 1) {
3160                Temp divided = bld.tmp(v1);
3161                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3162                index = bld.vadd32(bld.def(v1), start_instance, divided);
3163             } else {
3164                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3165             }
3166          } else {
3167             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3168          }
3169       } else {
3170          index = bld.vadd32(bld.def(v1),
3171                             get_arg(ctx, ctx->args->ac.base_vertex),
3172                             get_arg(ctx, ctx->args->ac.vertex_id));
3173       }
3174
3175       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3176          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3177          attrib_offset = attrib_offset % attrib_stride;
3178       }
3179
3180       Operand soffset(0u);
3181       if (attrib_offset >= 4096) {
3182          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3183          attrib_offset = 0;
3184       }
3185
3186       aco_opcode opcode;
3187       switch (num_channels) {
3188       case 1:
3189          opcode = aco_opcode::tbuffer_load_format_x;
3190          break;
3191       case 2:
3192          opcode = aco_opcode::tbuffer_load_format_xy;
3193          break;
3194       case 3:
3195          opcode = aco_opcode::tbuffer_load_format_xyz;
3196          break;
3197       case 4:
3198          opcode = aco_opcode::tbuffer_load_format_xyzw;
3199          break;
3200       default:
3201          unreachable("Unimplemented load_input vector size");
3202       }
3203
3204       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3205
3206       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3207       mubuf->operands[0] = Operand(index);
3208       mubuf->operands[1] = Operand(list);
3209       mubuf->operands[2] = soffset;
3210       mubuf->definitions[0] = Definition(tmp);
3211       mubuf->idxen = true;
3212       mubuf->can_reorder = true;
3213       mubuf->dfmt = dfmt;
3214       mubuf->nfmt = nfmt;
3215       assert(attrib_offset < 4096);
3216       mubuf->offset = attrib_offset;
3217       ctx->block->instructions.emplace_back(std::move(mubuf));
3218
3219       emit_split_vector(ctx, tmp, tmp.size());
3220
3221       if (tmp.id() != dst.id()) {
3222          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3223                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3224
3225          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3226          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3227          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3228
3229          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3230          for (unsigned i = 0; i < dst.size(); i++) {
3231             unsigned idx = i + component;
3232             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3233                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3234                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3235             } else if (idx < num_channels) {
3236                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3237             } else if (is_float && idx == 3) {
3238                vec->operands[i] = Operand(0x3f800000u);
3239             } else if (!is_float && idx == 3) {
3240                vec->operands[i] = Operand(1u);
3241             } else {
3242                vec->operands[i] = Operand(0u);
3243             }
3244          }
3245          vec->definitions[0] = Definition(dst);
3246          ctx->block->instructions.emplace_back(std::move(vec));
3247          emit_split_vector(ctx, dst, dst.size());
3248       }
3249
3250    } else if (ctx->stage == fragment_fs) {
3251       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3252       if (off_instr->type != nir_instr_type_load_const ||
3253           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3254          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3255          nir_print_instr(off_instr, stderr);
3256          fprintf(stderr, "\n");
3257       }
3258
3259       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3260       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3261       if (offset) {
3262          assert(offset->u32 == 0);
3263       } else {
3264          /* the lower 15bit of the prim_mask contain the offset into LDS
3265           * while the upper bits contain the number of prims */
3266          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3267          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3268          Builder bld(ctx->program, ctx->block);
3269          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3270          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3271          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3272          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3273          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3274       }
3275
3276       unsigned idx = nir_intrinsic_base(instr);
3277       unsigned component = nir_intrinsic_component(instr);
3278
3279       if (dst.size() == 1) {
3280          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3281       } else {
3282          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3283          for (unsigned i = 0; i < dst.size(); i++)
3284             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3285          vec->definitions[0] = Definition(dst);
3286          bld.insert(std::move(vec));
3287       }
3288
3289    } else {
3290       unreachable("Shader stage not implemented");
3291    }
3292 }
3293
3294 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3295 {
3296    if (ctx->program->info->need_indirect_descriptor_sets) {
3297       Builder bld(ctx->program, ctx->block);
3298       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3299       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3300    }
3301
3302    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3303 }
3304
3305
3306 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3307 {
3308    Builder bld(ctx->program, ctx->block);
3309    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3310    if (!ctx->divergent_vals[instr->dest.ssa.index])
3311       index = bld.as_uniform(index);
3312    unsigned desc_set = nir_intrinsic_desc_set(instr);
3313    unsigned binding = nir_intrinsic_binding(instr);
3314
3315    Temp desc_ptr;
3316    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3317    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3318    unsigned offset = layout->binding[binding].offset;
3319    unsigned stride;
3320    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3321        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3322       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3323       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3324       offset = pipeline_layout->push_constant_size + 16 * idx;
3325       stride = 16;
3326    } else {
3327       desc_ptr = load_desc_ptr(ctx, desc_set);
3328       stride = layout->binding[binding].size;
3329    }
3330
3331    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3332    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3333    if (stride != 1) {
3334       if (nir_const_index) {
3335          const_index = const_index * stride;
3336       } else if (index.type() == RegType::vgpr) {
3337          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3338          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3339       } else {
3340          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3341       }
3342    }
3343    if (offset) {
3344       if (nir_const_index) {
3345          const_index = const_index + offset;
3346       } else if (index.type() == RegType::vgpr) {
3347          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3348       } else {
3349          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3350       }
3351    }
3352
3353    if (nir_const_index && const_index == 0) {
3354       index = desc_ptr;
3355    } else if (index.type() == RegType::vgpr) {
3356       index = bld.vadd32(bld.def(v1),
3357                          nir_const_index ? Operand(const_index) : Operand(index),
3358                          Operand(desc_ptr));
3359    } else {
3360       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3361                        nir_const_index ? Operand(const_index) : Operand(index),
3362                        Operand(desc_ptr));
3363    }
3364
3365    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3366 }
3367
3368 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3369                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3370 {
3371    Builder bld(ctx->program, ctx->block);
3372
3373    unsigned num_bytes = dst.size() * 4;
3374    bool dlc = glc && ctx->options->chip_class >= GFX10;
3375
3376    aco_opcode op;
3377    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3378       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3379       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3380       unsigned const_offset = 0;
3381
3382       Temp lower = Temp();
3383       if (num_bytes > 16) {
3384          assert(num_components == 3 || num_components == 4);
3385          op = aco_opcode::buffer_load_dwordx4;
3386          lower = bld.tmp(v4);
3387          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3388          mubuf->definitions[0] = Definition(lower);
3389          mubuf->operands[0] = vaddr;
3390          mubuf->operands[1] = Operand(rsrc);
3391          mubuf->operands[2] = soffset;
3392          mubuf->offen = (offset.type() == RegType::vgpr);
3393          mubuf->glc = glc;
3394          mubuf->dlc = dlc;
3395          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3396          mubuf->can_reorder = readonly;
3397          bld.insert(std::move(mubuf));
3398          emit_split_vector(ctx, lower, 2);
3399          num_bytes -= 16;
3400          const_offset = 16;
3401       }
3402
3403       switch (num_bytes) {
3404          case 4:
3405             op = aco_opcode::buffer_load_dword;
3406             break;
3407          case 8:
3408             op = aco_opcode::buffer_load_dwordx2;
3409             break;
3410          case 12:
3411             op = aco_opcode::buffer_load_dwordx3;
3412             break;
3413          case 16:
3414             op = aco_opcode::buffer_load_dwordx4;
3415             break;
3416          default:
3417             unreachable("Load SSBO not implemented for this size.");
3418       }
3419       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3420       mubuf->operands[0] = vaddr;
3421       mubuf->operands[1] = Operand(rsrc);
3422       mubuf->operands[2] = soffset;
3423       mubuf->offen = (offset.type() == RegType::vgpr);
3424       mubuf->glc = glc;
3425       mubuf->dlc = dlc;
3426       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3427       mubuf->can_reorder = readonly;
3428       mubuf->offset = const_offset;
3429       aco_ptr<Instruction> instr = std::move(mubuf);
3430
3431       if (dst.size() > 4) {
3432          assert(lower != Temp());
3433          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3434          instr->definitions[0] = Definition(upper);
3435          bld.insert(std::move(instr));
3436          if (dst.size() == 8)
3437             emit_split_vector(ctx, upper, 2);
3438          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3439          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3440          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3441          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3442          if (dst.size() == 8)
3443             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3444       }
3445
3446       if (dst.type() == RegType::sgpr) {
3447          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3448          instr->definitions[0] = Definition(vec);
3449          bld.insert(std::move(instr));
3450          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3451       } else {
3452          instr->definitions[0] = Definition(dst);
3453          bld.insert(std::move(instr));
3454          emit_split_vector(ctx, dst, num_components);
3455       }
3456    } else {
3457       switch (num_bytes) {
3458          case 4:
3459             op = aco_opcode::s_buffer_load_dword;
3460             break;
3461          case 8:
3462             op = aco_opcode::s_buffer_load_dwordx2;
3463             break;
3464          case 12:
3465          case 16:
3466             op = aco_opcode::s_buffer_load_dwordx4;
3467             break;
3468          case 24:
3469          case 32:
3470             op = aco_opcode::s_buffer_load_dwordx8;
3471             break;
3472          default:
3473             unreachable("Load SSBO not implemented for this size.");
3474       }
3475       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3476       load->operands[0] = Operand(rsrc);
3477       load->operands[1] = Operand(bld.as_uniform(offset));
3478       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3479       load->definitions[0] = Definition(dst);
3480       load->glc = glc;
3481       load->dlc = dlc;
3482       load->barrier = readonly ? barrier_none : barrier_buffer;
3483       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3484       assert(ctx->options->chip_class >= GFX8 || !glc);
3485
3486       /* trim vector */
3487       if (dst.size() == 3) {
3488          Temp vec = bld.tmp(s4);
3489          load->definitions[0] = Definition(vec);
3490          bld.insert(std::move(load));
3491          emit_split_vector(ctx, vec, 4);
3492
3493          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3494                     emit_extract_vector(ctx, vec, 0, s1),
3495                     emit_extract_vector(ctx, vec, 1, s1),
3496                     emit_extract_vector(ctx, vec, 2, s1));
3497       } else if (dst.size() == 6) {
3498          Temp vec = bld.tmp(s8);
3499          load->definitions[0] = Definition(vec);
3500          bld.insert(std::move(load));
3501          emit_split_vector(ctx, vec, 4);
3502
3503          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3504                     emit_extract_vector(ctx, vec, 0, s2),
3505                     emit_extract_vector(ctx, vec, 1, s2),
3506                     emit_extract_vector(ctx, vec, 2, s2));
3507       } else {
3508          bld.insert(std::move(load));
3509       }
3510       emit_split_vector(ctx, dst, num_components);
3511    }
3512 }
3513
3514 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3515 {
3516    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3517    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3518
3519    Builder bld(ctx->program, ctx->block);
3520
3521    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3522    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3523    unsigned binding = nir_intrinsic_binding(idx_instr);
3524    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3525
3526    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3527       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3528                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3529                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3530                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3531       if (ctx->options->chip_class >= GFX10) {
3532          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3533                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3534                       S_008F0C_RESOURCE_LEVEL(1);
3535       } else {
3536          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3537                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3538       }
3539       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3540                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3541                                      Operand(0xFFFFFFFFu),
3542                                      Operand(desc_type));
3543       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3544                         rsrc, upper_dwords);
3545    } else {
3546       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3547       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3548    }
3549
3550    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3551 }
3552
3553 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3554 {
3555    Builder bld(ctx->program, ctx->block);
3556    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3557
3558    unsigned offset = nir_intrinsic_base(instr);
3559    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3560    if (index_cv && instr->dest.ssa.bit_size == 32) {
3561
3562       unsigned count = instr->dest.ssa.num_components;
3563       unsigned start = (offset + index_cv->u32) / 4u;
3564       start -= ctx->args->ac.base_inline_push_consts;
3565       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3566          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3567          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3568          for (unsigned i = 0; i < count; ++i) {
3569             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3570             vec->operands[i] = Operand{elems[i]};
3571          }
3572          vec->definitions[0] = Definition(dst);
3573          ctx->block->instructions.emplace_back(std::move(vec));
3574          ctx->allocated_vec.emplace(dst.id(), elems);
3575          return;
3576       }
3577    }
3578
3579    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3580    if (offset != 0) // TODO check if index != 0 as well
3581       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3582    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3583    Temp vec = dst;
3584    bool trim = false;
3585    aco_opcode op;
3586
3587    switch (dst.size()) {
3588    case 1:
3589       op = aco_opcode::s_load_dword;
3590       break;
3591    case 2:
3592       op = aco_opcode::s_load_dwordx2;
3593       break;
3594    case 3:
3595       vec = bld.tmp(s4);
3596       trim = true;
3597    case 4:
3598       op = aco_opcode::s_load_dwordx4;
3599       break;
3600    case 6:
3601       vec = bld.tmp(s8);
3602       trim = true;
3603    case 8:
3604       op = aco_opcode::s_load_dwordx8;
3605       break;
3606    default:
3607       unreachable("unimplemented or forbidden load_push_constant.");
3608    }
3609
3610    bld.smem(op, Definition(vec), ptr, index);
3611
3612    if (trim) {
3613       emit_split_vector(ctx, vec, 4);
3614       RegClass rc = dst.size() == 3 ? s1 : s2;
3615       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3616                  emit_extract_vector(ctx, vec, 0, rc),
3617                  emit_extract_vector(ctx, vec, 1, rc),
3618                  emit_extract_vector(ctx, vec, 2, rc));
3619
3620    }
3621    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3622 }
3623
3624 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3625 {
3626    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3627
3628    Builder bld(ctx->program, ctx->block);
3629
3630    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3631                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3632                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3633                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3634    if (ctx->options->chip_class >= GFX10) {
3635       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3636                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3637                    S_008F0C_RESOURCE_LEVEL(1);
3638    } else {
3639       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3640                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3641    }
3642
3643    unsigned base = nir_intrinsic_base(instr);
3644    unsigned range = nir_intrinsic_range(instr);
3645
3646    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3647    if (base && offset.type() == RegType::sgpr)
3648       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3649    else if (base && offset.type() == RegType::vgpr)
3650       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3651
3652    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3653                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3654                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3655                           Operand(desc_type));
3656
3657    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3658 }
3659
3660 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3661 {
3662    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3663       ctx->cf_info.exec_potentially_empty = true;
3664
3665    ctx->program->needs_exact = true;
3666
3667    // TODO: optimize uniform conditions
3668    Builder bld(ctx->program, ctx->block);
3669    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3670    assert(src.regClass() == bld.lm);
3671    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3672    bld.pseudo(aco_opcode::p_discard_if, src);
3673    ctx->block->kind |= block_kind_uses_discard_if;
3674    return;
3675 }
3676
3677 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3678 {
3679    Builder bld(ctx->program, ctx->block);
3680
3681    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3682       ctx->cf_info.exec_potentially_empty = true;
3683
3684    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3685                     ctx->cf_info.parent_loop.has_divergent_continue;
3686
3687    if (ctx->block->loop_nest_depth &&
3688        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3689       /* we handle discards the same way as jump instructions */
3690       append_logical_end(ctx->block);
3691
3692       /* in loops, discard behaves like break */
3693       Block *linear_target = ctx->cf_info.parent_loop.exit;
3694       ctx->block->kind |= block_kind_discard;
3695
3696       if (!divergent) {
3697          /* uniform discard - loop ends here */
3698          assert(nir_instr_is_last(&instr->instr));
3699          ctx->block->kind |= block_kind_uniform;
3700          ctx->cf_info.has_branch = true;
3701          bld.branch(aco_opcode::p_branch);
3702          add_linear_edge(ctx->block->index, linear_target);
3703          return;
3704       }
3705
3706       /* we add a break right behind the discard() instructions */
3707       ctx->block->kind |= block_kind_break;
3708       unsigned idx = ctx->block->index;
3709
3710       /* remove critical edges from linear CFG */
3711       bld.branch(aco_opcode::p_branch);
3712       Block* break_block = ctx->program->create_and_insert_block();
3713       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3714       break_block->kind |= block_kind_uniform;
3715       add_linear_edge(idx, break_block);
3716       add_linear_edge(break_block->index, linear_target);
3717       bld.reset(break_block);
3718       bld.branch(aco_opcode::p_branch);
3719
3720       Block* continue_block = ctx->program->create_and_insert_block();
3721       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3722       add_linear_edge(idx, continue_block);
3723       append_logical_start(continue_block);
3724       ctx->block = continue_block;
3725
3726       return;
3727    }
3728
3729    /* it can currently happen that NIR doesn't remove the unreachable code */
3730    if (!nir_instr_is_last(&instr->instr)) {
3731       ctx->program->needs_exact = true;
3732       /* save exec somewhere temporarily so that it doesn't get
3733        * overwritten before the discard from outer exec masks */
3734       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3735       bld.pseudo(aco_opcode::p_discard_if, cond);
3736       ctx->block->kind |= block_kind_uses_discard_if;
3737       return;
3738    }
3739
3740    /* This condition is incorrect for uniformly branched discards in a loop
3741     * predicated by a divergent condition, but the above code catches that case
3742     * and the discard would end up turning into a discard_if.
3743     * For example:
3744     * if (divergent) {
3745     *    while (...) {
3746     *       if (uniform) {
3747     *          discard;
3748     *       }
3749     *    }
3750     * }
3751     */
3752    if (!ctx->cf_info.parent_if.is_divergent) {
3753       /* program just ends here */
3754       ctx->block->kind |= block_kind_uniform;
3755       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3756               0 /* enabled mask */, 9 /* dest */,
3757               false /* compressed */, true/* done */, true /* valid mask */);
3758       bld.sopp(aco_opcode::s_endpgm);
3759       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3760    } else {
3761       ctx->block->kind |= block_kind_discard;
3762       /* branch and linear edge is added by visit_if() */
3763    }
3764 }
3765
3766 enum aco_descriptor_type {
3767    ACO_DESC_IMAGE,
3768    ACO_DESC_FMASK,
3769    ACO_DESC_SAMPLER,
3770    ACO_DESC_BUFFER,
3771    ACO_DESC_PLANE_0,
3772    ACO_DESC_PLANE_1,
3773    ACO_DESC_PLANE_2,
3774 };
3775
3776 static bool
3777 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3778    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3779       return false;
3780    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3781    return dim == ac_image_cube ||
3782           dim == ac_image_1darray ||
3783           dim == ac_image_2darray ||
3784           dim == ac_image_2darraymsaa;
3785 }
3786
3787 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3788                       enum aco_descriptor_type desc_type,
3789                       const nir_tex_instr *tex_instr, bool image, bool write)
3790 {
3791 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3792    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3793    if (it != ctx->tex_desc.end())
3794       return it->second;
3795 */
3796    Temp index = Temp();
3797    bool index_set = false;
3798    unsigned constant_index = 0;
3799    unsigned descriptor_set;
3800    unsigned base_index;
3801    Builder bld(ctx->program, ctx->block);
3802
3803    if (!deref_instr) {
3804       assert(tex_instr && !image);
3805       descriptor_set = 0;
3806       base_index = tex_instr->sampler_index;
3807    } else {
3808       while(deref_instr->deref_type != nir_deref_type_var) {
3809          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3810          if (!array_size)
3811             array_size = 1;
3812
3813          assert(deref_instr->deref_type == nir_deref_type_array);
3814          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3815          if (const_value) {
3816             constant_index += array_size * const_value->u32;
3817          } else {
3818             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3819             if (indirect.type() == RegType::vgpr)
3820                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3821
3822             if (array_size != 1)
3823                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3824
3825             if (!index_set) {
3826                index = indirect;
3827                index_set = true;
3828             } else {
3829                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3830             }
3831          }
3832
3833          deref_instr = nir_src_as_deref(deref_instr->parent);
3834       }
3835       descriptor_set = deref_instr->var->data.descriptor_set;
3836       base_index = deref_instr->var->data.binding;
3837    }
3838
3839    Temp list = load_desc_ptr(ctx, descriptor_set);
3840    list = convert_pointer_to_64_bit(ctx, list);
3841
3842    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3843    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3844    unsigned offset = binding->offset;
3845    unsigned stride = binding->size;
3846    aco_opcode opcode;
3847    RegClass type;
3848
3849    assert(base_index < layout->binding_count);
3850
3851    switch (desc_type) {
3852    case ACO_DESC_IMAGE:
3853       type = s8;
3854       opcode = aco_opcode::s_load_dwordx8;
3855       break;
3856    case ACO_DESC_FMASK:
3857       type = s8;
3858       opcode = aco_opcode::s_load_dwordx8;
3859       offset += 32;
3860       break;
3861    case ACO_DESC_SAMPLER:
3862       type = s4;
3863       opcode = aco_opcode::s_load_dwordx4;
3864       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3865          offset += radv_combined_image_descriptor_sampler_offset(binding);
3866       break;
3867    case ACO_DESC_BUFFER:
3868       type = s4;
3869       opcode = aco_opcode::s_load_dwordx4;
3870       break;
3871    case ACO_DESC_PLANE_0:
3872    case ACO_DESC_PLANE_1:
3873       type = s8;
3874       opcode = aco_opcode::s_load_dwordx8;
3875       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3876       break;
3877    case ACO_DESC_PLANE_2:
3878       type = s4;
3879       opcode = aco_opcode::s_load_dwordx4;
3880       offset += 64;
3881       break;
3882    default:
3883       unreachable("invalid desc_type\n");
3884    }
3885
3886    offset += constant_index * stride;
3887
3888    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3889       (!index_set || binding->immutable_samplers_equal)) {
3890       if (binding->immutable_samplers_equal)
3891          constant_index = 0;
3892
3893       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3894       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3895                         Operand(samplers[constant_index * 4 + 0]),
3896                         Operand(samplers[constant_index * 4 + 1]),
3897                         Operand(samplers[constant_index * 4 + 2]),
3898                         Operand(samplers[constant_index * 4 + 3]));
3899    }
3900
3901    Operand off;
3902    if (!index_set) {
3903       off = Operand(offset);
3904    } else {
3905       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3906                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3907    }
3908
3909    Temp res = bld.smem(opcode, bld.def(type), list, off);
3910
3911    if (desc_type == ACO_DESC_PLANE_2) {
3912       Temp components[8];
3913       for (unsigned i = 0; i < 8; i++)
3914          components[i] = bld.tmp(s1);
3915       bld.pseudo(aco_opcode::p_split_vector,
3916                  Definition(components[0]),
3917                  Definition(components[1]),
3918                  Definition(components[2]),
3919                  Definition(components[3]),
3920                  res);
3921
3922       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3923       bld.pseudo(aco_opcode::p_split_vector,
3924                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3925                  Definition(components[4]),
3926                  Definition(components[5]),
3927                  Definition(components[6]),
3928                  Definition(components[7]),
3929                  desc2);
3930
3931       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3932                        components[0], components[1], components[2], components[3],
3933                        components[4], components[5], components[6], components[7]);
3934    }
3935
3936    return res;
3937 }
3938
3939 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3940 {
3941    switch (dim) {
3942    case GLSL_SAMPLER_DIM_BUF:
3943       return 1;
3944    case GLSL_SAMPLER_DIM_1D:
3945       return array ? 2 : 1;
3946    case GLSL_SAMPLER_DIM_2D:
3947       return array ? 3 : 2;
3948    case GLSL_SAMPLER_DIM_MS:
3949       return array ? 4 : 3;
3950    case GLSL_SAMPLER_DIM_3D:
3951    case GLSL_SAMPLER_DIM_CUBE:
3952       return 3;
3953    case GLSL_SAMPLER_DIM_RECT:
3954    case GLSL_SAMPLER_DIM_SUBPASS:
3955       return 2;
3956    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3957       return 3;
3958    default:
3959       break;
3960    }
3961    return 0;
3962 }
3963
3964
3965 /* Adjust the sample index according to FMASK.
3966  *
3967  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3968  * which is the identity mapping. Each nibble says which physical sample
3969  * should be fetched to get that sample.
3970  *
3971  * For example, 0x11111100 means there are only 2 samples stored and
3972  * the second sample covers 3/4 of the pixel. When reading samples 0
3973  * and 1, return physical sample 0 (determined by the first two 0s
3974  * in FMASK), otherwise return physical sample 1.
3975  *
3976  * The sample index should be adjusted as follows:
3977  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3978  */
3979 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3980 {
3981    Builder bld(ctx->program, ctx->block);
3982    Temp fmask = bld.tmp(v1);
3983    unsigned dim = ctx->options->chip_class >= GFX10
3984                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3985                   : 0;
3986
3987    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3988    load->operands[0] = Operand(coords);
3989    load->operands[1] = Operand(fmask_desc_ptr);
3990    load->definitions[0] = Definition(fmask);
3991    load->glc = false;
3992    load->dlc = false;
3993    load->dmask = 0x1;
3994    load->unrm = true;
3995    load->da = da;
3996    load->dim = dim;
3997    load->can_reorder = true; /* fmask images shouldn't be modified */
3998    ctx->block->instructions.emplace_back(std::move(load));
3999
4000    Operand sample_index4;
4001    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4002       sample_index4 = Operand(sample_index.constantValue() << 2);
4003    } else if (sample_index.regClass() == s1) {
4004       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4005    } else {
4006       assert(sample_index.regClass() == v1);
4007       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4008    }
4009
4010    Temp final_sample;
4011    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4012       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4013    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4014       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4015    else
4016       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4017
4018    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4019     * resource descriptor is 0 (invalid),
4020     */
4021    Temp compare = bld.tmp(bld.lm);
4022    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4023                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4024
4025    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4026
4027    /* Replace the MSAA sample index. */
4028    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4029 }
4030
4031 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4032 {
4033
4034    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4035    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4036    bool is_array = glsl_sampler_type_is_array(type);
4037    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4038    assert(!add_frag_pos && "Input attachments should be lowered.");
4039    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4040    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4041    int count = image_type_to_components_count(dim, is_array);
4042    std::vector<Operand> coords(count);
4043
4044    if (is_ms) {
4045       Operand sample_index;
4046       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4047       if (sample_cv)
4048          sample_index = Operand(sample_cv->u32);
4049       else
4050          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4051
4052       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4053          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4054          for (unsigned i = 0; i < vec->operands.size(); i++)
4055             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4056          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4057          vec->definitions[0] = Definition(fmask_load_address);
4058          ctx->block->instructions.emplace_back(std::move(vec));
4059
4060          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4061          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4062       }
4063       count--;
4064       coords[count] = sample_index;
4065    }
4066
4067    if (count == 1 && !gfx9_1d)
4068       return emit_extract_vector(ctx, src0, 0, v1);
4069
4070    if (gfx9_1d) {
4071       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4072       coords.resize(coords.size() + 1);
4073       coords[1] = Operand((uint32_t) 0);
4074       if (is_array)
4075          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4076    } else {
4077       for (int i = 0; i < count; i++)
4078          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4079    }
4080
4081    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4082        instr->intrinsic == nir_intrinsic_image_deref_store) {
4083       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4084       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4085
4086       if (!level_zero)
4087          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4088    }
4089
4090    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4091    for (unsigned i = 0; i < coords.size(); i++)
4092       vec->operands[i] = coords[i];
4093    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4094    vec->definitions[0] = Definition(res);
4095    ctx->block->instructions.emplace_back(std::move(vec));
4096    return res;
4097 }
4098
4099
4100 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4101 {
4102    Builder bld(ctx->program, ctx->block);
4103    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4104    const struct glsl_type *type = glsl_without_array(var->type);
4105    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4106    bool is_array = glsl_sampler_type_is_array(type);
4107    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4108
4109    if (dim == GLSL_SAMPLER_DIM_BUF) {
4110       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4111       unsigned num_channels = util_last_bit(mask);
4112       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4113       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4114
4115       aco_opcode opcode;
4116       switch (num_channels) {
4117       case 1:
4118          opcode = aco_opcode::buffer_load_format_x;
4119          break;
4120       case 2:
4121          opcode = aco_opcode::buffer_load_format_xy;
4122          break;
4123       case 3:
4124          opcode = aco_opcode::buffer_load_format_xyz;
4125          break;
4126       case 4:
4127          opcode = aco_opcode::buffer_load_format_xyzw;
4128          break;
4129       default:
4130          unreachable(">4 channel buffer image load");
4131       }
4132       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4133       load->operands[0] = Operand(vindex);
4134       load->operands[1] = Operand(rsrc);
4135       load->operands[2] = Operand((uint32_t) 0);
4136       Temp tmp;
4137       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4138          tmp = dst;
4139       else
4140          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4141       load->definitions[0] = Definition(tmp);
4142       load->idxen = true;
4143       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4144       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4145       load->barrier = barrier_image;
4146       ctx->block->instructions.emplace_back(std::move(load));
4147
4148       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4149       return;
4150    }
4151
4152    Temp coords = get_image_coords(ctx, instr, type);
4153    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4154
4155    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4156    unsigned num_components = util_bitcount(dmask);
4157    Temp tmp;
4158    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4159       tmp = dst;
4160    else
4161       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4162
4163    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4164    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4165
4166    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4167    load->operands[0] = Operand(coords);
4168    load->operands[1] = Operand(resource);
4169    load->definitions[0] = Definition(tmp);
4170    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4171    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4172    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4173    load->dmask = dmask;
4174    load->unrm = true;
4175    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4176    load->barrier = barrier_image;
4177    ctx->block->instructions.emplace_back(std::move(load));
4178
4179    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4180    return;
4181 }
4182
4183 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4184 {
4185    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4186    const struct glsl_type *type = glsl_without_array(var->type);
4187    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4188    bool is_array = glsl_sampler_type_is_array(type);
4189    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4190
4191    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4192
4193    if (dim == GLSL_SAMPLER_DIM_BUF) {
4194       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4195       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4196       aco_opcode opcode;
4197       switch (data.size()) {
4198       case 1:
4199          opcode = aco_opcode::buffer_store_format_x;
4200          break;
4201       case 2:
4202          opcode = aco_opcode::buffer_store_format_xy;
4203          break;
4204       case 3:
4205          opcode = aco_opcode::buffer_store_format_xyz;
4206          break;
4207       case 4:
4208          opcode = aco_opcode::buffer_store_format_xyzw;
4209          break;
4210       default:
4211          unreachable(">4 channel buffer image store");
4212       }
4213       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4214       store->operands[0] = Operand(vindex);
4215       store->operands[1] = Operand(rsrc);
4216       store->operands[2] = Operand((uint32_t) 0);
4217       store->operands[3] = Operand(data);
4218       store->idxen = true;
4219       store->glc = glc;
4220       store->dlc = false;
4221       store->disable_wqm = true;
4222       store->barrier = barrier_image;
4223       ctx->program->needs_exact = true;
4224       ctx->block->instructions.emplace_back(std::move(store));
4225       return;
4226    }
4227
4228    assert(data.type() == RegType::vgpr);
4229    Temp coords = get_image_coords(ctx, instr, type);
4230    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4231
4232    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4233    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4234
4235    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4236    store->operands[0] = Operand(coords);
4237    store->operands[1] = Operand(resource);
4238    store->operands[2] = Operand(s4);
4239    store->operands[3] = Operand(data);
4240    store->glc = glc;
4241    store->dlc = false;
4242    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4243    store->dmask = (1 << data.size()) - 1;
4244    store->unrm = true;
4245    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4246    store->disable_wqm = true;
4247    store->barrier = barrier_image;
4248    ctx->program->needs_exact = true;
4249    ctx->block->instructions.emplace_back(std::move(store));
4250    return;
4251 }
4252
4253 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4254 {
4255    /* return the previous value if dest is ever used */
4256    bool return_previous = false;
4257    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4258       return_previous = true;
4259       break;
4260    }
4261    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4262       return_previous = true;
4263       break;
4264    }
4265
4266    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4267    const struct glsl_type *type = glsl_without_array(var->type);
4268    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4269    bool is_array = glsl_sampler_type_is_array(type);
4270    Builder bld(ctx->program, ctx->block);
4271
4272    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4273    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4274
4275    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4276       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4277
4278    aco_opcode buf_op, image_op;
4279    switch (instr->intrinsic) {
4280       case nir_intrinsic_image_deref_atomic_add:
4281          buf_op = aco_opcode::buffer_atomic_add;
4282          image_op = aco_opcode::image_atomic_add;
4283          break;
4284       case nir_intrinsic_image_deref_atomic_umin:
4285          buf_op = aco_opcode::buffer_atomic_umin;
4286          image_op = aco_opcode::image_atomic_umin;
4287          break;
4288       case nir_intrinsic_image_deref_atomic_imin:
4289          buf_op = aco_opcode::buffer_atomic_smin;
4290          image_op = aco_opcode::image_atomic_smin;
4291          break;
4292       case nir_intrinsic_image_deref_atomic_umax:
4293          buf_op = aco_opcode::buffer_atomic_umax;
4294          image_op = aco_opcode::image_atomic_umax;
4295          break;
4296       case nir_intrinsic_image_deref_atomic_imax:
4297          buf_op = aco_opcode::buffer_atomic_smax;
4298          image_op = aco_opcode::image_atomic_smax;
4299          break;
4300       case nir_intrinsic_image_deref_atomic_and:
4301          buf_op = aco_opcode::buffer_atomic_and;
4302          image_op = aco_opcode::image_atomic_and;
4303          break;
4304       case nir_intrinsic_image_deref_atomic_or:
4305          buf_op = aco_opcode::buffer_atomic_or;
4306          image_op = aco_opcode::image_atomic_or;
4307          break;
4308       case nir_intrinsic_image_deref_atomic_xor:
4309          buf_op = aco_opcode::buffer_atomic_xor;
4310          image_op = aco_opcode::image_atomic_xor;
4311          break;
4312       case nir_intrinsic_image_deref_atomic_exchange:
4313          buf_op = aco_opcode::buffer_atomic_swap;
4314          image_op = aco_opcode::image_atomic_swap;
4315          break;
4316       case nir_intrinsic_image_deref_atomic_comp_swap:
4317          buf_op = aco_opcode::buffer_atomic_cmpswap;
4318          image_op = aco_opcode::image_atomic_cmpswap;
4319          break;
4320       default:
4321          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4322    }
4323
4324    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4325
4326    if (dim == GLSL_SAMPLER_DIM_BUF) {
4327       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4328       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4329       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4330       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4331       mubuf->operands[0] = Operand(vindex);
4332       mubuf->operands[1] = Operand(resource);
4333       mubuf->operands[2] = Operand((uint32_t)0);
4334       mubuf->operands[3] = Operand(data);
4335       if (return_previous)
4336          mubuf->definitions[0] = Definition(dst);
4337       mubuf->offset = 0;
4338       mubuf->idxen = true;
4339       mubuf->glc = return_previous;
4340       mubuf->dlc = false; /* Not needed for atomics */
4341       mubuf->disable_wqm = true;
4342       mubuf->barrier = barrier_image;
4343       ctx->program->needs_exact = true;
4344       ctx->block->instructions.emplace_back(std::move(mubuf));
4345       return;
4346    }
4347
4348    Temp coords = get_image_coords(ctx, instr, type);
4349    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4350    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4351    mimg->operands[0] = Operand(coords);
4352    mimg->operands[1] = Operand(resource);
4353    mimg->operands[2] = Operand(s4); /* no sampler */
4354    mimg->operands[3] = Operand(data);
4355    if (return_previous)
4356       mimg->definitions[0] = Definition(dst);
4357    mimg->glc = return_previous;
4358    mimg->dlc = false; /* Not needed for atomics */
4359    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4360    mimg->dmask = (1 << data.size()) - 1;
4361    mimg->unrm = true;
4362    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4363    mimg->disable_wqm = true;
4364    mimg->barrier = barrier_image;
4365    ctx->program->needs_exact = true;
4366    ctx->block->instructions.emplace_back(std::move(mimg));
4367    return;
4368 }
4369
4370 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4371 {
4372    if (in_elements && ctx->options->chip_class == GFX8) {
4373       Builder bld(ctx->program, ctx->block);
4374
4375       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4376       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4377       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4378       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4379
4380       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4381       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4382
4383       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4384       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4385       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4386
4387       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4388       /* idea
4389        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4390        * in case 12 (or 3?), we have to divide by 3:
4391        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4392        * use v_mul_hi_u32 with magic number to divide
4393        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4394        * disable v_skip
4395        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4396        */
4397
4398    } else {
4399       emit_extract_vector(ctx, desc, 2, dst);
4400    }
4401 }
4402
4403 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4404 {
4405    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4406    const struct glsl_type *type = glsl_without_array(var->type);
4407    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4408    bool is_array = glsl_sampler_type_is_array(type);
4409    Builder bld(ctx->program, ctx->block);
4410
4411    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4412       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4413       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4414    }
4415
4416    /* LOD */
4417    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4418
4419    /* Resource */
4420    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4421
4422    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4423
4424    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4425    mimg->operands[0] = Operand(lod);
4426    mimg->operands[1] = Operand(resource);
4427    uint8_t& dmask = mimg->dmask;
4428    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4429    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4430    mimg->da = glsl_sampler_type_is_array(type);
4431    mimg->can_reorder = true;
4432    Definition& def = mimg->definitions[0];
4433    ctx->block->instructions.emplace_back(std::move(mimg));
4434
4435    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4436        glsl_sampler_type_is_array(type)) {
4437
4438       assert(instr->dest.ssa.num_components == 3);
4439       Temp tmp = {ctx->program->allocateId(), v3};
4440       def = Definition(tmp);
4441       emit_split_vector(ctx, tmp, 3);
4442
4443       /* divide 3rd value by 6 by multiplying with magic number */
4444       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4445       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4446
4447       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4448                  emit_extract_vector(ctx, tmp, 0, v1),
4449                  emit_extract_vector(ctx, tmp, 1, v1),
4450                  by_6);
4451
4452    } else if (ctx->options->chip_class == GFX9 &&
4453               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4454               glsl_sampler_type_is_array(type)) {
4455       assert(instr->dest.ssa.num_components == 2);
4456       def = Definition(dst);
4457       dmask = 0x5;
4458    } else {
4459       def = Definition(dst);
4460    }
4461
4462    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4463 }
4464
4465 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4466 {
4467    Builder bld(ctx->program, ctx->block);
4468    unsigned num_components = instr->num_components;
4469
4470    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4471    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4472    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4473
4474    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4475    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4476 }
4477
4478 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4479 {
4480    Builder bld(ctx->program, ctx->block);
4481    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4482    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4483    unsigned writemask = nir_intrinsic_write_mask(instr);
4484    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4485
4486    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4487    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4488
4489    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4490                ctx->options->chip_class >= GFX8;
4491    if (smem)
4492       offset = bld.as_uniform(offset);
4493    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4494
4495    while (writemask) {
4496       int start, count;
4497       u_bit_scan_consecutive_range(&writemask, &start, &count);
4498       if (count == 3 && smem) {
4499          writemask |= 1u << (start + 2);
4500          count = 2;
4501       }
4502       int num_bytes = count * elem_size_bytes;
4503
4504       if (num_bytes > 16) {
4505          assert(elem_size_bytes == 8);
4506          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4507          count = 2;
4508          num_bytes = 16;
4509       }
4510
4511       // TODO: check alignment of sub-dword stores
4512       // TODO: split 3 bytes. there is no store instruction for that
4513
4514       Temp write_data;
4515       if (count != instr->num_components) {
4516          emit_split_vector(ctx, data, instr->num_components);
4517          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4518          for (int i = 0; i < count; i++) {
4519             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4520             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4521          }
4522          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4523          vec->definitions[0] = Definition(write_data);
4524          ctx->block->instructions.emplace_back(std::move(vec));
4525       } else if (!smem && data.type() != RegType::vgpr) {
4526          assert(num_bytes % 4 == 0);
4527          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4528       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4529          assert(num_bytes % 4 == 0);
4530          write_data = bld.as_uniform(data);
4531       } else {
4532          write_data = data;
4533       }
4534
4535       aco_opcode vmem_op, smem_op;
4536       switch (num_bytes) {
4537          case 4:
4538             vmem_op = aco_opcode::buffer_store_dword;
4539             smem_op = aco_opcode::s_buffer_store_dword;
4540             break;
4541          case 8:
4542             vmem_op = aco_opcode::buffer_store_dwordx2;
4543             smem_op = aco_opcode::s_buffer_store_dwordx2;
4544             break;
4545          case 12:
4546             vmem_op = aco_opcode::buffer_store_dwordx3;
4547             smem_op = aco_opcode::last_opcode;
4548             assert(!smem);
4549             break;
4550          case 16:
4551             vmem_op = aco_opcode::buffer_store_dwordx4;
4552             smem_op = aco_opcode::s_buffer_store_dwordx4;
4553             break;
4554          default:
4555             unreachable("Store SSBO not implemented for this size.");
4556       }
4557       if (ctx->stage == fragment_fs)
4558          smem_op = aco_opcode::p_fs_buffer_store_smem;
4559
4560       if (smem) {
4561          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4562          store->operands[0] = Operand(rsrc);
4563          if (start) {
4564             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4565                                 offset, Operand(start * elem_size_bytes));
4566             store->operands[1] = Operand(off);
4567          } else {
4568             store->operands[1] = Operand(offset);
4569          }
4570          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4571             store->operands[1].setFixed(m0);
4572          store->operands[2] = Operand(write_data);
4573          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4574          store->dlc = false;
4575          store->disable_wqm = true;
4576          store->barrier = barrier_buffer;
4577          ctx->block->instructions.emplace_back(std::move(store));
4578          ctx->program->wb_smem_l1_on_end = true;
4579          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4580             ctx->block->kind |= block_kind_needs_lowering;
4581             ctx->program->needs_exact = true;
4582          }
4583       } else {
4584          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4585          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4586          store->operands[1] = Operand(rsrc);
4587          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4588          store->operands[3] = Operand(write_data);
4589          store->offset = start * elem_size_bytes;
4590          store->offen = (offset.type() == RegType::vgpr);
4591          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4592          store->dlc = false;
4593          store->disable_wqm = true;
4594          store->barrier = barrier_buffer;
4595          ctx->program->needs_exact = true;
4596          ctx->block->instructions.emplace_back(std::move(store));
4597       }
4598    }
4599 }
4600
4601 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4602 {
4603    /* return the previous value if dest is ever used */
4604    bool return_previous = false;
4605    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4606       return_previous = true;
4607       break;
4608    }
4609    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4610       return_previous = true;
4611       break;
4612    }
4613
4614    Builder bld(ctx->program, ctx->block);
4615    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4616
4617    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4618       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4619                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4620
4621    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4622    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4623    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4624
4625    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4626
4627    aco_opcode op32, op64;
4628    switch (instr->intrinsic) {
4629       case nir_intrinsic_ssbo_atomic_add:
4630          op32 = aco_opcode::buffer_atomic_add;
4631          op64 = aco_opcode::buffer_atomic_add_x2;
4632          break;
4633       case nir_intrinsic_ssbo_atomic_imin:
4634          op32 = aco_opcode::buffer_atomic_smin;
4635          op64 = aco_opcode::buffer_atomic_smin_x2;
4636          break;
4637       case nir_intrinsic_ssbo_atomic_umin:
4638          op32 = aco_opcode::buffer_atomic_umin;
4639          op64 = aco_opcode::buffer_atomic_umin_x2;
4640          break;
4641       case nir_intrinsic_ssbo_atomic_imax:
4642          op32 = aco_opcode::buffer_atomic_smax;
4643          op64 = aco_opcode::buffer_atomic_smax_x2;
4644          break;
4645       case nir_intrinsic_ssbo_atomic_umax:
4646          op32 = aco_opcode::buffer_atomic_umax;
4647          op64 = aco_opcode::buffer_atomic_umax_x2;
4648          break;
4649       case nir_intrinsic_ssbo_atomic_and:
4650          op32 = aco_opcode::buffer_atomic_and;
4651          op64 = aco_opcode::buffer_atomic_and_x2;
4652          break;
4653       case nir_intrinsic_ssbo_atomic_or:
4654          op32 = aco_opcode::buffer_atomic_or;
4655          op64 = aco_opcode::buffer_atomic_or_x2;
4656          break;
4657       case nir_intrinsic_ssbo_atomic_xor:
4658          op32 = aco_opcode::buffer_atomic_xor;
4659          op64 = aco_opcode::buffer_atomic_xor_x2;
4660          break;
4661       case nir_intrinsic_ssbo_atomic_exchange:
4662          op32 = aco_opcode::buffer_atomic_swap;
4663          op64 = aco_opcode::buffer_atomic_swap_x2;
4664          break;
4665       case nir_intrinsic_ssbo_atomic_comp_swap:
4666          op32 = aco_opcode::buffer_atomic_cmpswap;
4667          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4668          break;
4669       default:
4670          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4671    }
4672    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4673    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4674    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4675    mubuf->operands[1] = Operand(rsrc);
4676    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4677    mubuf->operands[3] = Operand(data);
4678    if (return_previous)
4679       mubuf->definitions[0] = Definition(dst);
4680    mubuf->offset = 0;
4681    mubuf->offen = (offset.type() == RegType::vgpr);
4682    mubuf->glc = return_previous;
4683    mubuf->dlc = false; /* Not needed for atomics */
4684    mubuf->disable_wqm = true;
4685    mubuf->barrier = barrier_buffer;
4686    ctx->program->needs_exact = true;
4687    ctx->block->instructions.emplace_back(std::move(mubuf));
4688 }
4689
4690 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4691
4692    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4693    Builder bld(ctx->program, ctx->block);
4694    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4695    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4696 }
4697
4698 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4699 {
4700    Builder bld(ctx->program, ctx->block);
4701    unsigned num_components = instr->num_components;
4702    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4703
4704    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4705    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4706
4707    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4708    bool dlc = glc && ctx->options->chip_class >= GFX10;
4709    aco_opcode op;
4710    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4711       bool global = ctx->options->chip_class >= GFX9;
4712       aco_opcode op;
4713       switch (num_bytes) {
4714       case 4:
4715          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4716          break;
4717       case 8:
4718          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4719          break;
4720       case 12:
4721          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4722          break;
4723       case 16:
4724          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4725          break;
4726       default:
4727          unreachable("load_global not implemented for this size.");
4728       }
4729       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4730       flat->operands[0] = Operand(addr);
4731       flat->operands[1] = Operand(s1);
4732       flat->glc = glc;
4733       flat->dlc = dlc;
4734       flat->barrier = barrier_buffer;
4735
4736       if (dst.type() == RegType::sgpr) {
4737          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4738          flat->definitions[0] = Definition(vec);
4739          ctx->block->instructions.emplace_back(std::move(flat));
4740          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4741       } else {
4742          flat->definitions[0] = Definition(dst);
4743          ctx->block->instructions.emplace_back(std::move(flat));
4744       }
4745       emit_split_vector(ctx, dst, num_components);
4746    } else {
4747       switch (num_bytes) {
4748          case 4:
4749             op = aco_opcode::s_load_dword;
4750             break;
4751          case 8:
4752             op = aco_opcode::s_load_dwordx2;
4753             break;
4754          case 12:
4755          case 16:
4756             op = aco_opcode::s_load_dwordx4;
4757             break;
4758          default:
4759             unreachable("load_global not implemented for this size.");
4760       }
4761       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4762       load->operands[0] = Operand(addr);
4763       load->operands[1] = Operand(0u);
4764       load->definitions[0] = Definition(dst);
4765       load->glc = glc;
4766       load->dlc = dlc;
4767       load->barrier = barrier_buffer;
4768       assert(ctx->options->chip_class >= GFX8 || !glc);
4769
4770       if (dst.size() == 3) {
4771          /* trim vector */
4772          Temp vec = bld.tmp(s4);
4773          load->definitions[0] = Definition(vec);
4774          ctx->block->instructions.emplace_back(std::move(load));
4775          emit_split_vector(ctx, vec, 4);
4776
4777          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4778                     emit_extract_vector(ctx, vec, 0, s1),
4779                     emit_extract_vector(ctx, vec, 1, s1),
4780                     emit_extract_vector(ctx, vec, 2, s1));
4781       } else {
4782          ctx->block->instructions.emplace_back(std::move(load));
4783       }
4784    }
4785 }
4786
4787 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4788 {
4789    Builder bld(ctx->program, ctx->block);
4790    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4791
4792    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4793    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4794
4795    unsigned writemask = nir_intrinsic_write_mask(instr);
4796    while (writemask) {
4797       int start, count;
4798       u_bit_scan_consecutive_range(&writemask, &start, &count);
4799       unsigned num_bytes = count * elem_size_bytes;
4800
4801       Temp write_data = data;
4802       if (count != instr->num_components) {
4803          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4804          for (int i = 0; i < count; i++)
4805             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4806          write_data = bld.tmp(RegType::vgpr, count);
4807          vec->definitions[0] = Definition(write_data);
4808          ctx->block->instructions.emplace_back(std::move(vec));
4809       }
4810
4811       unsigned offset = start * elem_size_bytes;
4812       if (offset > 0 && ctx->options->chip_class < GFX9) {
4813          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4814          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4815          Temp carry = bld.tmp(bld.lm);
4816          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4817
4818          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4819                   Operand(offset), addr0);
4820          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4821                   Operand(0u), addr1,
4822                   carry).def(1).setHint(vcc);
4823
4824          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4825
4826          offset = 0;
4827       }
4828
4829       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4830       bool global = ctx->options->chip_class >= GFX9;
4831       aco_opcode op;
4832       switch (num_bytes) {
4833       case 4:
4834          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4835          break;
4836       case 8:
4837          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4838          break;
4839       case 12:
4840          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4841          break;
4842       case 16:
4843          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4844          break;
4845       default:
4846          unreachable("store_global not implemented for this size.");
4847       }
4848       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4849       flat->operands[0] = Operand(addr);
4850       flat->operands[1] = Operand(s1);
4851       flat->operands[2] = Operand(data);
4852       flat->glc = glc;
4853       flat->dlc = false;
4854       flat->offset = offset;
4855       flat->disable_wqm = true;
4856       flat->barrier = barrier_buffer;
4857       ctx->program->needs_exact = true;
4858       ctx->block->instructions.emplace_back(std::move(flat));
4859    }
4860 }
4861
4862 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4863 {
4864    /* return the previous value if dest is ever used */
4865    bool return_previous = false;
4866    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4867       return_previous = true;
4868       break;
4869    }
4870    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4871       return_previous = true;
4872       break;
4873    }
4874
4875    Builder bld(ctx->program, ctx->block);
4876    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4877    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4878
4879    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4880       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4881                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4882
4883    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4884
4885    bool global = ctx->options->chip_class >= GFX9;
4886    aco_opcode op32, op64;
4887    switch (instr->intrinsic) {
4888       case nir_intrinsic_global_atomic_add:
4889          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4890          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4891          break;
4892       case nir_intrinsic_global_atomic_imin:
4893          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4894          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4895          break;
4896       case nir_intrinsic_global_atomic_umin:
4897          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4898          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4899          break;
4900       case nir_intrinsic_global_atomic_imax:
4901          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4902          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4903          break;
4904       case nir_intrinsic_global_atomic_umax:
4905          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4906          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4907          break;
4908       case nir_intrinsic_global_atomic_and:
4909          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4910          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4911          break;
4912       case nir_intrinsic_global_atomic_or:
4913          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4914          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4915          break;
4916       case nir_intrinsic_global_atomic_xor:
4917          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4918          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4919          break;
4920       case nir_intrinsic_global_atomic_exchange:
4921          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4922          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4923          break;
4924       case nir_intrinsic_global_atomic_comp_swap:
4925          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4926          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4927          break;
4928       default:
4929          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4930    }
4931    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4932    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4933    flat->operands[0] = Operand(addr);
4934    flat->operands[1] = Operand(s1);
4935    flat->operands[2] = Operand(data);
4936    if (return_previous)
4937       flat->definitions[0] = Definition(dst);
4938    flat->glc = return_previous;
4939    flat->dlc = false; /* Not needed for atomics */
4940    flat->offset = 0;
4941    flat->disable_wqm = true;
4942    flat->barrier = barrier_buffer;
4943    ctx->program->needs_exact = true;
4944    ctx->block->instructions.emplace_back(std::move(flat));
4945 }
4946
4947 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4948    Builder bld(ctx->program, ctx->block);
4949    switch(instr->intrinsic) {
4950       case nir_intrinsic_group_memory_barrier:
4951       case nir_intrinsic_memory_barrier:
4952          bld.barrier(aco_opcode::p_memory_barrier_all);
4953          break;
4954       case nir_intrinsic_memory_barrier_atomic_counter:
4955          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4956          break;
4957       case nir_intrinsic_memory_barrier_buffer:
4958          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4959          break;
4960       case nir_intrinsic_memory_barrier_image:
4961          bld.barrier(aco_opcode::p_memory_barrier_image);
4962          break;
4963       case nir_intrinsic_memory_barrier_shared:
4964          bld.barrier(aco_opcode::p_memory_barrier_shared);
4965          break;
4966       default:
4967          unreachable("Unimplemented memory barrier intrinsic");
4968          break;
4969    }
4970 }
4971
4972 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4973 {
4974    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4975    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4976    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4977    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4978    Builder bld(ctx->program, ctx->block);
4979
4980    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4981    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4982    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4983 }
4984
4985 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4986 {
4987    unsigned writemask = nir_intrinsic_write_mask(instr);
4988    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4989    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4990    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4991    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4992
4993    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4994    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4995 }
4996
4997 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4998 {
4999    unsigned offset = nir_intrinsic_base(instr);
5000    Operand m = load_lds_size_m0(ctx);
5001    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5002    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5003
5004    unsigned num_operands = 3;
5005    aco_opcode op32, op64, op32_rtn, op64_rtn;
5006    switch(instr->intrinsic) {
5007       case nir_intrinsic_shared_atomic_add:
5008          op32 = aco_opcode::ds_add_u32;
5009          op64 = aco_opcode::ds_add_u64;
5010          op32_rtn = aco_opcode::ds_add_rtn_u32;
5011          op64_rtn = aco_opcode::ds_add_rtn_u64;
5012          break;
5013       case nir_intrinsic_shared_atomic_imin:
5014          op32 = aco_opcode::ds_min_i32;
5015          op64 = aco_opcode::ds_min_i64;
5016          op32_rtn = aco_opcode::ds_min_rtn_i32;
5017          op64_rtn = aco_opcode::ds_min_rtn_i64;
5018          break;
5019       case nir_intrinsic_shared_atomic_umin:
5020          op32 = aco_opcode::ds_min_u32;
5021          op64 = aco_opcode::ds_min_u64;
5022          op32_rtn = aco_opcode::ds_min_rtn_u32;
5023          op64_rtn = aco_opcode::ds_min_rtn_u64;
5024          break;
5025       case nir_intrinsic_shared_atomic_imax:
5026          op32 = aco_opcode::ds_max_i32;
5027          op64 = aco_opcode::ds_max_i64;
5028          op32_rtn = aco_opcode::ds_max_rtn_i32;
5029          op64_rtn = aco_opcode::ds_max_rtn_i64;
5030          break;
5031       case nir_intrinsic_shared_atomic_umax:
5032          op32 = aco_opcode::ds_max_u32;
5033          op64 = aco_opcode::ds_max_u64;
5034          op32_rtn = aco_opcode::ds_max_rtn_u32;
5035          op64_rtn = aco_opcode::ds_max_rtn_u64;
5036          break;
5037       case nir_intrinsic_shared_atomic_and:
5038          op32 = aco_opcode::ds_and_b32;
5039          op64 = aco_opcode::ds_and_b64;
5040          op32_rtn = aco_opcode::ds_and_rtn_b32;
5041          op64_rtn = aco_opcode::ds_and_rtn_b64;
5042          break;
5043       case nir_intrinsic_shared_atomic_or:
5044          op32 = aco_opcode::ds_or_b32;
5045          op64 = aco_opcode::ds_or_b64;
5046          op32_rtn = aco_opcode::ds_or_rtn_b32;
5047          op64_rtn = aco_opcode::ds_or_rtn_b64;
5048          break;
5049       case nir_intrinsic_shared_atomic_xor:
5050          op32 = aco_opcode::ds_xor_b32;
5051          op64 = aco_opcode::ds_xor_b64;
5052          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5053          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5054          break;
5055       case nir_intrinsic_shared_atomic_exchange:
5056          op32 = aco_opcode::ds_write_b32;
5057          op64 = aco_opcode::ds_write_b64;
5058          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5059          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5060          break;
5061       case nir_intrinsic_shared_atomic_comp_swap:
5062          op32 = aco_opcode::ds_cmpst_b32;
5063          op64 = aco_opcode::ds_cmpst_b64;
5064          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5065          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5066          num_operands = 4;
5067          break;
5068       default:
5069          unreachable("Unhandled shared atomic intrinsic");
5070    }
5071
5072    /* return the previous value if dest is ever used */
5073    bool return_previous = false;
5074    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5075       return_previous = true;
5076       break;
5077    }
5078    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5079       return_previous = true;
5080       break;
5081    }
5082
5083    aco_opcode op;
5084    if (data.size() == 1) {
5085       assert(instr->dest.ssa.bit_size == 32);
5086       op = return_previous ? op32_rtn : op32;
5087    } else {
5088       assert(instr->dest.ssa.bit_size == 64);
5089       op = return_previous ? op64_rtn : op64;
5090    }
5091
5092    if (offset > 65535) {
5093       Builder bld(ctx->program, ctx->block);
5094       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5095       offset = 0;
5096    }
5097
5098    aco_ptr<DS_instruction> ds;
5099    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5100    ds->operands[0] = Operand(address);
5101    ds->operands[1] = Operand(data);
5102    if (num_operands == 4)
5103       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5104    ds->operands[num_operands - 1] = m;
5105    ds->offset0 = offset;
5106    if (return_previous)
5107       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5108    ctx->block->instructions.emplace_back(std::move(ds));
5109 }
5110
5111 Temp get_scratch_resource(isel_context *ctx)
5112 {
5113    Builder bld(ctx->program, ctx->block);
5114    Temp scratch_addr = ctx->program->private_segment_buffer;
5115    if (ctx->stage != compute_cs)
5116       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5117
5118    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5119                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5120
5121    if (ctx->program->chip_class >= GFX10) {
5122       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5123                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5124                    S_008F0C_RESOURCE_LEVEL(1);
5125    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5126       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5127                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5128    }
5129
5130    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5131    if (ctx->program->chip_class <= GFX8)
5132       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5133
5134    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5135 }
5136
5137 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5138    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5139    Builder bld(ctx->program, ctx->block);
5140    Temp rsrc = get_scratch_resource(ctx);
5141    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5142    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5143
5144    aco_opcode op;
5145    switch (dst.size()) {
5146       case 1:
5147          op = aco_opcode::buffer_load_dword;
5148          break;
5149       case 2:
5150          op = aco_opcode::buffer_load_dwordx2;
5151          break;
5152       case 3:
5153          op = aco_opcode::buffer_load_dwordx3;
5154          break;
5155       case 4:
5156          op = aco_opcode::buffer_load_dwordx4;
5157          break;
5158       case 6:
5159       case 8: {
5160          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5161          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5162                                 bld.def(v4), offset, rsrc,
5163                                 ctx->program->scratch_offset, 0, true);
5164          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5165                                                   aco_opcode::buffer_load_dwordx4,
5166                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5167                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5168          emit_split_vector(ctx, lower, 2);
5169          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5170          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5171          if (dst.size() == 8) {
5172             emit_split_vector(ctx, upper, 2);
5173             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5174             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5175          } else {
5176             elems[2] = upper;
5177          }
5178
5179          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5180                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5181          for (unsigned i = 0; i < dst.size() / 2; i++)
5182             vec->operands[i] = Operand(elems[i]);
5183          vec->definitions[0] = Definition(dst);
5184          bld.insert(std::move(vec));
5185          ctx->allocated_vec.emplace(dst.id(), elems);
5186          return;
5187       }
5188       default:
5189          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5190    }
5191
5192    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5193    emit_split_vector(ctx, dst, instr->num_components);
5194 }
5195
5196 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5197    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5198    Builder bld(ctx->program, ctx->block);
5199    Temp rsrc = get_scratch_resource(ctx);
5200    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5201    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5202
5203    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5204    unsigned writemask = nir_intrinsic_write_mask(instr);
5205
5206    while (writemask) {
5207       int start, count;
5208       u_bit_scan_consecutive_range(&writemask, &start, &count);
5209       int num_bytes = count * elem_size_bytes;
5210
5211       if (num_bytes > 16) {
5212          assert(elem_size_bytes == 8);
5213          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5214          count = 2;
5215          num_bytes = 16;
5216       }
5217
5218       // TODO: check alignment of sub-dword stores
5219       // TODO: split 3 bytes. there is no store instruction for that
5220
5221       Temp write_data;
5222       if (count != instr->num_components) {
5223          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5224          for (int i = 0; i < count; i++) {
5225             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5226             vec->operands[i] = Operand(elem);
5227          }
5228          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5229          vec->definitions[0] = Definition(write_data);
5230          ctx->block->instructions.emplace_back(std::move(vec));
5231       } else {
5232          write_data = data;
5233       }
5234
5235       aco_opcode op;
5236       switch (num_bytes) {
5237          case 4:
5238             op = aco_opcode::buffer_store_dword;
5239             break;
5240          case 8:
5241             op = aco_opcode::buffer_store_dwordx2;
5242             break;
5243          case 12:
5244             op = aco_opcode::buffer_store_dwordx3;
5245             break;
5246          case 16:
5247             op = aco_opcode::buffer_store_dwordx4;
5248             break;
5249          default:
5250             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5251       }
5252
5253       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5254    }
5255 }
5256
5257 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5258    uint8_t log2_ps_iter_samples;
5259    if (ctx->program->info->ps.force_persample) {
5260       log2_ps_iter_samples =
5261          util_logbase2(ctx->options->key.fs.num_samples);
5262    } else {
5263       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5264    }
5265
5266    /* The bit pattern matches that used by fixed function fragment
5267     * processing. */
5268    static const unsigned ps_iter_masks[] = {
5269       0xffff, /* not used */
5270       0x5555,
5271       0x1111,
5272       0x0101,
5273       0x0001,
5274    };
5275    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5276
5277    Builder bld(ctx->program, ctx->block);
5278
5279    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5280                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5281    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5282    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5283    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5284    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5285 }
5286
5287 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5288 {
5289    Builder bld(ctx->program, ctx->block);
5290
5291    if (cluster_size == 1) {
5292       return src;
5293    } if (op == nir_op_iand && cluster_size == 4) {
5294       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5295       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5296       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5297                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5298    } else if (op == nir_op_ior && cluster_size == 4) {
5299       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5300       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5301                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5302    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5303       //subgroupAnd(val) -> (exec & ~val) == 0
5304       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5305       return bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5306    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5307       //subgroupOr(val) -> (val & exec) != 0
5308       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5309       return bool_to_vector_condition(ctx, tmp);
5310    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5311       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5312       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5313       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5314       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5315       return bool_to_vector_condition(ctx, tmp);
5316    } else {
5317       //subgroupClustered{And,Or,Xor}(val, n) ->
5318       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5319       //cluster_offset = ~(n - 1) & lane_id
5320       //cluster_mask = ((1 << n) - 1)
5321       //subgroupClusteredAnd():
5322       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5323       //subgroupClusteredOr():
5324       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5325       //subgroupClusteredXor():
5326       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5327       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5328       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5329
5330       Temp tmp;
5331       if (op == nir_op_iand)
5332          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5333       else
5334          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5335
5336       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5337
5338       if (ctx->program->chip_class <= GFX7)
5339          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5340       else if (ctx->program->wave_size == 64)
5341          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5342       else
5343          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5344       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5345       if (cluster_mask != 0xffffffff)
5346          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5347
5348       Definition cmp_def = Definition();
5349       if (op == nir_op_iand) {
5350          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5351       } else if (op == nir_op_ior) {
5352          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5353       } else if (op == nir_op_ixor) {
5354          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5355                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5356          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5357       }
5358       cmp_def.setHint(vcc);
5359       return cmp_def.getTemp();
5360    }
5361 }
5362
5363 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5364 {
5365    Builder bld(ctx->program, ctx->block);
5366
5367    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5368    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5369    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5370    Temp tmp;
5371    if (op == nir_op_iand)
5372       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5373    else
5374       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5375
5376    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5377    Temp lo = lohi.def(0).getTemp();
5378    Temp hi = lohi.def(1).getTemp();
5379    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5380
5381    Definition cmp_def = Definition();
5382    if (op == nir_op_iand)
5383       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5384    else if (op == nir_op_ior)
5385       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5386    else if (op == nir_op_ixor)
5387       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5388                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5389    cmp_def.setHint(vcc);
5390    return cmp_def.getTemp();
5391 }
5392
5393 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5394 {
5395    Builder bld(ctx->program, ctx->block);
5396
5397    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5398    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5399    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5400    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5401    if (op == nir_op_iand)
5402       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5403    else if (op == nir_op_ior)
5404       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5405    else if (op == nir_op_ixor)
5406       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5407
5408    assert(false);
5409    return Temp();
5410 }
5411
5412 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5413 {
5414    Builder bld(ctx->program, ctx->block);
5415    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5416    if (src.regClass().type() == RegType::vgpr) {
5417       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5418    } else if (src.regClass() == s1) {
5419       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5420    } else if (src.regClass() == s2) {
5421       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5422    } else {
5423       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5424       nir_print_instr(&instr->instr, stderr);
5425       fprintf(stderr, "\n");
5426    }
5427 }
5428
5429 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5430 {
5431    Builder bld(ctx->program, ctx->block);
5432    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5433    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5434    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5435
5436    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5437    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5438    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5439    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5440
5441    /* Build DD X/Y */
5442    if (ctx->program->chip_class >= GFX8) {
5443       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5444       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5445       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5446       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5447       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5448       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5449    } else {
5450       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5451       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5452       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5453       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5454       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5455       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5456       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5457       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5458       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5459       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5460    }
5461
5462    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5463    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5464    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5465    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5466    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5467    Temp wqm1 = bld.tmp(v1);
5468    emit_wqm(ctx, tmp1, wqm1, true);
5469    Temp wqm2 = bld.tmp(v1);
5470    emit_wqm(ctx, tmp2, wqm2, true);
5471    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5472    return;
5473 }
5474
5475 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5476 {
5477    Builder bld(ctx->program, ctx->block);
5478    switch(instr->intrinsic) {
5479    case nir_intrinsic_load_barycentric_sample:
5480    case nir_intrinsic_load_barycentric_pixel:
5481    case nir_intrinsic_load_barycentric_centroid: {
5482       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5483       Temp bary = Temp(0, s2);
5484       switch (mode) {
5485       case INTERP_MODE_SMOOTH:
5486       case INTERP_MODE_NONE:
5487          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5488             bary = get_arg(ctx, ctx->args->ac.persp_center);
5489          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5490             bary = ctx->persp_centroid;
5491          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5492             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5493          break;
5494       case INTERP_MODE_NOPERSPECTIVE:
5495          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5496             bary = get_arg(ctx, ctx->args->ac.linear_center);
5497          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5498             bary = ctx->linear_centroid;
5499          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5500             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5501          break;
5502       default:
5503          break;
5504       }
5505       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5506       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5507       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5508       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5509                  Operand(p1), Operand(p2));
5510       emit_split_vector(ctx, dst, 2);
5511       break;
5512    }
5513    case nir_intrinsic_load_barycentric_at_sample: {
5514       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5515       switch (ctx->options->key.fs.num_samples) {
5516          case 2: sample_pos_offset += 1 << 3; break;
5517          case 4: sample_pos_offset += 3 << 3; break;
5518          case 8: sample_pos_offset += 7 << 3; break;
5519          default: break;
5520       }
5521       Temp sample_pos;
5522       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5523       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5524       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5525       if (addr.type() == RegType::sgpr) {
5526          Operand offset;
5527          if (const_addr) {
5528             sample_pos_offset += const_addr->u32 << 3;
5529             offset = Operand(sample_pos_offset);
5530          } else if (ctx->options->chip_class >= GFX9) {
5531             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5532          } else {
5533             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5534             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5535          }
5536          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5537
5538       } else if (ctx->options->chip_class >= GFX9) {
5539          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5540          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5541       } else {
5542          /* addr += private_segment_buffer + sample_pos_offset */
5543          Temp tmp0 = bld.tmp(s1);
5544          Temp tmp1 = bld.tmp(s1);
5545          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5546          Definition scc_tmp = bld.def(s1, scc);
5547          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5548          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5549          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5550          Temp pck0 = bld.tmp(v1);
5551          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5552          tmp1 = as_vgpr(ctx, tmp1);
5553          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5554          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5555
5556          /* sample_pos = flat_load_dwordx2 addr */
5557          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5558       }
5559
5560       /* sample_pos -= 0.5 */
5561       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5562       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5563       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5564       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5565       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5566
5567       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5568       break;
5569    }
5570    case nir_intrinsic_load_barycentric_at_offset: {
5571       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5572       RegClass rc = RegClass(offset.type(), 1);
5573       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5574       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5575       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5576       break;
5577    }
5578    case nir_intrinsic_load_front_face: {
5579       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5580                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5581       break;
5582    }
5583    case nir_intrinsic_load_view_index:
5584    case nir_intrinsic_load_layer_id: {
5585       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5586          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5587          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5588          break;
5589       }
5590
5591       unsigned idx = nir_intrinsic_base(instr);
5592       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5593                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5594       break;
5595    }
5596    case nir_intrinsic_load_frag_coord: {
5597       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5598       break;
5599    }
5600    case nir_intrinsic_load_sample_pos: {
5601       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5602       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5603       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5604                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5605                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5606       break;
5607    }
5608    case nir_intrinsic_load_interpolated_input:
5609       visit_load_interpolated_input(ctx, instr);
5610       break;
5611    case nir_intrinsic_store_output:
5612       visit_store_output(ctx, instr);
5613       break;
5614    case nir_intrinsic_load_input:
5615       visit_load_input(ctx, instr);
5616       break;
5617    case nir_intrinsic_load_ubo:
5618       visit_load_ubo(ctx, instr);
5619       break;
5620    case nir_intrinsic_load_push_constant:
5621       visit_load_push_constant(ctx, instr);
5622       break;
5623    case nir_intrinsic_load_constant:
5624       visit_load_constant(ctx, instr);
5625       break;
5626    case nir_intrinsic_vulkan_resource_index:
5627       visit_load_resource(ctx, instr);
5628       break;
5629    case nir_intrinsic_discard:
5630       visit_discard(ctx, instr);
5631       break;
5632    case nir_intrinsic_discard_if:
5633       visit_discard_if(ctx, instr);
5634       break;
5635    case nir_intrinsic_load_shared:
5636       visit_load_shared(ctx, instr);
5637       break;
5638    case nir_intrinsic_store_shared:
5639       visit_store_shared(ctx, instr);
5640       break;
5641    case nir_intrinsic_shared_atomic_add:
5642    case nir_intrinsic_shared_atomic_imin:
5643    case nir_intrinsic_shared_atomic_umin:
5644    case nir_intrinsic_shared_atomic_imax:
5645    case nir_intrinsic_shared_atomic_umax:
5646    case nir_intrinsic_shared_atomic_and:
5647    case nir_intrinsic_shared_atomic_or:
5648    case nir_intrinsic_shared_atomic_xor:
5649    case nir_intrinsic_shared_atomic_exchange:
5650    case nir_intrinsic_shared_atomic_comp_swap:
5651       visit_shared_atomic(ctx, instr);
5652       break;
5653    case nir_intrinsic_image_deref_load:
5654       visit_image_load(ctx, instr);
5655       break;
5656    case nir_intrinsic_image_deref_store:
5657       visit_image_store(ctx, instr);
5658       break;
5659    case nir_intrinsic_image_deref_atomic_add:
5660    case nir_intrinsic_image_deref_atomic_umin:
5661    case nir_intrinsic_image_deref_atomic_imin:
5662    case nir_intrinsic_image_deref_atomic_umax:
5663    case nir_intrinsic_image_deref_atomic_imax:
5664    case nir_intrinsic_image_deref_atomic_and:
5665    case nir_intrinsic_image_deref_atomic_or:
5666    case nir_intrinsic_image_deref_atomic_xor:
5667    case nir_intrinsic_image_deref_atomic_exchange:
5668    case nir_intrinsic_image_deref_atomic_comp_swap:
5669       visit_image_atomic(ctx, instr);
5670       break;
5671    case nir_intrinsic_image_deref_size:
5672       visit_image_size(ctx, instr);
5673       break;
5674    case nir_intrinsic_load_ssbo:
5675       visit_load_ssbo(ctx, instr);
5676       break;
5677    case nir_intrinsic_store_ssbo:
5678       visit_store_ssbo(ctx, instr);
5679       break;
5680    case nir_intrinsic_load_global:
5681       visit_load_global(ctx, instr);
5682       break;
5683    case nir_intrinsic_store_global:
5684       visit_store_global(ctx, instr);
5685       break;
5686    case nir_intrinsic_global_atomic_add:
5687    case nir_intrinsic_global_atomic_imin:
5688    case nir_intrinsic_global_atomic_umin:
5689    case nir_intrinsic_global_atomic_imax:
5690    case nir_intrinsic_global_atomic_umax:
5691    case nir_intrinsic_global_atomic_and:
5692    case nir_intrinsic_global_atomic_or:
5693    case nir_intrinsic_global_atomic_xor:
5694    case nir_intrinsic_global_atomic_exchange:
5695    case nir_intrinsic_global_atomic_comp_swap:
5696       visit_global_atomic(ctx, instr);
5697       break;
5698    case nir_intrinsic_ssbo_atomic_add:
5699    case nir_intrinsic_ssbo_atomic_imin:
5700    case nir_intrinsic_ssbo_atomic_umin:
5701    case nir_intrinsic_ssbo_atomic_imax:
5702    case nir_intrinsic_ssbo_atomic_umax:
5703    case nir_intrinsic_ssbo_atomic_and:
5704    case nir_intrinsic_ssbo_atomic_or:
5705    case nir_intrinsic_ssbo_atomic_xor:
5706    case nir_intrinsic_ssbo_atomic_exchange:
5707    case nir_intrinsic_ssbo_atomic_comp_swap:
5708       visit_atomic_ssbo(ctx, instr);
5709       break;
5710    case nir_intrinsic_load_scratch:
5711       visit_load_scratch(ctx, instr);
5712       break;
5713    case nir_intrinsic_store_scratch:
5714       visit_store_scratch(ctx, instr);
5715       break;
5716    case nir_intrinsic_get_buffer_size:
5717       visit_get_buffer_size(ctx, instr);
5718       break;
5719    case nir_intrinsic_barrier: {
5720       unsigned* bsize = ctx->program->info->cs.block_size;
5721       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5722       if (workgroup_size > ctx->program->wave_size)
5723          bld.sopp(aco_opcode::s_barrier);
5724       break;
5725    }
5726    case nir_intrinsic_group_memory_barrier:
5727    case nir_intrinsic_memory_barrier:
5728    case nir_intrinsic_memory_barrier_atomic_counter:
5729    case nir_intrinsic_memory_barrier_buffer:
5730    case nir_intrinsic_memory_barrier_image:
5731    case nir_intrinsic_memory_barrier_shared:
5732       emit_memory_barrier(ctx, instr);
5733       break;
5734    case nir_intrinsic_load_num_work_groups: {
5735       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5736       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5737       emit_split_vector(ctx, dst, 3);
5738       break;
5739    }
5740    case nir_intrinsic_load_local_invocation_id: {
5741       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5742       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5743       emit_split_vector(ctx, dst, 3);
5744       break;
5745    }
5746    case nir_intrinsic_load_work_group_id: {
5747       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5748       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5749       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5750                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5751                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5752                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5753       emit_split_vector(ctx, dst, 3);
5754       break;
5755    }
5756    case nir_intrinsic_load_local_invocation_index: {
5757       Temp id = emit_mbcnt(ctx, bld.def(v1));
5758
5759       /* The tg_size bits [6:11] contain the subgroup id,
5760        * we need this multiplied by the wave size, and then OR the thread id to it.
5761        */
5762       if (ctx->program->wave_size == 64) {
5763          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5764          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5765                                 get_arg(ctx, ctx->args->ac.tg_size));
5766          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5767       } else {
5768          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5769          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5770                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5771          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5772       }
5773       break;
5774    }
5775    case nir_intrinsic_load_subgroup_id: {
5776       if (ctx->stage == compute_cs) {
5777          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5778                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5779       } else {
5780          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5781       }
5782       break;
5783    }
5784    case nir_intrinsic_load_subgroup_invocation: {
5785       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5786       break;
5787    }
5788    case nir_intrinsic_load_num_subgroups: {
5789       if (ctx->stage == compute_cs)
5790          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5791                   get_arg(ctx, ctx->args->ac.tg_size));
5792       else
5793          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5794       break;
5795    }
5796    case nir_intrinsic_ballot: {
5797       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5798       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5799       Definition tmp = bld.def(dst.regClass());
5800       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5801       if (instr->src[0].ssa->bit_size == 1) {
5802          assert(src.regClass() == bld.lm);
5803          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5804       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5805          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5806       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5807          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5808       } else {
5809          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5810          nir_print_instr(&instr->instr, stderr);
5811          fprintf(stderr, "\n");
5812       }
5813       if (dst.size() != bld.lm.size()) {
5814          /* Wave32 with ballot size set to 64 */
5815          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5816       }
5817       emit_wqm(ctx, tmp.getTemp(), dst);
5818       break;
5819    }
5820    case nir_intrinsic_shuffle:
5821    case nir_intrinsic_read_invocation: {
5822       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5823       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5824          emit_uniform_subgroup(ctx, instr, src);
5825       } else {
5826          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5827          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5828             tid = bld.as_uniform(tid);
5829          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5830          if (src.regClass() == v1) {
5831             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5832          } else if (src.regClass() == v2) {
5833             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5834             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5835             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5836             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5837             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5838             emit_split_vector(ctx, dst, 2);
5839          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5840             assert(src.regClass() == bld.lm);
5841             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5842             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5843          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5844             assert(src.regClass() == bld.lm);
5845             Temp tmp;
5846             if (ctx->program->chip_class <= GFX7)
5847                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
5848             else if (ctx->program->wave_size == 64)
5849                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5850             else
5851                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5852             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5853             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5854             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5855          } else {
5856             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5857             nir_print_instr(&instr->instr, stderr);
5858             fprintf(stderr, "\n");
5859          }
5860       }
5861       break;
5862    }
5863    case nir_intrinsic_load_sample_id: {
5864       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5865                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5866       break;
5867    }
5868    case nir_intrinsic_load_sample_mask_in: {
5869       visit_load_sample_mask_in(ctx, instr);
5870       break;
5871    }
5872    case nir_intrinsic_read_first_invocation: {
5873       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5874       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5875       if (src.regClass() == v1) {
5876          emit_wqm(ctx,
5877                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5878                   dst);
5879       } else if (src.regClass() == v2) {
5880          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5881          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5882          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5883          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5884          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5885          emit_split_vector(ctx, dst, 2);
5886       } else if (instr->dest.ssa.bit_size == 1) {
5887          assert(src.regClass() == bld.lm);
5888          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5889                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5890          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5891       } else if (src.regClass() == s1) {
5892          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5893       } else if (src.regClass() == s2) {
5894          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5895       } else {
5896          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5897          nir_print_instr(&instr->instr, stderr);
5898          fprintf(stderr, "\n");
5899       }
5900       break;
5901    }
5902    case nir_intrinsic_vote_all: {
5903       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5904       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5905       assert(src.regClass() == bld.lm);
5906       assert(dst.regClass() == bld.lm);
5907
5908       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5909       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5910       emit_wqm(ctx, val, dst);
5911       break;
5912    }
5913    case nir_intrinsic_vote_any: {
5914       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5915       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5916       assert(src.regClass() == bld.lm);
5917       assert(dst.regClass() == bld.lm);
5918
5919       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5920       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), Operand(0u), bld.scc(tmp));
5921       emit_wqm(ctx, val, dst);
5922       break;
5923    }
5924    case nir_intrinsic_reduce:
5925    case nir_intrinsic_inclusive_scan:
5926    case nir_intrinsic_exclusive_scan: {
5927       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5928       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5929       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5930       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5931          nir_intrinsic_cluster_size(instr) : 0;
5932       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5933
5934       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5935          emit_uniform_subgroup(ctx, instr, src);
5936       } else if (instr->dest.ssa.bit_size == 1) {
5937          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5938             op = nir_op_iand;
5939          else if (op == nir_op_iadd)
5940             op = nir_op_ixor;
5941          else if (op == nir_op_umax || op == nir_op_imax)
5942             op = nir_op_ior;
5943          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5944
5945          switch (instr->intrinsic) {
5946          case nir_intrinsic_reduce:
5947             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5948             break;
5949          case nir_intrinsic_exclusive_scan:
5950             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5951             break;
5952          case nir_intrinsic_inclusive_scan:
5953             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5954             break;
5955          default:
5956             assert(false);
5957          }
5958       } else if (cluster_size == 1) {
5959          bld.copy(Definition(dst), src);
5960       } else {
5961          src = as_vgpr(ctx, src);
5962
5963          ReduceOp reduce_op;
5964          switch (op) {
5965          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5966             CASE(iadd)
5967             CASE(imul)
5968             CASE(fadd)
5969             CASE(fmul)
5970             CASE(imin)
5971             CASE(umin)
5972             CASE(fmin)
5973             CASE(imax)
5974             CASE(umax)
5975             CASE(fmax)
5976             CASE(iand)
5977             CASE(ior)
5978             CASE(ixor)
5979             default:
5980                unreachable("unknown reduction op");
5981          #undef CASE
5982          }
5983
5984          aco_opcode aco_op;
5985          switch (instr->intrinsic) {
5986             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5987             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5988             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5989             default:
5990                unreachable("unknown reduce intrinsic");
5991          }
5992
5993          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5994          reduce->operands[0] = Operand(src);
5995          // filled in by aco_reduce_assign.cpp, used internally as part of the
5996          // reduce sequence
5997          assert(dst.size() == 1 || dst.size() == 2);
5998          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5999          reduce->operands[2] = Operand(v1.as_linear());
6000
6001          Temp tmp_dst = bld.tmp(dst.regClass());
6002          reduce->definitions[0] = Definition(tmp_dst);
6003          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6004          reduce->definitions[2] = Definition();
6005          reduce->definitions[3] = Definition(scc, s1);
6006          reduce->definitions[4] = Definition();
6007          reduce->reduce_op = reduce_op;
6008          reduce->cluster_size = cluster_size;
6009          ctx->block->instructions.emplace_back(std::move(reduce));
6010
6011          emit_wqm(ctx, tmp_dst, dst);
6012       }
6013       break;
6014    }
6015    case nir_intrinsic_quad_broadcast: {
6016       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6017       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6018          emit_uniform_subgroup(ctx, instr, src);
6019       } else {
6020          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6021          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6022          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6023
6024          if (instr->dest.ssa.bit_size == 1) {
6025             assert(src.regClass() == bld.lm);
6026             assert(dst.regClass() == bld.lm);
6027             uint32_t half_mask = 0x11111111u << lane;
6028             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6029             Temp tmp = bld.tmp(bld.lm);
6030             bld.sop1(Builder::s_wqm, Definition(tmp),
6031                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6032                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6033             emit_wqm(ctx, tmp, dst);
6034          } else if (instr->dest.ssa.bit_size == 32) {
6035             if (ctx->program->chip_class >= GFX8)
6036                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6037             else
6038                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6039          } else if (instr->dest.ssa.bit_size == 64) {
6040             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6041             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6042             if (ctx->program->chip_class >= GFX8) {
6043                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6044                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6045             } else {
6046                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6047                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6048             }
6049             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6050             emit_split_vector(ctx, dst, 2);
6051          } else {
6052             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6053             nir_print_instr(&instr->instr, stderr);
6054             fprintf(stderr, "\n");
6055          }
6056       }
6057       break;
6058    }
6059    case nir_intrinsic_quad_swap_horizontal:
6060    case nir_intrinsic_quad_swap_vertical:
6061    case nir_intrinsic_quad_swap_diagonal:
6062    case nir_intrinsic_quad_swizzle_amd: {
6063       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6064       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6065          emit_uniform_subgroup(ctx, instr, src);
6066          break;
6067       }
6068       uint16_t dpp_ctrl = 0;
6069       switch (instr->intrinsic) {
6070       case nir_intrinsic_quad_swap_horizontal:
6071          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6072          break;
6073       case nir_intrinsic_quad_swap_vertical:
6074          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6075          break;
6076       case nir_intrinsic_quad_swap_diagonal:
6077          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6078          break;
6079       case nir_intrinsic_quad_swizzle_amd:
6080          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6081          break;
6082       default:
6083          break;
6084       }
6085       if (ctx->program->chip_class < GFX8)
6086          dpp_ctrl |= (1 << 15);
6087
6088       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6089       if (instr->dest.ssa.bit_size == 1) {
6090          assert(src.regClass() == bld.lm);
6091          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6092          if (ctx->program->chip_class >= GFX8)
6093             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6094          else
6095             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6096          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6097          emit_wqm(ctx, tmp, dst);
6098       } else if (instr->dest.ssa.bit_size == 32) {
6099          Temp tmp;
6100          if (ctx->program->chip_class >= GFX8)
6101             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6102          else
6103             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6104          emit_wqm(ctx, tmp, dst);
6105       } else if (instr->dest.ssa.bit_size == 64) {
6106          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6107          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6108          if (ctx->program->chip_class >= GFX8) {
6109             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6110             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6111          } else {
6112             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6113             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6114          }
6115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6116          emit_split_vector(ctx, dst, 2);
6117       } else {
6118          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6119          nir_print_instr(&instr->instr, stderr);
6120          fprintf(stderr, "\n");
6121       }
6122       break;
6123    }
6124    case nir_intrinsic_masked_swizzle_amd: {
6125       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6126       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6127          emit_uniform_subgroup(ctx, instr, src);
6128          break;
6129       }
6130       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6131       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6132       if (dst.regClass() == v1) {
6133          emit_wqm(ctx,
6134                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6135                   dst);
6136       } else if (dst.regClass() == v2) {
6137          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6138          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6139          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6140          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6141          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6142          emit_split_vector(ctx, dst, 2);
6143       } else {
6144          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6145          nir_print_instr(&instr->instr, stderr);
6146          fprintf(stderr, "\n");
6147       }
6148       break;
6149    }
6150    case nir_intrinsic_write_invocation_amd: {
6151       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6152       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6153       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6154       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6155       if (dst.regClass() == v1) {
6156          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6157          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6158       } else if (dst.regClass() == v2) {
6159          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6160          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6161          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6162          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6163          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6164          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6165          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6166          emit_split_vector(ctx, dst, 2);
6167       } else {
6168          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6169          nir_print_instr(&instr->instr, stderr);
6170          fprintf(stderr, "\n");
6171       }
6172       break;
6173    }
6174    case nir_intrinsic_mbcnt_amd: {
6175       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6176       RegClass rc = RegClass(src.type(), 1);
6177       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6178       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6179       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6180       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6181       emit_wqm(ctx, wqm_tmp, dst);
6182       break;
6183    }
6184    case nir_intrinsic_load_helper_invocation: {
6185       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6186       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6187       ctx->block->kind |= block_kind_needs_lowering;
6188       ctx->program->needs_exact = true;
6189       break;
6190    }
6191    case nir_intrinsic_is_helper_invocation: {
6192       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6193       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6194       ctx->block->kind |= block_kind_needs_lowering;
6195       ctx->program->needs_exact = true;
6196       break;
6197    }
6198    case nir_intrinsic_demote:
6199       bld.pseudo(aco_opcode::p_demote_to_helper);
6200       ctx->block->kind |= block_kind_uses_demote;
6201       ctx->program->needs_exact = true;
6202       break;
6203    case nir_intrinsic_demote_if: {
6204       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6205       assert(src.regClass() == bld.lm);
6206       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6207       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6208       ctx->block->kind |= block_kind_uses_demote;
6209       ctx->program->needs_exact = true;
6210       break;
6211    }
6212    case nir_intrinsic_first_invocation: {
6213       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6214                get_ssa_temp(ctx, &instr->dest.ssa));
6215       break;
6216    }
6217    case nir_intrinsic_shader_clock:
6218       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6219       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6220       break;
6221    case nir_intrinsic_load_vertex_id_zero_base: {
6222       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6223       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6224       break;
6225    }
6226    case nir_intrinsic_load_first_vertex: {
6227       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6228       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6229       break;
6230    }
6231    case nir_intrinsic_load_base_instance: {
6232       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6233       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6234       break;
6235    }
6236    case nir_intrinsic_load_instance_id: {
6237       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6238       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6239       break;
6240    }
6241    case nir_intrinsic_load_draw_id: {
6242       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6243       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6244       break;
6245    }
6246    default:
6247       fprintf(stderr, "Unimplemented intrinsic instr: ");
6248       nir_print_instr(&instr->instr, stderr);
6249       fprintf(stderr, "\n");
6250       abort();
6251
6252       break;
6253    }
6254 }
6255
6256
6257 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6258                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6259                     enum glsl_base_type *stype)
6260 {
6261    nir_deref_instr *texture_deref_instr = NULL;
6262    nir_deref_instr *sampler_deref_instr = NULL;
6263    int plane = -1;
6264
6265    for (unsigned i = 0; i < instr->num_srcs; i++) {
6266       switch (instr->src[i].src_type) {
6267       case nir_tex_src_texture_deref:
6268          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6269          break;
6270       case nir_tex_src_sampler_deref:
6271          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6272          break;
6273       case nir_tex_src_plane:
6274          plane = nir_src_as_int(instr->src[i].src);
6275          break;
6276       default:
6277          break;
6278       }
6279    }
6280
6281    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6282
6283    if (!sampler_deref_instr)
6284       sampler_deref_instr = texture_deref_instr;
6285
6286    if (plane >= 0) {
6287       assert(instr->op != nir_texop_txf_ms &&
6288              instr->op != nir_texop_samples_identical);
6289       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6290       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6291    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6292       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6293    } else {
6294       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6295    }
6296    if (samp_ptr) {
6297       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6298
6299       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6300          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6301          Builder bld(ctx->program, ctx->block);
6302
6303          /* to avoid unnecessary moves, we split and recombine sampler and image */
6304          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6305                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6306          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6307          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6308                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6309                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6310          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6311                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6312
6313          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6314          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6315                                img[0], img[1], img[2], img[3],
6316                                img[4], img[5], img[6], img[7]);
6317          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6318                                 samp[0], samp[1], samp[2], samp[3]);
6319       }
6320    }
6321    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6322                      instr->op == nir_texop_samples_identical))
6323       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6324 }
6325
6326 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6327                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6328 {
6329    Builder bld(ctx->program, ctx->block);
6330
6331    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6332    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6333    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6334
6335    Operand neg_one(0xbf800000u);
6336    Operand one(0x3f800000u);
6337    Operand two(0x40000000u);
6338    Operand four(0x40800000u);
6339
6340    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6341    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6342    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6343
6344    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6345    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6346    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6347    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6348
6349    // select sc
6350    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6351    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6352                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6353                        one, is_ma_y);
6354    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6355
6356    // select tc
6357    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6358    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6359    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6360
6361    // select ma
6362    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6363                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6364                   deriv_z, is_ma_z);
6365    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6366    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6367 }
6368
6369 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6370 {
6371    Builder bld(ctx->program, ctx->block);
6372    Temp coord_args[4], ma, tc, sc, id;
6373    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6374       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6375
6376    if (is_array) {
6377       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6378
6379       // see comment in ac_prepare_cube_coords()
6380       if (ctx->options->chip_class <= GFX8)
6381          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6382    }
6383
6384    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6385
6386    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6387    vop3a->operands[0] = Operand(ma);
6388    vop3a->abs[0] = true;
6389    Temp invma = bld.tmp(v1);
6390    vop3a->definitions[0] = Definition(invma);
6391    ctx->block->instructions.emplace_back(std::move(vop3a));
6392
6393    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6394    if (!is_deriv)
6395       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6396
6397    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6398    if (!is_deriv)
6399       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6400
6401    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6402
6403    if (is_deriv) {
6404       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6405       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6406
6407       for (unsigned i = 0; i < 2; i++) {
6408          // see comment in ac_prepare_cube_coords()
6409          Temp deriv_ma;
6410          Temp deriv_sc, deriv_tc;
6411          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6412                            &deriv_ma, &deriv_sc, &deriv_tc);
6413
6414          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6415
6416          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6417                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6418                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6419          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6420                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6421                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6422          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6423       }
6424
6425       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6426       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6427    }
6428
6429    if (is_array)
6430       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6431    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6432
6433 }
6434
6435 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6436 {
6437    Temp coord_vec[3];
6438    for (unsigned i = 0; i < coords.size(); i++)
6439       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6440
6441    Builder bld(ctx->program, ctx->block);
6442    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6443
6444    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6445    for (unsigned i = 0; i < coords.size(); i++)
6446       vec->operands[i] = Operand(coord_vec[i]);
6447    Temp res = bld.tmp(RegType::vgpr, coords.size());
6448    vec->definitions[0] = Definition(res);
6449    ctx->block->instructions.emplace_back(std::move(vec));
6450    return res;
6451 }
6452
6453 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6454 {
6455    if (vec->parent_instr->type != nir_instr_type_alu)
6456       return;
6457    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6458    if (vec_instr->op != nir_op_vec(vec->num_components))
6459       return;
6460
6461    for (unsigned i = 0; i < vec->num_components; i++) {
6462       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6463               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6464    }
6465 }
6466
6467 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6468 {
6469    Builder bld(ctx->program, ctx->block);
6470    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6471         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6472    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6473         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6474    nir_const_value *sample_index_cv = NULL;
6475    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6476    enum glsl_base_type stype;
6477    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6478
6479    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6480                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6481    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6482                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6483
6484    for (unsigned i = 0; i < instr->num_srcs; i++) {
6485       switch (instr->src[i].src_type) {
6486       case nir_tex_src_coord:
6487          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6488          break;
6489       case nir_tex_src_bias:
6490          if (instr->op == nir_texop_txb) {
6491             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6492             has_bias = true;
6493          }
6494          break;
6495       case nir_tex_src_lod: {
6496          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6497
6498          if (val && val->f32 <= 0.0) {
6499             level_zero = true;
6500          } else {
6501             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6502             has_lod = true;
6503          }
6504          break;
6505       }
6506       case nir_tex_src_comparator:
6507          if (instr->is_shadow) {
6508             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6509             has_compare = true;
6510          }
6511          break;
6512       case nir_tex_src_offset:
6513          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6514          get_const_vec(instr->src[i].src.ssa, const_offset);
6515          has_offset = true;
6516          break;
6517       case nir_tex_src_ddx:
6518          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6519          has_ddx = true;
6520          break;
6521       case nir_tex_src_ddy:
6522          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6523          has_ddy = true;
6524          break;
6525       case nir_tex_src_ms_index:
6526          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6527          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6528          has_sample_index = true;
6529          break;
6530       case nir_tex_src_texture_offset:
6531       case nir_tex_src_sampler_offset:
6532       default:
6533          break;
6534       }
6535    }
6536 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6537    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6538       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6539
6540    if (instr->op == nir_texop_texture_samples) {
6541       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6542
6543       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6544       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6545       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6546       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6547
6548       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6549                samples, Operand(1u), bld.scc(is_msaa));
6550       return;
6551    }
6552
6553    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6554       aco_ptr<Instruction> tmp_instr;
6555       Temp acc, pack = Temp();
6556
6557       uint32_t pack_const = 0;
6558       for (unsigned i = 0; i < offset.size(); i++) {
6559          if (!const_offset[i])
6560             continue;
6561          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6562       }
6563
6564       if (offset.type() == RegType::sgpr) {
6565          for (unsigned i = 0; i < offset.size(); i++) {
6566             if (const_offset[i])
6567                continue;
6568
6569             acc = emit_extract_vector(ctx, offset, i, s1);
6570             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6571
6572             if (i) {
6573                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6574             }
6575
6576             if (pack == Temp()) {
6577                pack = acc;
6578             } else {
6579                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6580             }
6581          }
6582
6583          if (pack_const && pack != Temp())
6584             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6585       } else {
6586          for (unsigned i = 0; i < offset.size(); i++) {
6587             if (const_offset[i])
6588                continue;
6589
6590             acc = emit_extract_vector(ctx, offset, i, v1);
6591             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6592
6593             if (i) {
6594                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6595             }
6596
6597             if (pack == Temp()) {
6598                pack = acc;
6599             } else {
6600                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6601             }
6602          }
6603
6604          if (pack_const && pack != Temp())
6605             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6606       }
6607       if (pack_const && pack == Temp())
6608          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6609       else if (pack == Temp())
6610          has_offset = false;
6611       else
6612          offset = pack;
6613    }
6614
6615    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6616       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6617
6618    /* pack derivatives */
6619    if (has_ddx || has_ddy) {
6620       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6621          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6622                              ddx, Operand(0u), ddy, Operand(0u));
6623       } else {
6624          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6625       }
6626       has_derivs = true;
6627    }
6628
6629    if (instr->coord_components > 1 &&
6630        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6631        instr->is_array &&
6632        instr->op != nir_texop_txf)
6633       coords = apply_round_slice(ctx, coords, 1);
6634
6635    if (instr->coord_components > 2 &&
6636       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6637        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6638        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6639        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6640        instr->is_array &&
6641        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6642       coords = apply_round_slice(ctx, coords, 2);
6643
6644    if (ctx->options->chip_class == GFX9 &&
6645        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6646        instr->op != nir_texop_lod && instr->coord_components) {
6647       assert(coords.size() > 0 && coords.size() < 3);
6648
6649       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6650       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6651       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6652       if (coords.size() > 1)
6653          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6654       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6655       vec->definitions[0] = Definition(coords);
6656       ctx->block->instructions.emplace_back(std::move(vec));
6657    }
6658
6659    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6660
6661    if (instr->op == nir_texop_samples_identical)
6662       resource = fmask_ptr;
6663
6664    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6665              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6666             instr->op != nir_texop_txs) {
6667       assert(has_sample_index);
6668       Operand op(sample_index);
6669       if (sample_index_cv)
6670          op = Operand(sample_index_cv->u32);
6671       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6672    }
6673
6674    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6675       Temp split_coords[coords.size()];
6676       emit_split_vector(ctx, coords, coords.size());
6677       for (unsigned i = 0; i < coords.size(); i++)
6678          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6679
6680       unsigned i = 0;
6681       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6682          Temp off = emit_extract_vector(ctx, offset, i, v1);
6683          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6684       }
6685
6686       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6687       for (unsigned i = 0; i < coords.size(); i++)
6688          vec->operands[i] = Operand(split_coords[i]);
6689       coords = bld.tmp(coords.regClass());
6690       vec->definitions[0] = Definition(coords);
6691       ctx->block->instructions.emplace_back(std::move(vec));
6692
6693       has_offset = false;
6694    }
6695
6696    /* Build tex instruction */
6697    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6698    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6699                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6700                   : 0;
6701    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6702    Temp tmp_dst = dst;
6703
6704    /* gather4 selects the component by dmask and always returns vec4 */
6705    if (instr->op == nir_texop_tg4) {
6706       assert(instr->dest.ssa.num_components == 4);
6707       if (instr->is_shadow)
6708          dmask = 1;
6709       else
6710          dmask = 1 << instr->component;
6711       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6712          tmp_dst = bld.tmp(v4);
6713    } else if (instr->op == nir_texop_samples_identical) {
6714       tmp_dst = bld.tmp(v1);
6715    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6716       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6717    }
6718
6719    aco_ptr<MIMG_instruction> tex;
6720    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6721       if (!has_lod)
6722          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6723
6724       bool div_by_6 = instr->op == nir_texop_txs &&
6725                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6726                       instr->is_array &&
6727                       (dmask & (1 << 2));
6728       if (tmp_dst.id() == dst.id() && div_by_6)
6729          tmp_dst = bld.tmp(tmp_dst.regClass());
6730
6731       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6732       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6733       tex->operands[1] = Operand(resource);
6734       if (ctx->options->chip_class == GFX9 &&
6735           instr->op == nir_texop_txs &&
6736           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6737           instr->is_array) {
6738          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6739       } else if (instr->op == nir_texop_query_levels) {
6740          tex->dmask = 1 << 3;
6741       } else {
6742          tex->dmask = dmask;
6743       }
6744       tex->da = da;
6745       tex->definitions[0] = Definition(tmp_dst);
6746       tex->dim = dim;
6747       tex->can_reorder = true;
6748       ctx->block->instructions.emplace_back(std::move(tex));
6749
6750       if (div_by_6) {
6751          /* divide 3rd value by 6 by multiplying with magic number */
6752          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6753          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6754          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6755          assert(instr->dest.ssa.num_components == 3);
6756          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6757          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6758                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6759                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6760                               by_6);
6761
6762       }
6763
6764       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6765       return;
6766    }
6767
6768    Temp tg4_compare_cube_wa64 = Temp();
6769
6770    if (tg4_integer_workarounds) {
6771       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6772       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6773       tex->operands[1] = Operand(resource);
6774       tex->dim = dim;
6775       tex->dmask = 0x3;
6776       tex->da = da;
6777       Temp size = bld.tmp(v2);
6778       tex->definitions[0] = Definition(size);
6779       tex->can_reorder = true;
6780       ctx->block->instructions.emplace_back(std::move(tex));
6781       emit_split_vector(ctx, size, size.size());
6782
6783       Temp half_texel[2];
6784       for (unsigned i = 0; i < 2; i++) {
6785          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6786          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6787          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6788          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6789       }
6790
6791       Temp orig_coords[2] = {
6792          emit_extract_vector(ctx, coords, 0, v1),
6793          emit_extract_vector(ctx, coords, 1, v1)};
6794       Temp new_coords[2] = {
6795          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6796          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6797       };
6798
6799       if (tg4_integer_cube_workaround) {
6800          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6801          Temp desc[resource.size()];
6802          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6803                                                                            Format::PSEUDO, 1, resource.size())};
6804          split->operands[0] = Operand(resource);
6805          for (unsigned i = 0; i < resource.size(); i++) {
6806             desc[i] = bld.tmp(s1);
6807             split->definitions[i] = Definition(desc[i]);
6808          }
6809          ctx->block->instructions.emplace_back(std::move(split));
6810
6811          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6812          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6813                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6814
6815          Temp nfmt;
6816          if (stype == GLSL_TYPE_UINT) {
6817             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6818                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6819                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6820                             bld.scc(compare_cube_wa));
6821          } else {
6822             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6823                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6824                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6825                             bld.scc(compare_cube_wa));
6826          }
6827          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6828          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6829
6830          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6831
6832          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6833                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6834          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6835
6836          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6837                                                                          Format::PSEUDO, resource.size(), 1)};
6838          for (unsigned i = 0; i < resource.size(); i++)
6839             vec->operands[i] = Operand(desc[i]);
6840          resource = bld.tmp(resource.regClass());
6841          vec->definitions[0] = Definition(resource);
6842          ctx->block->instructions.emplace_back(std::move(vec));
6843
6844          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6845                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6846          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6847                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6848       }
6849
6850       if (coords.size() == 3) {
6851          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6852                              new_coords[0], new_coords[1],
6853                              emit_extract_vector(ctx, coords, 2, v1));
6854       } else {
6855          assert(coords.size() == 2);
6856          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6857                              new_coords[0], new_coords[1]);
6858       }
6859    }
6860
6861    std::vector<Operand> args;
6862    if (has_offset)
6863       args.emplace_back(Operand(offset));
6864    if (has_bias)
6865       args.emplace_back(Operand(bias));
6866    if (has_compare)
6867       args.emplace_back(Operand(compare));
6868    if (has_derivs)
6869       args.emplace_back(Operand(derivs));
6870    args.emplace_back(Operand(coords));
6871    if (has_sample_index)
6872       args.emplace_back(Operand(sample_index));
6873    if (has_lod)
6874       args.emplace_back(lod);
6875
6876    Temp arg;
6877    if (args.size() > 1) {
6878       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6879       unsigned size = 0;
6880       for (unsigned i = 0; i < args.size(); i++) {
6881          size += args[i].size();
6882          vec->operands[i] = args[i];
6883       }
6884       RegClass rc = RegClass(RegType::vgpr, size);
6885       Temp tmp = bld.tmp(rc);
6886       vec->definitions[0] = Definition(tmp);
6887       ctx->block->instructions.emplace_back(std::move(vec));
6888       arg = tmp;
6889    } else {
6890       assert(args[0].isTemp());
6891       arg = as_vgpr(ctx, args[0].getTemp());
6892    }
6893
6894    /* we don't need the bias, sample index, compare value or offset to be
6895     * computed in WQM but if the p_create_vector copies the coordinates, then it
6896     * needs to be in WQM */
6897    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6898        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6899        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6900       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6901
6902    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6903       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6904
6905       assert(coords.size() == 1);
6906       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6907       aco_opcode op;
6908       switch (last_bit) {
6909       case 1:
6910          op = aco_opcode::buffer_load_format_x; break;
6911       case 2:
6912          op = aco_opcode::buffer_load_format_xy; break;
6913       case 3:
6914          op = aco_opcode::buffer_load_format_xyz; break;
6915       case 4:
6916          op = aco_opcode::buffer_load_format_xyzw; break;
6917       default:
6918          unreachable("Tex instruction loads more than 4 components.");
6919       }
6920
6921       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6922       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6923          tmp_dst = dst;
6924       else
6925          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6926
6927       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6928       mubuf->operands[0] = Operand(coords);
6929       mubuf->operands[1] = Operand(resource);
6930       mubuf->operands[2] = Operand((uint32_t) 0);
6931       mubuf->definitions[0] = Definition(tmp_dst);
6932       mubuf->idxen = true;
6933       mubuf->can_reorder = true;
6934       ctx->block->instructions.emplace_back(std::move(mubuf));
6935
6936       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6937       return;
6938    }
6939
6940
6941    if (instr->op == nir_texop_txf ||
6942        instr->op == nir_texop_txf_ms ||
6943        instr->op == nir_texop_samples_identical) {
6944       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6945       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6946       tex->operands[0] = Operand(arg);
6947       tex->operands[1] = Operand(resource);
6948       tex->dim = dim;
6949       tex->dmask = dmask;
6950       tex->unrm = true;
6951       tex->da = da;
6952       tex->definitions[0] = Definition(tmp_dst);
6953       tex->can_reorder = true;
6954       ctx->block->instructions.emplace_back(std::move(tex));
6955
6956       if (instr->op == nir_texop_samples_identical) {
6957          assert(dmask == 1 && dst.regClass() == v1);
6958          assert(dst.id() != tmp_dst.id());
6959
6960          Temp tmp = bld.tmp(bld.lm);
6961          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6962          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6963
6964       } else {
6965          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6966       }
6967       return;
6968    }
6969
6970    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6971    aco_opcode opcode = aco_opcode::image_sample;
6972    if (has_offset) { /* image_sample_*_o */
6973       if (has_compare) {
6974          opcode = aco_opcode::image_sample_c_o;
6975          if (has_derivs)
6976             opcode = aco_opcode::image_sample_c_d_o;
6977          if (has_bias)
6978             opcode = aco_opcode::image_sample_c_b_o;
6979          if (level_zero)
6980             opcode = aco_opcode::image_sample_c_lz_o;
6981          if (has_lod)
6982             opcode = aco_opcode::image_sample_c_l_o;
6983       } else {
6984          opcode = aco_opcode::image_sample_o;
6985          if (has_derivs)
6986             opcode = aco_opcode::image_sample_d_o;
6987          if (has_bias)
6988             opcode = aco_opcode::image_sample_b_o;
6989          if (level_zero)
6990             opcode = aco_opcode::image_sample_lz_o;
6991          if (has_lod)
6992             opcode = aco_opcode::image_sample_l_o;
6993       }
6994    } else { /* no offset */
6995       if (has_compare) {
6996          opcode = aco_opcode::image_sample_c;
6997          if (has_derivs)
6998             opcode = aco_opcode::image_sample_c_d;
6999          if (has_bias)
7000             opcode = aco_opcode::image_sample_c_b;
7001          if (level_zero)
7002             opcode = aco_opcode::image_sample_c_lz;
7003          if (has_lod)
7004             opcode = aco_opcode::image_sample_c_l;
7005       } else {
7006          opcode = aco_opcode::image_sample;
7007          if (has_derivs)
7008             opcode = aco_opcode::image_sample_d;
7009          if (has_bias)
7010             opcode = aco_opcode::image_sample_b;
7011          if (level_zero)
7012             opcode = aco_opcode::image_sample_lz;
7013          if (has_lod)
7014             opcode = aco_opcode::image_sample_l;
7015       }
7016    }
7017
7018    if (instr->op == nir_texop_tg4) {
7019       if (has_offset) {
7020          opcode = aco_opcode::image_gather4_lz_o;
7021          if (has_compare)
7022             opcode = aco_opcode::image_gather4_c_lz_o;
7023       } else {
7024          opcode = aco_opcode::image_gather4_lz;
7025          if (has_compare)
7026             opcode = aco_opcode::image_gather4_c_lz;
7027       }
7028    } else if (instr->op == nir_texop_lod) {
7029       opcode = aco_opcode::image_get_lod;
7030    }
7031
7032    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7033    tex->operands[0] = Operand(arg);
7034    tex->operands[1] = Operand(resource);
7035    tex->operands[2] = Operand(sampler);
7036    tex->dim = dim;
7037    tex->dmask = dmask;
7038    tex->da = da;
7039    tex->definitions[0] = Definition(tmp_dst);
7040    tex->can_reorder = true;
7041    ctx->block->instructions.emplace_back(std::move(tex));
7042
7043    if (tg4_integer_cube_workaround) {
7044       assert(tmp_dst.id() != dst.id());
7045       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7046
7047       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7048       Temp val[4];
7049       for (unsigned i = 0; i < dst.size(); i++) {
7050          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7051          Temp cvt_val;
7052          if (stype == GLSL_TYPE_UINT)
7053             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7054          else
7055             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7056          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7057       }
7058       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7059       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7060                            val[0], val[1], val[2], val[3]);
7061    }
7062    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7063    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7064
7065 }
7066
7067
7068 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7069 {
7070    Temp tmp = get_ssa_temp(ctx, ssa);
7071    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7072       return Operand(tmp.regClass());
7073    else
7074       return Operand(tmp);
7075 }
7076
7077 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7078 {
7079    aco_ptr<Pseudo_instruction> phi;
7080    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7081    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7082
7083    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7084    logical |= ctx->block->kind & block_kind_merge;
7085    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7086
7087    /* we want a sorted list of sources, since the predecessor list is also sorted */
7088    std::map<unsigned, nir_ssa_def*> phi_src;
7089    nir_foreach_phi_src(src, instr)
7090       phi_src[src->pred->index] = src->src.ssa;
7091
7092    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7093    unsigned num_operands = 0;
7094    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7095    unsigned num_defined = 0;
7096    unsigned cur_pred_idx = 0;
7097    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7098       if (cur_pred_idx < preds.size()) {
7099          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7100          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7101          unsigned skipped = 0;
7102          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7103             skipped++;
7104          if (cur_pred_idx + skipped < preds.size()) {
7105             for (unsigned i = 0; i < skipped; i++)
7106                operands[num_operands++] = Operand(dst.regClass());
7107             cur_pred_idx += skipped;
7108          } else {
7109             continue;
7110          }
7111       }
7112       cur_pred_idx++;
7113       Operand op = get_phi_operand(ctx, src.second);
7114       operands[num_operands++] = op;
7115       num_defined += !op.isUndefined();
7116    }
7117    /* handle block_kind_continue_or_break at loop exit blocks */
7118    while (cur_pred_idx++ < preds.size())
7119       operands[num_operands++] = Operand(dst.regClass());
7120
7121    if (num_defined == 0) {
7122       Builder bld(ctx->program, ctx->block);
7123       if (dst.regClass() == s1) {
7124          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7125       } else if (dst.regClass() == v1) {
7126          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7127       } else {
7128          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7129          for (unsigned i = 0; i < dst.size(); i++)
7130             vec->operands[i] = Operand(0u);
7131          vec->definitions[0] = Definition(dst);
7132          ctx->block->instructions.emplace_back(std::move(vec));
7133       }
7134       return;
7135    }
7136
7137    /* we can use a linear phi in some cases if one src is undef */
7138    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7139       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7140
7141       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7142       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7143       assert(invert->kind & block_kind_invert);
7144
7145       unsigned then_block = invert->linear_preds[0];
7146
7147       Block* insert_block = NULL;
7148       for (unsigned i = 0; i < num_operands; i++) {
7149          Operand op = operands[i];
7150          if (op.isUndefined())
7151             continue;
7152          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7153          phi->operands[0] = op;
7154          break;
7155       }
7156       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7157       phi->operands[1] = Operand(dst.regClass());
7158       phi->definitions[0] = Definition(dst);
7159       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7160       return;
7161    }
7162
7163    /* try to scalarize vector phis */
7164    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7165       // TODO: scalarize linear phis on divergent ifs
7166       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7167       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7168       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7169          Operand src = operands[i];
7170          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7171             can_scalarize = false;
7172       }
7173       if (can_scalarize) {
7174          unsigned num_components = instr->dest.ssa.num_components;
7175          assert(dst.size() % num_components == 0);
7176          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7177
7178          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7179          for (unsigned k = 0; k < num_components; k++) {
7180             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7181             for (unsigned i = 0; i < num_operands; i++) {
7182                Operand src = operands[i];
7183                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7184             }
7185             Temp phi_dst = {ctx->program->allocateId(), rc};
7186             phi->definitions[0] = Definition(phi_dst);
7187             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7188             new_vec[k] = phi_dst;
7189             vec->operands[k] = Operand(phi_dst);
7190          }
7191          vec->definitions[0] = Definition(dst);
7192          ctx->block->instructions.emplace_back(std::move(vec));
7193          ctx->allocated_vec.emplace(dst.id(), new_vec);
7194          return;
7195       }
7196    }
7197
7198    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7199    for (unsigned i = 0; i < num_operands; i++)
7200       phi->operands[i] = operands[i];
7201    phi->definitions[0] = Definition(dst);
7202    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7203 }
7204
7205
7206 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7207 {
7208    Temp dst = get_ssa_temp(ctx, &instr->def);
7209
7210    assert(dst.type() == RegType::sgpr);
7211
7212    if (dst.size() == 1) {
7213       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7214    } else {
7215       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7216       for (unsigned i = 0; i < dst.size(); i++)
7217          vec->operands[i] = Operand(0u);
7218       vec->definitions[0] = Definition(dst);
7219       ctx->block->instructions.emplace_back(std::move(vec));
7220    }
7221 }
7222
7223 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7224 {
7225    Builder bld(ctx->program, ctx->block);
7226    Block *logical_target;
7227    append_logical_end(ctx->block);
7228    unsigned idx = ctx->block->index;
7229
7230    switch (instr->type) {
7231    case nir_jump_break:
7232       logical_target = ctx->cf_info.parent_loop.exit;
7233       add_logical_edge(idx, logical_target);
7234       ctx->block->kind |= block_kind_break;
7235
7236       if (!ctx->cf_info.parent_if.is_divergent &&
7237           !ctx->cf_info.parent_loop.has_divergent_continue) {
7238          /* uniform break - directly jump out of the loop */
7239          ctx->block->kind |= block_kind_uniform;
7240          ctx->cf_info.has_branch = true;
7241          bld.branch(aco_opcode::p_branch);
7242          add_linear_edge(idx, logical_target);
7243          return;
7244       }
7245       ctx->cf_info.parent_loop.has_divergent_branch = true;
7246       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7247       break;
7248    case nir_jump_continue:
7249       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7250       add_logical_edge(idx, logical_target);
7251       ctx->block->kind |= block_kind_continue;
7252
7253       if (ctx->cf_info.parent_if.is_divergent) {
7254          /* for potential uniform breaks after this continue,
7255             we must ensure that they are handled correctly */
7256          ctx->cf_info.parent_loop.has_divergent_continue = true;
7257          ctx->cf_info.parent_loop.has_divergent_branch = true;
7258          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7259       } else {
7260          /* uniform continue - directly jump to the loop header */
7261          ctx->block->kind |= block_kind_uniform;
7262          ctx->cf_info.has_branch = true;
7263          bld.branch(aco_opcode::p_branch);
7264          add_linear_edge(idx, logical_target);
7265          return;
7266       }
7267       break;
7268    default:
7269       fprintf(stderr, "Unknown NIR jump instr: ");
7270       nir_print_instr(&instr->instr, stderr);
7271       fprintf(stderr, "\n");
7272       abort();
7273    }
7274
7275    /* remove critical edges from linear CFG */
7276    bld.branch(aco_opcode::p_branch);
7277    Block* break_block = ctx->program->create_and_insert_block();
7278    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7279    break_block->kind |= block_kind_uniform;
7280    add_linear_edge(idx, break_block);
7281    /* the loop_header pointer might be invalidated by this point */
7282    if (instr->type == nir_jump_continue)
7283       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7284    add_linear_edge(break_block->index, logical_target);
7285    bld.reset(break_block);
7286    bld.branch(aco_opcode::p_branch);
7287
7288    Block* continue_block = ctx->program->create_and_insert_block();
7289    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7290    add_linear_edge(idx, continue_block);
7291    append_logical_start(continue_block);
7292    ctx->block = continue_block;
7293    return;
7294 }
7295
7296 void visit_block(isel_context *ctx, nir_block *block)
7297 {
7298    nir_foreach_instr(instr, block) {
7299       switch (instr->type) {
7300       case nir_instr_type_alu:
7301          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7302          break;
7303       case nir_instr_type_load_const:
7304          visit_load_const(ctx, nir_instr_as_load_const(instr));
7305          break;
7306       case nir_instr_type_intrinsic:
7307          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7308          break;
7309       case nir_instr_type_tex:
7310          visit_tex(ctx, nir_instr_as_tex(instr));
7311          break;
7312       case nir_instr_type_phi:
7313          visit_phi(ctx, nir_instr_as_phi(instr));
7314          break;
7315       case nir_instr_type_ssa_undef:
7316          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7317          break;
7318       case nir_instr_type_deref:
7319          break;
7320       case nir_instr_type_jump:
7321          visit_jump(ctx, nir_instr_as_jump(instr));
7322          break;
7323       default:
7324          fprintf(stderr, "Unknown NIR instr type: ");
7325          nir_print_instr(instr, stderr);
7326          fprintf(stderr, "\n");
7327          //abort();
7328       }
7329    }
7330
7331    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7332       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7333 }
7334
7335
7336
7337 static void visit_loop(isel_context *ctx, nir_loop *loop)
7338 {
7339    append_logical_end(ctx->block);
7340    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7341    Builder bld(ctx->program, ctx->block);
7342    bld.branch(aco_opcode::p_branch);
7343    unsigned loop_preheader_idx = ctx->block->index;
7344
7345    Block loop_exit = Block();
7346    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7347    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7348
7349    Block* loop_header = ctx->program->create_and_insert_block();
7350    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7351    loop_header->kind |= block_kind_loop_header;
7352    add_edge(loop_preheader_idx, loop_header);
7353    ctx->block = loop_header;
7354
7355    /* emit loop body */
7356    unsigned loop_header_idx = loop_header->index;
7357    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7358    append_logical_start(ctx->block);
7359    visit_cf_list(ctx, &loop->body);
7360
7361    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7362    if (!ctx->cf_info.has_branch) {
7363       append_logical_end(ctx->block);
7364       if (ctx->cf_info.exec_potentially_empty) {
7365          /* Discards can result in code running with an empty exec mask.
7366           * This would result in divergent breaks not ever being taken. As a
7367           * workaround, break the loop when the loop mask is empty instead of
7368           * always continuing. */
7369          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7370          unsigned block_idx = ctx->block->index;
7371
7372          /* create helper blocks to avoid critical edges */
7373          Block *break_block = ctx->program->create_and_insert_block();
7374          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7375          break_block->kind = block_kind_uniform;
7376          bld.reset(break_block);
7377          bld.branch(aco_opcode::p_branch);
7378          add_linear_edge(block_idx, break_block);
7379          add_linear_edge(break_block->index, &loop_exit);
7380
7381          Block *continue_block = ctx->program->create_and_insert_block();
7382          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7383          continue_block->kind = block_kind_uniform;
7384          bld.reset(continue_block);
7385          bld.branch(aco_opcode::p_branch);
7386          add_linear_edge(block_idx, continue_block);
7387          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7388
7389          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7390          ctx->block = &ctx->program->blocks[block_idx];
7391       } else {
7392          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7393          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7394             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7395          else
7396             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7397       }
7398
7399       bld.reset(ctx->block);
7400       bld.branch(aco_opcode::p_branch);
7401    }
7402
7403    /* fixup phis in loop header from unreachable blocks */
7404    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7405       bool linear = ctx->cf_info.has_branch;
7406       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7407       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7408          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7409              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7410             /* the last operand should be the one that needs to be removed */
7411             instr->operands.pop_back();
7412          } else if (!is_phi(instr)) {
7413             break;
7414          }
7415       }
7416    }
7417
7418    ctx->cf_info.has_branch = false;
7419
7420    // TODO: if the loop has not a single exit, we must add one °°
7421    /* emit loop successor block */
7422    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7423    append_logical_start(ctx->block);
7424
7425    #if 0
7426    // TODO: check if it is beneficial to not branch on continues
7427    /* trim linear phis in loop header */
7428    for (auto&& instr : loop_entry->instructions) {
7429       if (instr->opcode == aco_opcode::p_linear_phi) {
7430          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7431          new_phi->definitions[0] = instr->definitions[0];
7432          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7433             new_phi->operands[i] = instr->operands[i];
7434          /* check that the remaining operands are all the same */
7435          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7436             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7437          instr.swap(new_phi);
7438       } else if (instr->opcode == aco_opcode::p_phi) {
7439          continue;
7440       } else {
7441          break;
7442       }
7443    }
7444    #endif
7445 }
7446
7447 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7448 {
7449    ic->cond = cond;
7450
7451    append_logical_end(ctx->block);
7452    ctx->block->kind |= block_kind_branch;
7453
7454    /* branch to linear then block */
7455    assert(cond.regClass() == ctx->program->lane_mask);
7456    aco_ptr<Pseudo_branch_instruction> branch;
7457    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7458    branch->operands[0] = Operand(cond);
7459    ctx->block->instructions.push_back(std::move(branch));
7460
7461    ic->BB_if_idx = ctx->block->index;
7462    ic->BB_invert = Block();
7463    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7464    /* Invert blocks are intentionally not marked as top level because they
7465     * are not part of the logical cfg. */
7466    ic->BB_invert.kind |= block_kind_invert;
7467    ic->BB_endif = Block();
7468    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7469    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7470
7471    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7472    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7473    ctx->cf_info.parent_if.is_divergent = true;
7474    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7475
7476    /** emit logical then block */
7477    Block* BB_then_logical = ctx->program->create_and_insert_block();
7478    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7479    add_edge(ic->BB_if_idx, BB_then_logical);
7480    ctx->block = BB_then_logical;
7481    append_logical_start(BB_then_logical);
7482 }
7483
7484 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7485 {
7486    Block *BB_then_logical = ctx->block;
7487    append_logical_end(BB_then_logical);
7488     /* branch from logical then block to invert block */
7489    aco_ptr<Pseudo_branch_instruction> branch;
7490    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7491    BB_then_logical->instructions.emplace_back(std::move(branch));
7492    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7493    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7494       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7495    BB_then_logical->kind |= block_kind_uniform;
7496    assert(!ctx->cf_info.has_branch);
7497    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7498    ctx->cf_info.parent_loop.has_divergent_branch = false;
7499
7500    /** emit linear then block */
7501    Block* BB_then_linear = ctx->program->create_and_insert_block();
7502    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7503    BB_then_linear->kind |= block_kind_uniform;
7504    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7505    /* branch from linear then block to invert block */
7506    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7507    BB_then_linear->instructions.emplace_back(std::move(branch));
7508    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7509
7510    /** emit invert merge block */
7511    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7512    ic->invert_idx = ctx->block->index;
7513
7514    /* branch to linear else block (skip else) */
7515    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7516    branch->operands[0] = Operand(ic->cond);
7517    ctx->block->instructions.push_back(std::move(branch));
7518
7519    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7520    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7521
7522    /** emit logical else block */
7523    Block* BB_else_logical = ctx->program->create_and_insert_block();
7524    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7525    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7526    add_linear_edge(ic->invert_idx, BB_else_logical);
7527    ctx->block = BB_else_logical;
7528    append_logical_start(BB_else_logical);
7529 }
7530
7531 static void end_divergent_if(isel_context *ctx, if_context *ic)
7532 {
7533    Block *BB_else_logical = ctx->block;
7534    append_logical_end(BB_else_logical);
7535
7536    /* branch from logical else block to endif block */
7537    aco_ptr<Pseudo_branch_instruction> branch;
7538    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7539    BB_else_logical->instructions.emplace_back(std::move(branch));
7540    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7541    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7542       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7543    BB_else_logical->kind |= block_kind_uniform;
7544
7545    assert(!ctx->cf_info.has_branch);
7546    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7547
7548
7549    /** emit linear else block */
7550    Block* BB_else_linear = ctx->program->create_and_insert_block();
7551    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7552    BB_else_linear->kind |= block_kind_uniform;
7553    add_linear_edge(ic->invert_idx, BB_else_linear);
7554
7555    /* branch from linear else block to endif block */
7556    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7557    BB_else_linear->instructions.emplace_back(std::move(branch));
7558    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7559
7560
7561    /** emit endif merge block */
7562    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7563    append_logical_start(ctx->block);
7564
7565
7566    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7567    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7568    /* uniform control flow never has an empty exec-mask */
7569    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7570       ctx->cf_info.exec_potentially_empty = false;
7571 }
7572
7573 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7574 {
7575    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7576    Builder bld(ctx->program, ctx->block);
7577    aco_ptr<Pseudo_branch_instruction> branch;
7578
7579    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7580       /**
7581        * Uniform conditionals are represented in the following way*) :
7582        *
7583        * The linear and logical CFG:
7584        *                        BB_IF
7585        *                        /    \
7586        *       BB_THEN (logical)      BB_ELSE (logical)
7587        *                        \    /
7588        *                        BB_ENDIF
7589        *
7590        * *) Exceptions may be due to break and continue statements within loops
7591        *    If a break/continue happens within uniform control flow, it branches
7592        *    to the loop exit/entry block. Otherwise, it branches to the next
7593        *    merge block.
7594        **/
7595       append_logical_end(ctx->block);
7596       ctx->block->kind |= block_kind_uniform;
7597
7598       /* emit branch */
7599       assert(cond.regClass() == bld.lm);
7600       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7601       cond = bool_to_scalar_condition(ctx, cond);
7602
7603       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7604       branch->operands[0] = Operand(cond);
7605       branch->operands[0].setFixed(scc);
7606       ctx->block->instructions.emplace_back(std::move(branch));
7607
7608       unsigned BB_if_idx = ctx->block->index;
7609       Block BB_endif = Block();
7610       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7611       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7612
7613       /** emit then block */
7614       Block* BB_then = ctx->program->create_and_insert_block();
7615       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7616       add_edge(BB_if_idx, BB_then);
7617       append_logical_start(BB_then);
7618       ctx->block = BB_then;
7619       visit_cf_list(ctx, &if_stmt->then_list);
7620       BB_then = ctx->block;
7621       bool then_branch = ctx->cf_info.has_branch;
7622       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7623
7624       if (!then_branch) {
7625          append_logical_end(BB_then);
7626          /* branch from then block to endif block */
7627          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7628          BB_then->instructions.emplace_back(std::move(branch));
7629          add_linear_edge(BB_then->index, &BB_endif);
7630          if (!then_branch_divergent)
7631             add_logical_edge(BB_then->index, &BB_endif);
7632          BB_then->kind |= block_kind_uniform;
7633       }
7634
7635       ctx->cf_info.has_branch = false;
7636       ctx->cf_info.parent_loop.has_divergent_branch = false;
7637
7638       /** emit else block */
7639       Block* BB_else = ctx->program->create_and_insert_block();
7640       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7641       add_edge(BB_if_idx, BB_else);
7642       append_logical_start(BB_else);
7643       ctx->block = BB_else;
7644       visit_cf_list(ctx, &if_stmt->else_list);
7645       BB_else = ctx->block;
7646
7647       if (!ctx->cf_info.has_branch) {
7648          append_logical_end(BB_else);
7649          /* branch from then block to endif block */
7650          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7651          BB_else->instructions.emplace_back(std::move(branch));
7652          add_linear_edge(BB_else->index, &BB_endif);
7653          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7654             add_logical_edge(BB_else->index, &BB_endif);
7655          BB_else->kind |= block_kind_uniform;
7656       }
7657
7658       ctx->cf_info.has_branch &= then_branch;
7659       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7660
7661       /** emit endif merge block */
7662       if (!ctx->cf_info.has_branch) {
7663          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7664          append_logical_start(ctx->block);
7665       }
7666    } else { /* non-uniform condition */
7667       /**
7668        * To maintain a logical and linear CFG without critical edges,
7669        * non-uniform conditionals are represented in the following way*) :
7670        *
7671        * The linear CFG:
7672        *                        BB_IF
7673        *                        /    \
7674        *       BB_THEN (logical)      BB_THEN (linear)
7675        *                        \    /
7676        *                        BB_INVERT (linear)
7677        *                        /    \
7678        *       BB_ELSE (logical)      BB_ELSE (linear)
7679        *                        \    /
7680        *                        BB_ENDIF
7681        *
7682        * The logical CFG:
7683        *                        BB_IF
7684        *                        /    \
7685        *       BB_THEN (logical)      BB_ELSE (logical)
7686        *                        \    /
7687        *                        BB_ENDIF
7688        *
7689        * *) Exceptions may be due to break and continue statements within loops
7690        **/
7691
7692       if_context ic;
7693
7694       begin_divergent_if_then(ctx, &ic, cond);
7695       visit_cf_list(ctx, &if_stmt->then_list);
7696
7697       begin_divergent_if_else(ctx, &ic);
7698       visit_cf_list(ctx, &if_stmt->else_list);
7699
7700       end_divergent_if(ctx, &ic);
7701    }
7702 }
7703
7704 static void visit_cf_list(isel_context *ctx,
7705                           struct exec_list *list)
7706 {
7707    foreach_list_typed(nir_cf_node, node, node, list) {
7708       switch (node->type) {
7709       case nir_cf_node_block:
7710          visit_block(ctx, nir_cf_node_as_block(node));
7711          break;
7712       case nir_cf_node_if:
7713          visit_if(ctx, nir_cf_node_as_if(node));
7714          break;
7715       case nir_cf_node_loop:
7716          visit_loop(ctx, nir_cf_node_as_loop(node));
7717          break;
7718       default:
7719          unreachable("unimplemented cf list type");
7720       }
7721    }
7722 }
7723
7724 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7725 {
7726    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7727    uint64_t mask = ctx->vs_output.mask[slot];
7728    if (!is_pos && !mask)
7729       return;
7730    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7731       return;
7732    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7733    exp->enabled_mask = mask;
7734    for (unsigned i = 0; i < 4; ++i) {
7735       if (mask & (1 << i))
7736          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7737       else
7738          exp->operands[i] = Operand(v1);
7739    }
7740    exp->valid_mask = false;
7741    exp->done = false;
7742    exp->compressed = false;
7743    if (is_pos)
7744       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7745    else
7746       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7747    ctx->block->instructions.emplace_back(std::move(exp));
7748 }
7749
7750 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7751 {
7752    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7753    exp->enabled_mask = 0;
7754    for (unsigned i = 0; i < 4; ++i)
7755       exp->operands[i] = Operand(v1);
7756    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7757       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7758       exp->enabled_mask |= 0x1;
7759    }
7760    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7761       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7762       exp->enabled_mask |= 0x4;
7763    }
7764    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7765       if (ctx->options->chip_class < GFX9) {
7766          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7767          exp->enabled_mask |= 0x8;
7768       } else {
7769          Builder bld(ctx->program, ctx->block);
7770
7771          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7772                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7773          if (exp->operands[2].isTemp())
7774             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7775
7776          exp->operands[2] = Operand(out);
7777          exp->enabled_mask |= 0x4;
7778       }
7779    }
7780    exp->valid_mask = false;
7781    exp->done = false;
7782    exp->compressed = false;
7783    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7784    ctx->block->instructions.emplace_back(std::move(exp));
7785 }
7786
7787 static void create_vs_exports(isel_context *ctx)
7788 {
7789    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7790
7791    if (outinfo->export_prim_id) {
7792       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7793       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7794    }
7795
7796    if (ctx->options->key.has_multiview_view_index) {
7797       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7798       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7799    }
7800
7801    /* the order these position exports are created is important */
7802    int next_pos = 0;
7803    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7804    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7805       export_vs_psiz_layer_viewport(ctx, &next_pos);
7806    }
7807    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7808       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7809    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7810       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7811
7812    if (ctx->options->key.vs_common_out.export_clip_dists) {
7813       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7814          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7815       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7816          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7817    }
7818
7819    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7820       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7821           i != VARYING_SLOT_PRIMITIVE_ID)
7822          continue;
7823
7824       export_vs_varying(ctx, i, false, NULL);
7825    }
7826 }
7827
7828 static void emit_stream_output(isel_context *ctx,
7829                                Temp const *so_buffers,
7830                                Temp const *so_write_offset,
7831                                const struct radv_stream_output *output)
7832 {
7833    unsigned num_comps = util_bitcount(output->component_mask);
7834    unsigned loc = output->location;
7835    unsigned buf = output->buffer;
7836    unsigned offset = output->offset;
7837
7838    assert(num_comps && num_comps <= 4);
7839    if (!num_comps || num_comps > 4)
7840       return;
7841
7842    unsigned start = ffs(output->component_mask) - 1;
7843
7844    Temp out[4];
7845    bool all_undef = true;
7846    assert(ctx->stage == vertex_vs);
7847    for (unsigned i = 0; i < num_comps; i++) {
7848       out[i] = ctx->vs_output.outputs[loc][start + i];
7849       all_undef = all_undef && !out[i].id();
7850    }
7851    if (all_undef)
7852       return;
7853
7854    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7855    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7856    for (unsigned i = 0; i < num_comps; ++i)
7857       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7858    vec->definitions[0] = Definition(write_data);
7859    ctx->block->instructions.emplace_back(std::move(vec));
7860
7861    aco_opcode opcode;
7862    switch (num_comps) {
7863    case 1:
7864       opcode = aco_opcode::buffer_store_dword;
7865       break;
7866    case 2:
7867       opcode = aco_opcode::buffer_store_dwordx2;
7868       break;
7869    case 3:
7870       opcode = aco_opcode::buffer_store_dwordx3;
7871       break;
7872    case 4:
7873       opcode = aco_opcode::buffer_store_dwordx4;
7874       break;
7875    }
7876
7877    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7878    store->operands[0] = Operand(so_write_offset[buf]);
7879    store->operands[1] = Operand(so_buffers[buf]);
7880    store->operands[2] = Operand((uint32_t) 0);
7881    store->operands[3] = Operand(write_data);
7882    if (offset > 4095) {
7883       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7884       Builder bld(ctx->program, ctx->block);
7885       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7886    } else {
7887       store->offset = offset;
7888    }
7889    store->offen = true;
7890    store->glc = true;
7891    store->dlc = false;
7892    store->slc = true;
7893    store->can_reorder = true;
7894    ctx->block->instructions.emplace_back(std::move(store));
7895 }
7896
7897 static void emit_streamout(isel_context *ctx, unsigned stream)
7898 {
7899    Builder bld(ctx->program, ctx->block);
7900
7901    Temp so_buffers[4];
7902    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7903    for (unsigned i = 0; i < 4; i++) {
7904       unsigned stride = ctx->program->info->so.strides[i];
7905       if (!stride)
7906          continue;
7907
7908       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7909    }
7910
7911    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7912                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7913
7914    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7915
7916    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7917
7918    if_context ic;
7919    begin_divergent_if_then(ctx, &ic, can_emit);
7920
7921    bld.reset(ctx->block);
7922
7923    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7924
7925    Temp so_write_offset[4];
7926
7927    for (unsigned i = 0; i < 4; i++) {
7928       unsigned stride = ctx->program->info->so.strides[i];
7929       if (!stride)
7930          continue;
7931
7932       if (stride == 1) {
7933          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7934                                 get_arg(ctx, ctx->args->streamout_write_idx),
7935                                 get_arg(ctx, ctx->args->streamout_offset[i]));
7936          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7937
7938          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7939       } else {
7940          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7941          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
7942                                  get_arg(ctx, ctx->args->streamout_offset[i]));
7943          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7944       }
7945    }
7946
7947    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7948       struct radv_stream_output *output =
7949          &ctx->program->info->so.outputs[i];
7950       if (stream != output->stream)
7951          continue;
7952
7953       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7954    }
7955
7956    begin_divergent_if_else(ctx, &ic);
7957    end_divergent_if(ctx, &ic);
7958 }
7959
7960 } /* end namespace */
7961
7962 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
7963 {
7964    /* Split all arguments except for the first (ring_offsets) and the last
7965     * (exec) so that the dead channels don't stay live throughout the program.
7966     */
7967    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
7968       if (startpgm->definitions[i].regClass().size() > 1) {
7969          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
7970                            startpgm->definitions[i].regClass().size());
7971       }
7972    }
7973 }
7974
7975 void handle_bc_optimize(isel_context *ctx)
7976 {
7977    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7978    Builder bld(ctx->program, ctx->block);
7979    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7980    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7981    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7982    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
7983    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
7984    if (uses_center && uses_centroid) {
7985       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
7986                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
7987
7988       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7989          Temp new_coord[2];
7990          for (unsigned i = 0; i < 2; i++) {
7991             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
7992             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
7993             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7994                                     persp_centroid, persp_center, sel);
7995          }
7996          ctx->persp_centroid = bld.tmp(v2);
7997          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
7998                     Operand(new_coord[0]), Operand(new_coord[1]));
7999          emit_split_vector(ctx, ctx->persp_centroid, 2);
8000       }
8001
8002       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8003          Temp new_coord[2];
8004          for (unsigned i = 0; i < 2; i++) {
8005             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8006             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8007             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8008                                     linear_centroid, linear_center, sel);
8009          }
8010          ctx->linear_centroid = bld.tmp(v2);
8011          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8012                     Operand(new_coord[0]), Operand(new_coord[1]));
8013          emit_split_vector(ctx, ctx->linear_centroid, 2);
8014       }
8015    }
8016 }
8017
8018 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8019 {
8020    Program *program = ctx->program;
8021
8022    unsigned float_controls = shader->info.float_controls_execution_mode;
8023
8024    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8025       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8026    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8027       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8028                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8029
8030    program->next_fp_mode.must_flush_denorms32 =
8031       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8032    program->next_fp_mode.must_flush_denorms16_64 =
8033       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8034                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8035
8036    program->next_fp_mode.care_about_round32 =
8037       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8038
8039    program->next_fp_mode.care_about_round16_64 =
8040       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8041                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8042
8043    /* default to preserving fp16 and fp64 denorms, since it's free */
8044    if (program->next_fp_mode.must_flush_denorms16_64)
8045       program->next_fp_mode.denorm16_64 = 0;
8046    else
8047       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8048
8049    /* preserving fp32 denorms is expensive, so only do it if asked */
8050    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8051       program->next_fp_mode.denorm32 = fp_denorm_keep;
8052    else
8053       program->next_fp_mode.denorm32 = 0;
8054
8055    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8056       program->next_fp_mode.round32 = fp_round_tz;
8057    else
8058       program->next_fp_mode.round32 = fp_round_ne;
8059
8060    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8061       program->next_fp_mode.round16_64 = fp_round_tz;
8062    else
8063       program->next_fp_mode.round16_64 = fp_round_ne;
8064
8065    ctx->block->fp_mode = program->next_fp_mode;
8066 }
8067
8068 void select_program(Program *program,
8069                     unsigned shader_count,
8070                     struct nir_shader *const *shaders,
8071                     ac_shader_config* config,
8072                     struct radv_shader_args *args)
8073 {
8074    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
8075
8076    for (unsigned i = 0; i < shader_count; i++) {
8077       nir_shader *nir = shaders[i];
8078       init_context(&ctx, nir);
8079
8080       setup_fp_mode(&ctx, nir);
8081
8082       if (!i) {
8083          /* needs to be after init_context() for FS */
8084          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8085          append_logical_start(ctx.block);
8086          split_arguments(&ctx, startpgm);
8087       }
8088
8089       if_context ic;
8090       if (shader_count >= 2) {
8091          Builder bld(ctx.program, ctx.block);
8092          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
8093          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8094          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8095
8096          begin_divergent_if_then(&ctx, &ic, cond);
8097       }
8098
8099       if (i) {
8100          Builder bld(ctx.program, ctx.block);
8101          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
8102          bld.sopp(aco_opcode::s_barrier);
8103       }
8104
8105       if (ctx.stage == fragment_fs)
8106          handle_bc_optimize(&ctx);
8107
8108       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8109       visit_cf_list(&ctx, &func->body);
8110
8111       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8112          emit_streamout(&ctx, 0);
8113
8114       if (ctx.stage == vertex_vs)
8115          create_vs_exports(&ctx);
8116
8117       if (shader_count >= 2) {
8118          begin_divergent_if_else(&ctx, &ic);
8119          end_divergent_if(&ctx, &ic);
8120       }
8121
8122       ralloc_free(ctx.divergent_vals);
8123    }
8124
8125    program->config->float_mode = program->blocks[0].fp_mode.val;
8126
8127    append_logical_end(ctx.block);
8128    ctx.block->kind |= block_kind_uniform;
8129    Builder bld(ctx.program, ctx.block);
8130    if (ctx.program->wb_smem_l1_on_end)
8131       bld.smem(aco_opcode::s_dcache_wb, false);
8132    bld.sopp(aco_opcode::s_endpgm);
8133
8134    /* cleanup CFG */
8135    for (Block& BB : program->blocks) {
8136       for (unsigned idx : BB.linear_preds)
8137          program->blocks[idx].linear_succs.emplace_back(BB.index);
8138       for (unsigned idx : BB.logical_preds)
8139          program->blocks[idx].logical_succs.emplace_back(BB.index);
8140    }
8141 }
8142 }