src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,4> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,4> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,4> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 439 {
 440    Builder bld(ctx->program, ctx->block);
 441    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 442    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 443    if (src1.type() == RegType::sgpr) {
 444       if (commutative && src0.type() == RegType::vgpr) {
 445          Temp t = src0;
 446          src0 = src1;
 447          src1 = t;
 448       } else if (src0.type() == RegType::vgpr &&
 449                  op != aco_opcode::v_madmk_f32 &&
 450                  op != aco_opcode::v_madak_f32 &&
 451                  op != aco_opcode::v_madmk_f16 &&
 452                  op != aco_opcode::v_madak_f16) {
 453          /* If the instruction is not commutative, we emit a VOP3A instruction */
 454          bld.vop2_e64(op, Definition(dst), src0, src1);
 455          return;
 456       } else {
 457          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 458       }
 459    }
 460    bld.vop2(op, Definition(dst), src0, src1);
 461 }
 462
 463 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 464 {
 465    Temp src0 = get_alu_src(ctx, instr->src[0]);
 466    Temp src1 = get_alu_src(ctx, instr->src[1]);
 467    Temp src2 = get_alu_src(ctx, instr->src[2]);
 468
 469    /* ensure that the instruction has at most 1 sgpr operand
 470     * The optimizer will inline constants for us */
 471    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 472       src0 = as_vgpr(ctx, src0);
 473    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 474       src1 = as_vgpr(ctx, src1);
 475    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 476       src2 = as_vgpr(ctx, src2);
 477
 478    Builder bld(ctx->program, ctx->block);
 479    bld.vop3(op, Definition(dst), src0, src1, src2);
 480 }
 481
 482 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 483 {
 484    Builder bld(ctx->program, ctx->block);
 485    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 486 }
 487
 488 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 489 {
 490    Temp src0 = get_alu_src(ctx, instr->src[0]);
 491    Temp src1 = get_alu_src(ctx, instr->src[1]);
 492    assert(src0.size() == src1.size());
 493
 494    aco_ptr<Instruction> vopc;
 495    if (src1.type() == RegType::sgpr) {
 496       if (src0.type() == RegType::vgpr) {
 497          /* to swap the operands, we might also have to change the opcode */
 498          switch (op) {
 499             case aco_opcode::v_cmp_lt_f32:
 500                op = aco_opcode::v_cmp_gt_f32;
 501                break;
 502             case aco_opcode::v_cmp_ge_f32:
 503                op = aco_opcode::v_cmp_le_f32;
 504                break;
 505             case aco_opcode::v_cmp_lt_i32:
 506                op = aco_opcode::v_cmp_gt_i32;
 507                break;
 508             case aco_opcode::v_cmp_ge_i32:
 509                op = aco_opcode::v_cmp_le_i32;
 510                break;
 511             case aco_opcode::v_cmp_lt_u32:
 512                op = aco_opcode::v_cmp_gt_u32;
 513                break;
 514             case aco_opcode::v_cmp_ge_u32:
 515                op = aco_opcode::v_cmp_le_u32;
 516                break;
 517             case aco_opcode::v_cmp_lt_f64:
 518                op = aco_opcode::v_cmp_gt_f64;
 519                break;
 520             case aco_opcode::v_cmp_ge_f64:
 521                op = aco_opcode::v_cmp_le_f64;
 522                break;
 523             case aco_opcode::v_cmp_lt_i64:
 524                op = aco_opcode::v_cmp_gt_i64;
 525                break;
 526             case aco_opcode::v_cmp_ge_i64:
 527                op = aco_opcode::v_cmp_le_i64;
 528                break;
 529             case aco_opcode::v_cmp_lt_u64:
 530                op = aco_opcode::v_cmp_gt_u64;
 531                break;
 532             case aco_opcode::v_cmp_ge_u64:
 533                op = aco_opcode::v_cmp_le_u64;
 534                break;
 535             default: /* eq and ne are commutative */
 536                break;
 537          }
 538          Temp t = src0;
 539          src0 = src1;
 540          src1 = t;
 541       } else {
 542          src1 = as_vgpr(ctx, src1);
 543       }
 544    }
 545
 546    Builder bld(ctx->program, ctx->block);
 547    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 548 }
 549
 550 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 551 {
 552    Temp src0 = get_alu_src(ctx, instr->src[0]);
 553    Temp src1 = get_alu_src(ctx, instr->src[1]);
 554    Builder bld(ctx->program, ctx->block);
 555
 556    assert(dst.regClass() == bld.lm);
 557    assert(src0.type() == RegType::sgpr);
 558    assert(src1.type() == RegType::sgpr);
 559    assert(src0.regClass() == src1.regClass());
 560
 561    /* Emit the SALU comparison instruction */
 562    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 563    /* Turn the result into a per-lane bool */
 564    bool_to_vector_condition(ctx, cmp, dst);
 565 }
 566
 567 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 568                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::last_opcode, aco_opcode s64_op = aco_opcode::last_opcode)
 569 {
 570    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 571    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 572    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 573    bool use_valu = s_op == aco_opcode::last_opcode ||
 574                    divergent_vals ||
 575                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 576                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 577    aco_opcode op = use_valu ? v_op : s_op;
 578    assert(op != aco_opcode::last_opcode);
 579
 580    if (use_valu)
 581       emit_vopc_instruction(ctx, instr, op, dst);
 582    else
 583       emit_sopc_instruction(ctx, instr, op, dst);
 584 }
 585
 586 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 587 {
 588    Builder bld(ctx->program, ctx->block);
 589    Temp src0 = get_alu_src(ctx, instr->src[0]);
 590    Temp src1 = get_alu_src(ctx, instr->src[1]);
 591
 592    assert(dst.regClass() == bld.lm);
 593    assert(src0.regClass() == bld.lm);
 594    assert(src1.regClass() == bld.lm);
 595
 596    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 597 }
 598
 599 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 600 {
 601    Builder bld(ctx->program, ctx->block);
 602    Temp cond = get_alu_src(ctx, instr->src[0]);
 603    Temp then = get_alu_src(ctx, instr->src[1]);
 604    Temp els = get_alu_src(ctx, instr->src[2]);
 605
 606    assert(cond.regClass() == bld.lm);
 607
 608    if (dst.type() == RegType::vgpr) {
 609       aco_ptr<Instruction> bcsel;
 610       if (dst.size() == 1) {
 611          then = as_vgpr(ctx, then);
 612          els = as_vgpr(ctx, els);
 613
 614          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 615       } else if (dst.size() == 2) {
 616          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 617          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 618          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 619          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 620
 621          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 622          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 623
 624          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 625       } else {
 626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 627          nir_print_instr(&instr->instr, stderr);
 628          fprintf(stderr, "\n");
 629       }
 630       return;
 631    }
 632
 633    if (instr->dest.dest.ssa.bit_size == 1) {
 634       assert(dst.regClass() == bld.lm);
 635       assert(then.regClass() == bld.lm);
 636       assert(els.regClass() == bld.lm);
 637    }
 638
 639    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 640       if (dst.regClass() == s1 || dst.regClass() == s2) {
 641          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 642          assert(dst.size() == then.size());
 643          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 644          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 645       } else {
 646          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 647          nir_print_instr(&instr->instr, stderr);
 648          fprintf(stderr, "\n");
 649       }
 650       return;
 651    }
 652
 653    /* divergent boolean bcsel
 654     * this implements bcsel on bools: dst = s0 ? s1 : s2
 655     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 656    assert(instr->dest.dest.ssa.bit_size == 1);
 657
 658    if (cond.id() != then.id())
 659       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 660
 661    if (cond.id() == els.id())
 662       bld.sop1(Builder::s_mov, Definition(dst), then);
 663    else
 664       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 665                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 666 }
 667
 668 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 669                     aco_opcode op, uint32_t undo)
 670 {
 671    /* multiply by 16777216 to handle denormals */
 672    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 673                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 674    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 675    scaled = bld.vop1(op, bld.def(v1), scaled);
 676    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 677
 678    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 679
 680    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 681 }
 682
 683 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 684 {
 685    if (ctx->block->fp_mode.denorm32 == 0) {
 686       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 687       return;
 688    }
 689
 690    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 691 }
 692
 693 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 694 {
 695    if (ctx->block->fp_mode.denorm32 == 0) {
 696       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 697       return;
 698    }
 699
 700    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 701 }
 702
 703 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 704 {
 705    if (ctx->block->fp_mode.denorm32 == 0) {
 706       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 707       return;
 708    }
 709
 710    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 711 }
 712
 713 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 714 {
 715    if (ctx->block->fp_mode.denorm32 == 0) {
 716       bld.vop1(aco_opcode::v_log_f32, dst, val);
 717       return;
 718    }
 719
 720    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 721 }
 722
 723 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 724 {
 725    if (!instr->dest.dest.is_ssa) {
 726       fprintf(stderr, "nir alu dst not in ssa: ");
 727       nir_print_instr(&instr->instr, stderr);
 728       fprintf(stderr, "\n");
 729       abort();
 730    }
 731    Builder bld(ctx->program, ctx->block);
 732    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 733    switch(instr->op) {
 734    case nir_op_vec2:
 735    case nir_op_vec3:
 736    case nir_op_vec4: {
 737       std::array<Temp,4> elems;
 738       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 739       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 740          elems[i] = get_alu_src(ctx, instr->src[i]);
 741          vec->operands[i] = Operand{elems[i]};
 742       }
 743       vec->definitions[0] = Definition(dst);
 744       ctx->block->instructions.emplace_back(std::move(vec));
 745       ctx->allocated_vec.emplace(dst.id(), elems);
 746       break;
 747    }
 748    case nir_op_mov: {
 749       Temp src = get_alu_src(ctx, instr->src[0]);
 750       aco_ptr<Instruction> mov;
 751       if (dst.type() == RegType::sgpr) {
 752          if (src.type() == RegType::vgpr)
 753             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 754          else if (src.regClass() == s1)
 755             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 756          else if (src.regClass() == s2)
 757             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 758          else
 759             unreachable("wrong src register class for nir_op_imov");
 760       } else if (dst.regClass() == v1) {
 761          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 762       } else if (dst.regClass() == v2) {
 763          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 764       } else {
 765          nir_print_instr(&instr->instr, stderr);
 766          unreachable("Should have been lowered to scalar.");
 767       }
 768       break;
 769    }
 770    case nir_op_inot: {
 771       Temp src = get_alu_src(ctx, instr->src[0]);
 772       if (instr->dest.dest.ssa.bit_size == 1) {
 773          assert(src.regClass() == bld.lm);
 774          assert(dst.regClass() == bld.lm);
 775          bld.sop2(Builder::s_andn2, Definition(dst), bld.def(s1, scc), Operand(exec, bld.lm), src);
 776       } else if (dst.regClass() == v1) {
 777          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 778       } else if (dst.type() == RegType::sgpr) {
 779          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 780          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 781       } else {
 782          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 783          nir_print_instr(&instr->instr, stderr);
 784          fprintf(stderr, "\n");
 785       }
 786       break;
 787    }
 788    case nir_op_ineg: {
 789       Temp src = get_alu_src(ctx, instr->src[0]);
 790       if (dst.regClass() == v1) {
 791          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 792       } else if (dst.regClass() == s1) {
 793          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 794       } else if (dst.size() == 2) {
 795          Temp src0 = bld.tmp(dst.type(), 1);
 796          Temp src1 = bld.tmp(dst.type(), 1);
 797          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 798
 799          if (dst.regClass() == s2) {
 800             Temp carry = bld.tmp(s1);
 801             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 802             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 804          } else {
 805             Temp lower = bld.tmp(v1);
 806             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 807             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 808             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 809          }
 810       } else {
 811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 812          nir_print_instr(&instr->instr, stderr);
 813          fprintf(stderr, "\n");
 814       }
 815       break;
 816    }
 817    case nir_op_iabs: {
 818       if (dst.regClass() == s1) {
 819          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 820       } else if (dst.regClass() == v1) {
 821          Temp src = get_alu_src(ctx, instr->src[0]);
 822          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 823       } else {
 824          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 825          nir_print_instr(&instr->instr, stderr);
 826          fprintf(stderr, "\n");
 827       }
 828       break;
 829    }
 830    case nir_op_isign: {
 831       Temp src = get_alu_src(ctx, instr->src[0]);
 832       if (dst.regClass() == s1) {
 833          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 834          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 835          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 836       } else if (dst.regClass() == s2) {
 837          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 838          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 839          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 840       } else if (dst.regClass() == v1) {
 841          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 842          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 843          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 844       } else if (dst.regClass() == v2) {
 845          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 846          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 847          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 848          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 849          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 850          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 851       } else {
 852          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 853          nir_print_instr(&instr->instr, stderr);
 854          fprintf(stderr, "\n");
 855       }
 856       break;
 857    }
 858    case nir_op_imax: {
 859       if (dst.regClass() == v1) {
 860          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 861       } else if (dst.regClass() == s1) {
 862          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 863       } else {
 864          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 865          nir_print_instr(&instr->instr, stderr);
 866          fprintf(stderr, "\n");
 867       }
 868       break;
 869    }
 870    case nir_op_umax: {
 871       if (dst.regClass() == v1) {
 872          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 873       } else if (dst.regClass() == s1) {
 874          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_imin: {
 883       if (dst.regClass() == v1) {
 884          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 885       } else if (dst.regClass() == s1) {
 886          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 887       } else {
 888          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 889          nir_print_instr(&instr->instr, stderr);
 890          fprintf(stderr, "\n");
 891       }
 892       break;
 893    }
 894    case nir_op_umin: {
 895       if (dst.regClass() == v1) {
 896          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 897       } else if (dst.regClass() == s1) {
 898          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 899       } else {
 900          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 901          nir_print_instr(&instr->instr, stderr);
 902          fprintf(stderr, "\n");
 903       }
 904       break;
 905    }
 906    case nir_op_ior: {
 907       if (instr->dest.dest.ssa.bit_size == 1) {
 908          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 909       } else if (dst.regClass() == v1) {
 910          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 911       } else if (dst.regClass() == s1) {
 912          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 913       } else if (dst.regClass() == s2) {
 914          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 915       } else {
 916          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 917          nir_print_instr(&instr->instr, stderr);
 918          fprintf(stderr, "\n");
 919       }
 920       break;
 921    }
 922    case nir_op_iand: {
 923       if (instr->dest.dest.ssa.bit_size == 1) {
 924          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 925       } else if (dst.regClass() == v1) {
 926          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 927       } else if (dst.regClass() == s1) {
 928          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 929       } else if (dst.regClass() == s2) {
 930          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 931       } else {
 932          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 933          nir_print_instr(&instr->instr, stderr);
 934          fprintf(stderr, "\n");
 935       }
 936       break;
 937    }
 938    case nir_op_ixor: {
 939       if (instr->dest.dest.ssa.bit_size == 1) {
 940          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 941       } else if (dst.regClass() == v1) {
 942          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 943       } else if (dst.regClass() == s1) {
 944          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 945       } else if (dst.regClass() == s2) {
 946          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 947       } else {
 948          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 949          nir_print_instr(&instr->instr, stderr);
 950          fprintf(stderr, "\n");
 951       }
 952       break;
 953    }
 954    case nir_op_ushr: {
 955       if (dst.regClass() == v1) {
 956          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 957       } else if (dst.regClass() == v2) {
 958          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 959                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 960       } else if (dst.regClass() == s2) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 962       } else if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 964       } else {
 965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 966          nir_print_instr(&instr->instr, stderr);
 967          fprintf(stderr, "\n");
 968       }
 969       break;
 970    }
 971    case nir_op_ishl: {
 972       if (dst.regClass() == v1) {
 973          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 974       } else if (dst.regClass() == v2) {
 975          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 976                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 977       } else if (dst.regClass() == s1) {
 978          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 979       } else if (dst.regClass() == s2) {
 980          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 981       } else {
 982          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 983          nir_print_instr(&instr->instr, stderr);
 984          fprintf(stderr, "\n");
 985       }
 986       break;
 987    }
 988    case nir_op_ishr: {
 989       if (dst.regClass() == v1) {
 990          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 991       } else if (dst.regClass() == v2) {
 992          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 993                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 994       } else if (dst.regClass() == s1) {
 995          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 996       } else if (dst.regClass() == s2) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_find_lsb: {
1006       Temp src = get_alu_src(ctx, instr->src[0]);
1007       if (src.regClass() == s1) {
1008          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1009       } else if (src.regClass() == v1) {
1010          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1011       } else if (src.regClass() == s2) {
1012          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1013       } else {
1014          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1015          nir_print_instr(&instr->instr, stderr);
1016          fprintf(stderr, "\n");
1017       }
1018       break;
1019    }
1020    case nir_op_ufind_msb:
1021    case nir_op_ifind_msb: {
1022       Temp src = get_alu_src(ctx, instr->src[0]);
1023       if (src.regClass() == s1 || src.regClass() == s2) {
1024          aco_opcode op = src.regClass() == s2 ?
1025                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1026                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1027          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1028
1029          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1030                                         Operand(src.size() * 32u - 1u), msb_rev);
1031          Temp msb = sub.def(0).getTemp();
1032          Temp carry = sub.def(1).getTemp();
1033
1034          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1035       } else if (src.regClass() == v1) {
1036          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1037          Temp msb_rev = bld.tmp(v1);
1038          emit_vop1_instruction(ctx, instr, op, msb_rev);
1039          Temp msb = bld.tmp(v1);
1040          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1041          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1042       } else {
1043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1044          nir_print_instr(&instr->instr, stderr);
1045          fprintf(stderr, "\n");
1046       }
1047       break;
1048    }
1049    case nir_op_bitfield_reverse: {
1050       if (dst.regClass() == s1) {
1051          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1052       } else if (dst.regClass() == v1) {
1053          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1054       } else {
1055          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1056          nir_print_instr(&instr->instr, stderr);
1057          fprintf(stderr, "\n");
1058       }
1059       break;
1060    }
1061    case nir_op_iadd: {
1062       if (dst.regClass() == s1) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1064          break;
1065       }
1066
1067       Temp src0 = get_alu_src(ctx, instr->src[0]);
1068       Temp src1 = get_alu_src(ctx, instr->src[1]);
1069       if (dst.regClass() == v1) {
1070          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1071          break;
1072       }
1073
1074       assert(src0.size() == 2 && src1.size() == 2);
1075       Temp src00 = bld.tmp(src0.type(), 1);
1076       Temp src01 = bld.tmp(dst.type(), 1);
1077       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1078       Temp src10 = bld.tmp(src1.type(), 1);
1079       Temp src11 = bld.tmp(dst.type(), 1);
1080       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1081
1082       if (dst.regClass() == s2) {
1083          Temp carry = bld.tmp(s1);
1084          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1085          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1086          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1087       } else if (dst.regClass() == v2) {
1088          Temp dst0 = bld.tmp(v1);
1089          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1090          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1091          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1092       } else {
1093          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1094          nir_print_instr(&instr->instr, stderr);
1095          fprintf(stderr, "\n");
1096       }
1097       break;
1098    }
1099    case nir_op_uadd_sat: {
1100       Temp src0 = get_alu_src(ctx, instr->src[0]);
1101       Temp src1 = get_alu_src(ctx, instr->src[1]);
1102       if (dst.regClass() == s1) {
1103          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1104          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1105                   src0, src1);
1106          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1107       } else if (dst.regClass() == v1) {
1108          if (ctx->options->chip_class >= GFX9) {
1109             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1110             add->operands[0] = Operand(src0);
1111             add->operands[1] = Operand(src1);
1112             add->definitions[0] = Definition(dst);
1113             add->clamp = 1;
1114             ctx->block->instructions.emplace_back(std::move(add));
1115          } else {
1116             if (src1.regClass() != v1)
1117                std::swap(src0, src1);
1118             assert(src1.regClass() == v1);
1119             Temp tmp = bld.tmp(v1);
1120             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1121             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1122          }
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_uadd_carry: {
1131       Temp src0 = get_alu_src(ctx, instr->src[0]);
1132       Temp src1 = get_alu_src(ctx, instr->src[1]);
1133       if (dst.regClass() == s1) {
1134          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1135          break;
1136       }
1137       if (dst.regClass() == v1) {
1138          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1139          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1140          break;
1141       }
1142
1143       Temp src00 = bld.tmp(src0.type(), 1);
1144       Temp src01 = bld.tmp(dst.type(), 1);
1145       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1146       Temp src10 = bld.tmp(src1.type(), 1);
1147       Temp src11 = bld.tmp(dst.type(), 1);
1148       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1149       if (dst.regClass() == s2) {
1150          Temp carry = bld.tmp(s1);
1151          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1152          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1153          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1154       } else if (dst.regClass() == v2) {
1155          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1156          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1157          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1158          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1159       } else {
1160          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1161          nir_print_instr(&instr->instr, stderr);
1162          fprintf(stderr, "\n");
1163       }
1164       break;
1165    }
1166    case nir_op_isub: {
1167       if (dst.regClass() == s1) {
1168          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1169          break;
1170       }
1171
1172       Temp src0 = get_alu_src(ctx, instr->src[0]);
1173       Temp src1 = get_alu_src(ctx, instr->src[1]);
1174       if (dst.regClass() == v1) {
1175          bld.vsub32(Definition(dst), src0, src1);
1176          break;
1177       }
1178
1179       Temp src00 = bld.tmp(src0.type(), 1);
1180       Temp src01 = bld.tmp(dst.type(), 1);
1181       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1182       Temp src10 = bld.tmp(src1.type(), 1);
1183       Temp src11 = bld.tmp(dst.type(), 1);
1184       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1185       if (dst.regClass() == s2) {
1186          Temp carry = bld.tmp(s1);
1187          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1188          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1189          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1190       } else if (dst.regClass() == v2) {
1191          Temp lower = bld.tmp(v1);
1192          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1193          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1195       } else {
1196          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1197          nir_print_instr(&instr->instr, stderr);
1198          fprintf(stderr, "\n");
1199       }
1200       break;
1201    }
1202    case nir_op_usub_borrow: {
1203       Temp src0 = get_alu_src(ctx, instr->src[0]);
1204       Temp src1 = get_alu_src(ctx, instr->src[1]);
1205       if (dst.regClass() == s1) {
1206          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1207          break;
1208       } else if (dst.regClass() == v1) {
1209          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1210          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1211          break;
1212       }
1213
1214       Temp src00 = bld.tmp(src0.type(), 1);
1215       Temp src01 = bld.tmp(dst.type(), 1);
1216       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1217       Temp src10 = bld.tmp(src1.type(), 1);
1218       Temp src11 = bld.tmp(dst.type(), 1);
1219       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1220       if (dst.regClass() == s2) {
1221          Temp borrow = bld.tmp(s1);
1222          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1223          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1224          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1225       } else if (dst.regClass() == v2) {
1226          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1227          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1228          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1230       } else {
1231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1232          nir_print_instr(&instr->instr, stderr);
1233          fprintf(stderr, "\n");
1234       }
1235       break;
1236    }
1237    case nir_op_imul: {
1238       if (dst.regClass() == v1) {
1239          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1240                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1241       } else if (dst.regClass() == s1) {
1242          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1243       } else {
1244          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1245          nir_print_instr(&instr->instr, stderr);
1246          fprintf(stderr, "\n");
1247       }
1248       break;
1249    }
1250    case nir_op_umul_high: {
1251       if (dst.regClass() == v1) {
1252          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1253       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1254          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1255       } else if (dst.regClass() == s1) {
1256          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1257                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1258          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1259       } else {
1260          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1261          nir_print_instr(&instr->instr, stderr);
1262          fprintf(stderr, "\n");
1263       }
1264       break;
1265    }
1266    case nir_op_imul_high: {
1267       if (dst.regClass() == v1) {
1268          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1269       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1270          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1271       } else if (dst.regClass() == s1) {
1272          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1273                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1274          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1275       } else {
1276          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1277          nir_print_instr(&instr->instr, stderr);
1278          fprintf(stderr, "\n");
1279       }
1280       break;
1281    }
1282    case nir_op_fmul: {
1283       if (dst.size() == 1) {
1284          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1285       } else if (dst.size() == 2) {
1286          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1287                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1288       } else {
1289          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1290          nir_print_instr(&instr->instr, stderr);
1291          fprintf(stderr, "\n");
1292       }
1293       break;
1294    }
1295    case nir_op_fadd: {
1296       if (dst.size() == 1) {
1297          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1298       } else if (dst.size() == 2) {
1299          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1300                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1301       } else {
1302          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1303          nir_print_instr(&instr->instr, stderr);
1304          fprintf(stderr, "\n");
1305       }
1306       break;
1307    }
1308    case nir_op_fsub: {
1309       Temp src0 = get_alu_src(ctx, instr->src[0]);
1310       Temp src1 = get_alu_src(ctx, instr->src[1]);
1311       if (dst.size() == 1) {
1312          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1313             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1314          else
1315             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1316       } else if (dst.size() == 2) {
1317          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1318                                      get_alu_src(ctx, instr->src[0]),
1319                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1320          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1321          sub->neg[1] = true;
1322       } else {
1323          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1324          nir_print_instr(&instr->instr, stderr);
1325          fprintf(stderr, "\n");
1326       }
1327       break;
1328    }
1329    case nir_op_fmax: {
1330       if (dst.size() == 1) {
1331          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1332       } else if (dst.size() == 2) {
1333          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1334                   get_alu_src(ctx, instr->src[0]),
1335                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1336       } else {
1337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1338          nir_print_instr(&instr->instr, stderr);
1339          fprintf(stderr, "\n");
1340       }
1341       break;
1342    }
1343    case nir_op_fmin: {
1344       if (dst.size() == 1) {
1345          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1346       } else if (dst.size() == 2) {
1347          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]),
1349                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1350       } else {
1351          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1352          nir_print_instr(&instr->instr, stderr);
1353          fprintf(stderr, "\n");
1354       }
1355       break;
1356    }
1357    case nir_op_fmax3: {
1358       if (dst.size() == 1) {
1359          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1360       } else {
1361          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1362          nir_print_instr(&instr->instr, stderr);
1363          fprintf(stderr, "\n");
1364       }
1365       break;
1366    }
1367    case nir_op_fmin3: {
1368       if (dst.size() == 1) {
1369          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1370       } else {
1371          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1372          nir_print_instr(&instr->instr, stderr);
1373          fprintf(stderr, "\n");
1374       }
1375       break;
1376    }
1377    case nir_op_fmed3: {
1378       if (dst.size() == 1) {
1379          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1380       } else {
1381          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1382          nir_print_instr(&instr->instr, stderr);
1383          fprintf(stderr, "\n");
1384       }
1385       break;
1386    }
1387    case nir_op_umax3: {
1388       if (dst.size() == 1) {
1389          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1390       } else {
1391          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1392          nir_print_instr(&instr->instr, stderr);
1393          fprintf(stderr, "\n");
1394       }
1395       break;
1396    }
1397    case nir_op_umin3: {
1398       if (dst.size() == 1) {
1399          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1400       } else {
1401          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1402          nir_print_instr(&instr->instr, stderr);
1403          fprintf(stderr, "\n");
1404       }
1405       break;
1406    }
1407    case nir_op_umed3: {
1408       if (dst.size() == 1) {
1409          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1410       } else {
1411          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1412          nir_print_instr(&instr->instr, stderr);
1413          fprintf(stderr, "\n");
1414       }
1415       break;
1416    }
1417    case nir_op_imax3: {
1418       if (dst.size() == 1) {
1419          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1420       } else {
1421          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1422          nir_print_instr(&instr->instr, stderr);
1423          fprintf(stderr, "\n");
1424       }
1425       break;
1426    }
1427    case nir_op_imin3: {
1428       if (dst.size() == 1) {
1429          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_imed3: {
1438       if (dst.size() == 1) {
1439          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1440       } else {
1441          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1442          nir_print_instr(&instr->instr, stderr);
1443          fprintf(stderr, "\n");
1444       }
1445       break;
1446    }
1447    case nir_op_cube_face_coord: {
1448       Temp in = get_alu_src(ctx, instr->src[0], 3);
1449       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1450                       emit_extract_vector(ctx, in, 1, v1),
1451                       emit_extract_vector(ctx, in, 2, v1) };
1452       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1453       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1454       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1455       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1456       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1457       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1458       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1459       break;
1460    }
1461    case nir_op_cube_face_index: {
1462       Temp in = get_alu_src(ctx, instr->src[0], 3);
1463       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1464                       emit_extract_vector(ctx, in, 1, v1),
1465                       emit_extract_vector(ctx, in, 2, v1) };
1466       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1467       break;
1468    }
1469    case nir_op_bcsel: {
1470       emit_bcsel(ctx, instr, dst);
1471       break;
1472    }
1473    case nir_op_frsq: {
1474       if (dst.size() == 1) {
1475          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1476       } else if (dst.size() == 2) {
1477          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1478       } else {
1479          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1480          nir_print_instr(&instr->instr, stderr);
1481          fprintf(stderr, "\n");
1482       }
1483       break;
1484    }
1485    case nir_op_fneg: {
1486       Temp src = get_alu_src(ctx, instr->src[0]);
1487       if (dst.size() == 1) {
1488          if (ctx->block->fp_mode.must_flush_denorms32)
1489             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1490          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1491       } else if (dst.size() == 2) {
1492          if (ctx->block->fp_mode.must_flush_denorms16_64)
1493             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1494          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1495          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1496          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1497          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1498       } else {
1499          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1500          nir_print_instr(&instr->instr, stderr);
1501          fprintf(stderr, "\n");
1502       }
1503       break;
1504    }
1505    case nir_op_fabs: {
1506       Temp src = get_alu_src(ctx, instr->src[0]);
1507       if (dst.size() == 1) {
1508          if (ctx->block->fp_mode.must_flush_denorms32)
1509             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1510          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1511       } else if (dst.size() == 2) {
1512          if (ctx->block->fp_mode.must_flush_denorms16_64)
1513             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1514          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1515          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1516          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1517          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1518       } else {
1519          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1520          nir_print_instr(&instr->instr, stderr);
1521          fprintf(stderr, "\n");
1522       }
1523       break;
1524    }
1525    case nir_op_fsat: {
1526       Temp src = get_alu_src(ctx, instr->src[0]);
1527       if (dst.size() == 1) {
1528          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1529       } else if (dst.size() == 2) {
1530          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1531          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1532          vop3->clamp = true;
1533       } else {
1534          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1535          nir_print_instr(&instr->instr, stderr);
1536          fprintf(stderr, "\n");
1537       }
1538       break;
1539    }
1540    case nir_op_flog2: {
1541       if (dst.size() == 1) {
1542          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1543       } else {
1544          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1545          nir_print_instr(&instr->instr, stderr);
1546          fprintf(stderr, "\n");
1547       }
1548       break;
1549    }
1550    case nir_op_frcp: {
1551       if (dst.size() == 1) {
1552          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1553       } else if (dst.size() == 2) {
1554          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1555       } else {
1556          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1557          nir_print_instr(&instr->instr, stderr);
1558          fprintf(stderr, "\n");
1559       }
1560       break;
1561    }
1562    case nir_op_fexp2: {
1563       if (dst.size() == 1) {
1564          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1565       } else {
1566          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1567          nir_print_instr(&instr->instr, stderr);
1568          fprintf(stderr, "\n");
1569       }
1570       break;
1571    }
1572    case nir_op_fsqrt: {
1573       if (dst.size() == 1) {
1574          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1575       } else if (dst.size() == 2) {
1576          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1577       } else {
1578          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1579          nir_print_instr(&instr->instr, stderr);
1580          fprintf(stderr, "\n");
1581       }
1582       break;
1583    }
1584    case nir_op_ffract: {
1585       if (dst.size() == 1) {
1586          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1587       } else if (dst.size() == 2) {
1588          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_ffloor: {
1597       if (dst.size() == 1) {
1598          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1599       } else if (dst.size() == 2) {
1600          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1601       } else {
1602          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1603          nir_print_instr(&instr->instr, stderr);
1604          fprintf(stderr, "\n");
1605       }
1606       break;
1607    }
1608    case nir_op_fceil: {
1609       if (dst.size() == 1) {
1610          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1611       } else if (dst.size() == 2) {
1612          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1613       } else {
1614          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1615          nir_print_instr(&instr->instr, stderr);
1616          fprintf(stderr, "\n");
1617       }
1618       break;
1619    }
1620    case nir_op_ftrunc: {
1621       if (dst.size() == 1) {
1622          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1623       } else if (dst.size() == 2) {
1624          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_fround_even: {
1633       if (dst.size() == 1) {
1634          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1635       } else if (dst.size() == 2) {
1636          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1637       } else {
1638          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1639          nir_print_instr(&instr->instr, stderr);
1640          fprintf(stderr, "\n");
1641       }
1642       break;
1643    }
1644    case nir_op_fsin:
1645    case nir_op_fcos: {
1646       Temp src = get_alu_src(ctx, instr->src[0]);
1647       aco_ptr<Instruction> norm;
1648       if (dst.size() == 1) {
1649          Temp tmp;
1650          Operand half_pi(0x3e22f983u);
1651          if (src.type() == RegType::sgpr)
1652             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1653          else
1654             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1655
1656          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1657          if (ctx->options->chip_class < GFX9)
1658             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1659
1660          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1661          bld.vop1(opcode, Definition(dst), tmp);
1662       } else {
1663          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1664          nir_print_instr(&instr->instr, stderr);
1665          fprintf(stderr, "\n");
1666       }
1667       break;
1668    }
1669    case nir_op_ldexp: {
1670       if (dst.size() == 1) {
1671          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1672                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1673                   get_alu_src(ctx, instr->src[1]));
1674       } else if (dst.size() == 2) {
1675          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1676                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1677                   get_alu_src(ctx, instr->src[1]));
1678       } else {
1679          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1680          nir_print_instr(&instr->instr, stderr);
1681          fprintf(stderr, "\n");
1682       }
1683       break;
1684    }
1685    case nir_op_frexp_sig: {
1686       if (dst.size() == 1) {
1687          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1688                   get_alu_src(ctx, instr->src[0]));
1689       } else if (dst.size() == 2) {
1690          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1691                   get_alu_src(ctx, instr->src[0]));
1692       } else {
1693          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1694          nir_print_instr(&instr->instr, stderr);
1695          fprintf(stderr, "\n");
1696       }
1697       break;
1698    }
1699    case nir_op_frexp_exp: {
1700       if (instr->src[0].src.ssa->bit_size == 32) {
1701          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1702                   get_alu_src(ctx, instr->src[0]));
1703       } else if (instr->src[0].src.ssa->bit_size == 64) {
1704          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1705                   get_alu_src(ctx, instr->src[0]));
1706       } else {
1707          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1708          nir_print_instr(&instr->instr, stderr);
1709          fprintf(stderr, "\n");
1710       }
1711       break;
1712    }
1713    case nir_op_fsign: {
1714       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1715       if (dst.size() == 1) {
1716          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1717          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1718          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1719          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1720       } else if (dst.size() == 2) {
1721          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1722          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1723          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1724
1725          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1726          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1727          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1728
1729          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1730       } else {
1731          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1732          nir_print_instr(&instr->instr, stderr);
1733          fprintf(stderr, "\n");
1734       }
1735       break;
1736    }
1737    case nir_op_f2f32: {
1738       if (instr->src[0].src.ssa->bit_size == 64) {
1739          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1740       } else {
1741          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1742          nir_print_instr(&instr->instr, stderr);
1743          fprintf(stderr, "\n");
1744       }
1745       break;
1746    }
1747    case nir_op_f2f64: {
1748       if (instr->src[0].src.ssa->bit_size == 32) {
1749          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1750       } else {
1751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1752          nir_print_instr(&instr->instr, stderr);
1753          fprintf(stderr, "\n");
1754       }
1755       break;
1756    }
1757    case nir_op_i2f32: {
1758       assert(dst.size() == 1);
1759       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1760       break;
1761    }
1762    case nir_op_i2f64: {
1763       if (instr->src[0].src.ssa->bit_size == 32) {
1764          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1765       } else if (instr->src[0].src.ssa->bit_size == 64) {
1766          Temp src = get_alu_src(ctx, instr->src[0]);
1767          RegClass rc = RegClass(src.type(), 1);
1768          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1769          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1770          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1771          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1772          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1773          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1774
1775       } else {
1776          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1777          nir_print_instr(&instr->instr, stderr);
1778          fprintf(stderr, "\n");
1779       }
1780       break;
1781    }
1782    case nir_op_u2f32: {
1783       assert(dst.size() == 1);
1784       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1785       break;
1786    }
1787    case nir_op_u2f64: {
1788       if (instr->src[0].src.ssa->bit_size == 32) {
1789          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1790       } else if (instr->src[0].src.ssa->bit_size == 64) {
1791          Temp src = get_alu_src(ctx, instr->src[0]);
1792          RegClass rc = RegClass(src.type(), 1);
1793          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1794          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1795          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1796          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1797          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1798          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1799       } else {
1800          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1801          nir_print_instr(&instr->instr, stderr);
1802          fprintf(stderr, "\n");
1803       }
1804       break;
1805    }
1806    case nir_op_f2i32: {
1807       Temp src = get_alu_src(ctx, instr->src[0]);
1808       if (instr->src[0].src.ssa->bit_size == 32) {
1809          if (dst.type() == RegType::vgpr)
1810             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1811          else
1812             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1813                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1814
1815       } else if (instr->src[0].src.ssa->bit_size == 64) {
1816          if (dst.type() == RegType::vgpr)
1817             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1818          else
1819             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1820                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1821
1822       } else {
1823          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1824          nir_print_instr(&instr->instr, stderr);
1825          fprintf(stderr, "\n");
1826       }
1827       break;
1828    }
1829    case nir_op_f2u32: {
1830       Temp src = get_alu_src(ctx, instr->src[0]);
1831       if (instr->src[0].src.ssa->bit_size == 32) {
1832          if (dst.type() == RegType::vgpr)
1833             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1834          else
1835             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1836                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1837
1838       } else if (instr->src[0].src.ssa->bit_size == 64) {
1839          if (dst.type() == RegType::vgpr)
1840             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1841          else
1842             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1843                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1844
1845       } else {
1846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1847          nir_print_instr(&instr->instr, stderr);
1848          fprintf(stderr, "\n");
1849       }
1850       break;
1851    }
1852    case nir_op_f2i64: {
1853       Temp src = get_alu_src(ctx, instr->src[0]);
1854       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1855          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1856          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1857          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1858          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1859          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1860          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1861          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1862          Temp new_exponent = bld.tmp(v1);
1863          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1864          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1865          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1866          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1867          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1868          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1869          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1870          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1871          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1872          Temp new_lower = bld.tmp(v1);
1873          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1874          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1875          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1876
1877       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1878          if (src.type() == RegType::vgpr)
1879             src = bld.as_uniform(src);
1880          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1881          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1882          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1883          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1884          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1885          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1886          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1887          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1888          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1889          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1890          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1891          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1892          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1893          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1894          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1895          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1896          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1897          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1898          Temp borrow = bld.tmp(s1);
1899          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1900          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1901          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1902
1903       } else if (instr->src[0].src.ssa->bit_size == 64) {
1904          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1905          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1906          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1907          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1908          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1909          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1910          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1911          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1912          if (dst.type() == RegType::sgpr) {
1913             lower = bld.as_uniform(lower);
1914             upper = bld.as_uniform(upper);
1915          }
1916          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1917
1918       } else {
1919          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1920          nir_print_instr(&instr->instr, stderr);
1921          fprintf(stderr, "\n");
1922       }
1923       break;
1924    }
1925    case nir_op_f2u64: {
1926       Temp src = get_alu_src(ctx, instr->src[0]);
1927       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1928          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1929          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1930          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1931          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1932          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1933          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1934          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1935          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1936          Temp new_exponent = bld.tmp(v1);
1937          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1938          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1939          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1940          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1941          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1942          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1943          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1944          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1945          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1946
1947       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1948          if (src.type() == RegType::vgpr)
1949             src = bld.as_uniform(src);
1950          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1951          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1952          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1953          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1954          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1955          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1956          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1957          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1958          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1959          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1960          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1961          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1962          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1963          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1964          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1965          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1966          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1967          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1968
1969       } else if (instr->src[0].src.ssa->bit_size == 64) {
1970          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1971          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1972          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1973          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1974          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1975          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1976          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1977          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1978          if (dst.type() == RegType::sgpr) {
1979             lower = bld.as_uniform(lower);
1980             upper = bld.as_uniform(upper);
1981          }
1982          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1983
1984       } else {
1985          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1986          nir_print_instr(&instr->instr, stderr);
1987          fprintf(stderr, "\n");
1988       }
1989       break;
1990    }
1991    case nir_op_b2f32: {
1992       Temp src = get_alu_src(ctx, instr->src[0]);
1993       assert(src.regClass() == bld.lm);
1994
1995       if (dst.regClass() == s1) {
1996          src = bool_to_scalar_condition(ctx, src);
1997          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1998       } else if (dst.regClass() == v1) {
1999          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2000       } else {
2001          unreachable("Wrong destination register class for nir_op_b2f32.");
2002       }
2003       break;
2004    }
2005    case nir_op_b2f64: {
2006       Temp src = get_alu_src(ctx, instr->src[0]);
2007       assert(src.regClass() == bld.lm);
2008
2009       if (dst.regClass() == s2) {
2010          src = bool_to_scalar_condition(ctx, src);
2011          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2012       } else if (dst.regClass() == v2) {
2013          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2014          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2015          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2016       } else {
2017          unreachable("Wrong destination register class for nir_op_b2f64.");
2018       }
2019       break;
2020    }
2021    case nir_op_i2i32: {
2022       Temp src = get_alu_src(ctx, instr->src[0]);
2023       if (instr->src[0].src.ssa->bit_size == 64) {
2024          /* we can actually just say dst = src, as it would map the lower register */
2025          emit_extract_vector(ctx, src, 0, dst);
2026       } else {
2027          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2028          nir_print_instr(&instr->instr, stderr);
2029          fprintf(stderr, "\n");
2030       }
2031       break;
2032    }
2033    case nir_op_u2u32: {
2034       Temp src = get_alu_src(ctx, instr->src[0]);
2035       if (instr->src[0].src.ssa->bit_size == 16) {
2036          if (dst.regClass() == s1) {
2037             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2038          } else {
2039             // TODO: do better with SDWA
2040             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2041          }
2042       } else if (instr->src[0].src.ssa->bit_size == 64) {
2043          /* we can actually just say dst = src, as it would map the lower register */
2044          emit_extract_vector(ctx, src, 0, dst);
2045       } else {
2046          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2047          nir_print_instr(&instr->instr, stderr);
2048          fprintf(stderr, "\n");
2049       }
2050       break;
2051    }
2052    case nir_op_i2i64: {
2053       Temp src = get_alu_src(ctx, instr->src[0]);
2054       if (src.regClass() == s1) {
2055          Temp high = bld.sopc(aco_opcode::s_ashr_i32, bld.def(s1, scc), src, Operand(31u));
2056          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2057       } else if (src.regClass() == v1) {
2058          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2059          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2060       } else {
2061          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2062          nir_print_instr(&instr->instr, stderr);
2063          fprintf(stderr, "\n");
2064       }
2065       break;
2066    }
2067    case nir_op_u2u64: {
2068       Temp src = get_alu_src(ctx, instr->src[0]);
2069       if (instr->src[0].src.ssa->bit_size == 32) {
2070          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2071       } else {
2072          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2073          nir_print_instr(&instr->instr, stderr);
2074          fprintf(stderr, "\n");
2075       }
2076       break;
2077    }
2078    case nir_op_b2i32: {
2079       Temp src = get_alu_src(ctx, instr->src[0]);
2080       assert(src.regClass() == bld.lm);
2081
2082       if (dst.regClass() == s1) {
2083          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2084          bool_to_scalar_condition(ctx, src, dst);
2085       } else if (dst.regClass() == v1) {
2086          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2087       } else {
2088          unreachable("Invalid register class for b2i32");
2089       }
2090       break;
2091    }
2092    case nir_op_i2b1: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       assert(dst.regClass() == bld.lm);
2095
2096       if (src.type() == RegType::vgpr) {
2097          assert(src.regClass() == v1 || src.regClass() == v2);
2098          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2099                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2100       } else {
2101          assert(src.regClass() == s1 || src.regClass() == s2);
2102          Temp tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2103                              bld.scc(bld.def(s1)), Operand(0u), src);
2104          bool_to_vector_condition(ctx, tmp, dst);
2105       }
2106       break;
2107    }
2108    case nir_op_pack_64_2x32_split: {
2109       Temp src0 = get_alu_src(ctx, instr->src[0]);
2110       Temp src1 = get_alu_src(ctx, instr->src[1]);
2111
2112       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2113       break;
2114    }
2115    case nir_op_unpack_64_2x32_split_x:
2116       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2117       break;
2118    case nir_op_unpack_64_2x32_split_y:
2119       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2120       break;
2121    case nir_op_pack_half_2x16: {
2122       Temp src = get_alu_src(ctx, instr->src[0], 2);
2123
2124       if (dst.regClass() == v1) {
2125          Temp src0 = bld.tmp(v1);
2126          Temp src1 = bld.tmp(v1);
2127          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2128          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2129             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2130          else
2131             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2132                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2133                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2134       } else {
2135          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2136          nir_print_instr(&instr->instr, stderr);
2137          fprintf(stderr, "\n");
2138       }
2139       break;
2140    }
2141    case nir_op_unpack_half_2x16_split_x: {
2142       if (dst.regClass() == v1) {
2143          Builder bld(ctx->program, ctx->block);
2144          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2145       } else {
2146          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2147          nir_print_instr(&instr->instr, stderr);
2148          fprintf(stderr, "\n");
2149       }
2150       break;
2151    }
2152    case nir_op_unpack_half_2x16_split_y: {
2153       if (dst.regClass() == v1) {
2154          Builder bld(ctx->program, ctx->block);
2155          /* TODO: use SDWA here */
2156          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2157                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2158       } else {
2159          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2160          nir_print_instr(&instr->instr, stderr);
2161          fprintf(stderr, "\n");
2162       }
2163       break;
2164    }
2165    case nir_op_fquantize2f16: {
2166       Temp src = get_alu_src(ctx, instr->src[0]);
2167       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2168       Temp f32, cmp_res;
2169
2170       if (ctx->program->chip_class >= GFX8) {
2171          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2172          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2173          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2174       } else {
2175          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2176           * so compare the result and flush to 0 if it's smaller.
2177           */
2178          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2179          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2180          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), f32, smallest);
2181          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2182          cmp_res = vop3->definitions[0].getTemp();
2183       }
2184
2185       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2186          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2187          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2188       } else {
2189          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2190       }
2191       break;
2192    }
2193    case nir_op_bfm: {
2194       Temp bits = get_alu_src(ctx, instr->src[0]);
2195       Temp offset = get_alu_src(ctx, instr->src[1]);
2196
2197       if (dst.regClass() == s1) {
2198          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2199       } else if (dst.regClass() == v1) {
2200          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2201       } else {
2202          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2203          nir_print_instr(&instr->instr, stderr);
2204          fprintf(stderr, "\n");
2205       }
2206       break;
2207    }
2208    case nir_op_bitfield_select: {
2209       /* (mask & insert) | (~mask & base) */
2210       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2211       Temp insert = get_alu_src(ctx, instr->src[1]);
2212       Temp base = get_alu_src(ctx, instr->src[2]);
2213
2214       /* dst = (insert & bitmask) | (base & ~bitmask) */
2215       if (dst.regClass() == s1) {
2216          aco_ptr<Instruction> sop2;
2217          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2218          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2219          Operand lhs;
2220          if (const_insert && const_bitmask) {
2221             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2222          } else {
2223             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2224             lhs = Operand(insert);
2225          }
2226
2227          Operand rhs;
2228          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2229          if (const_base && const_bitmask) {
2230             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2231          } else {
2232             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2233             rhs = Operand(base);
2234          }
2235
2236          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2237
2238       } else if (dst.regClass() == v1) {
2239          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2240             base = as_vgpr(ctx, base);
2241          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2242             insert = as_vgpr(ctx, insert);
2243
2244          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2245
2246       } else {
2247          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2248          nir_print_instr(&instr->instr, stderr);
2249          fprintf(stderr, "\n");
2250       }
2251       break;
2252    }
2253    case nir_op_ubfe:
2254    case nir_op_ibfe: {
2255       Temp base = get_alu_src(ctx, instr->src[0]);
2256       Temp offset = get_alu_src(ctx, instr->src[1]);
2257       Temp bits = get_alu_src(ctx, instr->src[2]);
2258
2259       if (dst.type() == RegType::sgpr) {
2260          Operand extract;
2261          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2262          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2263          if (const_offset && const_bits) {
2264             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2265             extract = Operand(const_extract);
2266          } else {
2267             Operand width;
2268             if (const_bits) {
2269                width = Operand(const_bits->u32 << 16);
2270             } else {
2271                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2272             }
2273             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2274          }
2275
2276          aco_opcode opcode;
2277          if (dst.regClass() == s1) {
2278             if (instr->op == nir_op_ubfe)
2279                opcode = aco_opcode::s_bfe_u32;
2280             else
2281                opcode = aco_opcode::s_bfe_i32;
2282          } else if (dst.regClass() == s2) {
2283             if (instr->op == nir_op_ubfe)
2284                opcode = aco_opcode::s_bfe_u64;
2285             else
2286                opcode = aco_opcode::s_bfe_i64;
2287          } else {
2288             unreachable("Unsupported BFE bit size");
2289          }
2290
2291          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2292
2293       } else {
2294          aco_opcode opcode;
2295          if (dst.regClass() == v1) {
2296             if (instr->op == nir_op_ubfe)
2297                opcode = aco_opcode::v_bfe_u32;
2298             else
2299                opcode = aco_opcode::v_bfe_i32;
2300          } else {
2301             unreachable("Unsupported BFE bit size");
2302          }
2303
2304          emit_vop3a_instruction(ctx, instr, opcode, dst);
2305       }
2306       break;
2307    }
2308    case nir_op_bit_count: {
2309       Temp src = get_alu_src(ctx, instr->src[0]);
2310       if (src.regClass() == s1) {
2311          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2312       } else if (src.regClass() == v1) {
2313          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2314       } else if (src.regClass() == v2) {
2315          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2316                   emit_extract_vector(ctx, src, 1, v1),
2317                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2318                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2319       } else if (src.regClass() == s2) {
2320          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_flt: {
2329       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2330       break;
2331    }
2332    case nir_op_fge: {
2333       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2334       break;
2335    }
2336    case nir_op_feq: {
2337       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2338       break;
2339    }
2340    case nir_op_fne: {
2341       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2342       break;
2343    }
2344    case nir_op_ilt: {
2345       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2346       break;
2347    }
2348    case nir_op_ige: {
2349       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2350       break;
2351    }
2352    case nir_op_ieq: {
2353       if (instr->src[0].src.ssa->bit_size == 1)
2354          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2355       else
2356          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32, aco_opcode::s_cmp_eq_u64);
2357       break;
2358    }
2359    case nir_op_ine: {
2360       if (instr->src[0].src.ssa->bit_size == 1)
2361          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2362       else
2363          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32, aco_opcode::s_cmp_lg_u64);
2364       break;
2365    }
2366    case nir_op_ult: {
2367       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2368       break;
2369    }
2370    case nir_op_uge: {
2371       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2372       break;
2373    }
2374    case nir_op_fddx:
2375    case nir_op_fddy:
2376    case nir_op_fddx_fine:
2377    case nir_op_fddy_fine:
2378    case nir_op_fddx_coarse:
2379    case nir_op_fddy_coarse: {
2380       Definition tl = bld.def(v1);
2381       uint16_t dpp_ctrl;
2382       if (instr->op == nir_op_fddx_fine) {
2383          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2384          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2385       } else if (instr->op == nir_op_fddy_fine) {
2386          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2387          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2388       } else {
2389          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2390          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2391             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2392          else
2393             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2394       }
2395
2396       Definition tmp = bld.def(v1);
2397       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2398       emit_wqm(ctx, tmp.getTemp(), dst, true);
2399       break;
2400    }
2401    default:
2402       fprintf(stderr, "Unknown NIR ALU instr: ");
2403       nir_print_instr(&instr->instr, stderr);
2404       fprintf(stderr, "\n");
2405    }
2406 }
2407
2408 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2409 {
2410    Temp dst = get_ssa_temp(ctx, &instr->def);
2411
2412    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2413    // which get truncated the lsb if double and msb if int
2414    // for now, we only use s_mov_b64 with 64bit inline constants
2415    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2416    assert(dst.type() == RegType::sgpr);
2417
2418    Builder bld(ctx->program, ctx->block);
2419
2420    if (instr->def.bit_size == 1) {
2421       assert(dst.regClass() == bld.lm);
2422       int val = instr->value[0].b ? -1 : 0;
2423       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2424       bld.sop1(Builder::s_mov, Definition(dst), op);
2425    } else if (dst.size() == 1) {
2426       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2427    } else {
2428       assert(dst.size() != 1);
2429       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2430       if (instr->def.bit_size == 64)
2431          for (unsigned i = 0; i < dst.size(); i++)
2432             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2433       else {
2434          for (unsigned i = 0; i < dst.size(); i++)
2435             vec->operands[i] = Operand{instr->value[i].u32};
2436       }
2437       vec->definitions[0] = Definition(dst);
2438       ctx->block->instructions.emplace_back(std::move(vec));
2439    }
2440 }
2441
2442 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2443 {
2444    uint32_t new_mask = 0;
2445    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2446       if (mask & (1u << i))
2447          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2448    return new_mask;
2449 }
2450
2451 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2452 {
2453    /* This wouldn't work inside control flow or with indirect offsets but
2454     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2455
2456    unsigned write_mask = nir_intrinsic_write_mask(instr);
2457    unsigned component = nir_intrinsic_component(instr);
2458    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2459    unsigned idx = nir_intrinsic_base(instr) + component;
2460
2461    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2462    if (off_instr->type != nir_instr_type_load_const) {
2463       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2464       nir_print_instr(off_instr, stderr);
2465       fprintf(stderr, "\n");
2466    }
2467    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2468
2469    if (instr->src[0].ssa->bit_size == 64)
2470       write_mask = widen_mask(write_mask, 2);
2471
2472    for (unsigned i = 0; i < 8; ++i) {
2473       if (write_mask & (1 << i)) {
2474          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2475          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2476       }
2477       idx++;
2478    }
2479 }
2480
2481 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2482 {
2483    Builder bld(ctx->program, ctx->block);
2484    unsigned write_mask = nir_intrinsic_write_mask(instr);
2485    Operand values[4];
2486    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2487    for (unsigned i = 0; i < 4; ++i) {
2488       if (write_mask & (1 << i)) {
2489          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2490          values[i] = Operand(tmp);
2491       } else {
2492          values[i] = Operand(v1);
2493       }
2494    }
2495
2496    unsigned index = nir_intrinsic_base(instr) / 4;
2497    unsigned target, col_format;
2498    unsigned enabled_channels = 0xF;
2499    aco_opcode compr_op = (aco_opcode)0;
2500
2501    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2502    assert(offset && "Non-const offsets on exports not yet supported");
2503    index += offset->u32;
2504
2505    assert(index != FRAG_RESULT_COLOR);
2506
2507    /* Unlike vertex shader exports, it's fine to use multiple exports to
2508     * export separate channels of one target. So shaders which export both
2509     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2510     * TODO: combine the exports in those cases and create better code
2511     */
2512
2513    if (index == FRAG_RESULT_SAMPLE_MASK) {
2514
2515       if (ctx->program->info->ps.writes_z) {
2516          target = V_008DFC_SQ_EXP_MRTZ;
2517          enabled_channels = 0x4;
2518          col_format = (unsigned) -1;
2519
2520          values[2] = values[0];
2521          values[0] = Operand(v1);
2522       } else {
2523          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2524                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2525          return;
2526       }
2527
2528    } else if (index == FRAG_RESULT_DEPTH) {
2529
2530       target = V_008DFC_SQ_EXP_MRTZ;
2531       enabled_channels = 0x1;
2532       col_format = (unsigned) -1;
2533
2534    } else if (index == FRAG_RESULT_STENCIL) {
2535
2536       if (ctx->program->info->ps.writes_z) {
2537          target = V_008DFC_SQ_EXP_MRTZ;
2538          enabled_channels = 0x2;
2539          col_format = (unsigned) -1;
2540
2541          values[1] = values[0];
2542          values[0] = Operand(v1);
2543       } else {
2544          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2545          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2546                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2547          return;
2548       }
2549
2550    } else {
2551       index -= FRAG_RESULT_DATA0;
2552       target = V_008DFC_SQ_EXP_MRT + index;
2553       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2554    }
2555    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2556    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2557
2558    switch (col_format)
2559    {
2560    case V_028714_SPI_SHADER_ZERO:
2561       enabled_channels = 0; /* writemask */
2562       target = V_008DFC_SQ_EXP_NULL;
2563       break;
2564
2565    case V_028714_SPI_SHADER_32_R:
2566       enabled_channels = 1;
2567       break;
2568
2569    case V_028714_SPI_SHADER_32_GR:
2570       enabled_channels = 0x3;
2571       break;
2572
2573    case V_028714_SPI_SHADER_32_AR:
2574       if (ctx->options->chip_class >= GFX10) {
2575          /* Special case: on GFX10, the outputs are different for 32_AR */
2576          enabled_channels = 0x3;
2577          values[1] = values[3];
2578          values[3] = Operand(v1);
2579       } else {
2580          enabled_channels = 0x9;
2581       }
2582       break;
2583
2584    case V_028714_SPI_SHADER_FP16_ABGR:
2585       enabled_channels = 0x5;
2586       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2587       break;
2588
2589    case V_028714_SPI_SHADER_UNORM16_ABGR:
2590       enabled_channels = 0x5;
2591       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2592       break;
2593
2594    case V_028714_SPI_SHADER_SNORM16_ABGR:
2595       enabled_channels = 0x5;
2596       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2597       break;
2598
2599    case V_028714_SPI_SHADER_UINT16_ABGR: {
2600       enabled_channels = 0x5;
2601       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2602       if (is_int8 || is_int10) {
2603          /* clamp */
2604          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2605          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2606
2607          for (unsigned i = 0; i < 4; i++) {
2608             if ((write_mask >> i) & 1) {
2609                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2610                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2611                                     values[i]);
2612             }
2613          }
2614       }
2615       break;
2616    }
2617
2618    case V_028714_SPI_SHADER_SINT16_ABGR:
2619       enabled_channels = 0x5;
2620       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2621       if (is_int8 || is_int10) {
2622          /* clamp */
2623          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2624          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2625          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2626          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2627
2628          for (unsigned i = 0; i < 4; i++) {
2629             if ((write_mask >> i) & 1) {
2630                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2631                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2632                                     values[i]);
2633                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2634                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2635                                     values[i]);
2636             }
2637          }
2638       }
2639       break;
2640
2641    case V_028714_SPI_SHADER_32_ABGR:
2642       enabled_channels = 0xF;
2643       break;
2644
2645    default:
2646       break;
2647    }
2648
2649    if (target == V_008DFC_SQ_EXP_NULL)
2650       return;
2651
2652    if ((bool) compr_op) {
2653       for (int i = 0; i < 2; i++) {
2654          /* check if at least one of the values to be compressed is enabled */
2655          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2656          if (enabled) {
2657             enabled_channels |= enabled << (i*2);
2658             values[i] = bld.vop3(compr_op, bld.def(v1),
2659                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2660                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2661          } else {
2662             values[i] = Operand(v1);
2663          }
2664       }
2665       values[2] = Operand(v1);
2666       values[3] = Operand(v1);
2667    } else {
2668       for (int i = 0; i < 4; i++)
2669          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2670    }
2671
2672    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2673            enabled_channels, target, (bool) compr_op);
2674 }
2675
2676 Operand load_lds_size_m0(isel_context *ctx)
2677 {
2678    /* TODO: m0 does not need to be initialized on GFX9+ */
2679    Builder bld(ctx->program, ctx->block);
2680    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2681 }
2682
2683 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2684               Temp address, unsigned base_offset, unsigned align)
2685 {
2686    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2687
2688    Builder bld(ctx->program, ctx->block);
2689
2690    Operand m = load_lds_size_m0(ctx);
2691
2692    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2693    unsigned bytes_read = 0;
2694    unsigned result_size = 0;
2695    unsigned total_bytes = num_components * elem_size_bytes;
2696    std::array<Temp, 4> result;
2697
2698    while (bytes_read < total_bytes) {
2699       unsigned todo = total_bytes - bytes_read;
2700       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2701       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2702
2703       aco_opcode op = aco_opcode::last_opcode;
2704       bool read2 = false;
2705       if (todo >= 16 && aligned16) {
2706          op = aco_opcode::ds_read_b128;
2707          todo = 16;
2708       } else if (todo >= 16 && aligned8) {
2709          op = aco_opcode::ds_read2_b64;
2710          read2 = true;
2711          todo = 16;
2712       } else if (todo >= 12 && aligned16) {
2713          op = aco_opcode::ds_read_b96;
2714          todo = 12;
2715       } else if (todo >= 8 && aligned8) {
2716          op = aco_opcode::ds_read_b64;
2717          todo = 8;
2718       } else if (todo >= 8) {
2719          op = aco_opcode::ds_read2_b32;
2720          read2 = true;
2721          todo = 8;
2722       } else if (todo >= 4) {
2723          op = aco_opcode::ds_read_b32;
2724          todo = 4;
2725       } else {
2726          assert(false);
2727       }
2728       assert(todo % elem_size_bytes == 0);
2729       unsigned num_elements = todo / elem_size_bytes;
2730       unsigned offset = base_offset + bytes_read;
2731       unsigned max_offset = read2 ? 1019 : 65535;
2732
2733       Temp address_offset = address;
2734       if (offset > max_offset) {
2735          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2736          offset = bytes_read;
2737       }
2738       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2739
2740       Temp res;
2741       if (num_components == 1 && dst.type() == RegType::vgpr)
2742          res = dst;
2743       else
2744          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2745
2746       if (read2)
2747          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2748       else
2749          res = bld.ds(op, Definition(res), address_offset, m, offset);
2750
2751       if (num_components == 1) {
2752          assert(todo == total_bytes);
2753          if (dst.type() == RegType::sgpr)
2754             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2755          return;
2756       }
2757
2758       if (dst.type() == RegType::sgpr)
2759          res = bld.as_uniform(res);
2760
2761       if (num_elements == 1) {
2762          result[result_size++] = res;
2763       } else {
2764          assert(res != dst && res.size() % num_elements == 0);
2765          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2766          split->operands[0] = Operand(res);
2767          for (unsigned i = 0; i < num_elements; i++)
2768             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2769          ctx->block->instructions.emplace_back(std::move(split));
2770       }
2771
2772       bytes_read += todo;
2773    }
2774
2775    assert(result_size == num_components && result_size > 1);
2776    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2777    for (unsigned i = 0; i < result_size; i++)
2778       vec->operands[i] = Operand(result[i]);
2779    vec->definitions[0] = Definition(dst);
2780    ctx->block->instructions.emplace_back(std::move(vec));
2781    ctx->allocated_vec.emplace(dst.id(), result);
2782 }
2783
2784 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2785 {
2786    if (start == 0 && size == data.size())
2787       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2788
2789    unsigned size_hint = 1;
2790    auto it = ctx->allocated_vec.find(data.id());
2791    if (it != ctx->allocated_vec.end())
2792       size_hint = it->second[0].size();
2793    if (size % size_hint || start % size_hint)
2794       size_hint = 1;
2795
2796    start /= size_hint;
2797    size /= size_hint;
2798
2799    Temp elems[size];
2800    for (unsigned i = 0; i < size; i++)
2801       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2802
2803    if (size == 1)
2804       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2805
2806    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2807    for (unsigned i = 0; i < size; i++)
2808       vec->operands[i] = Operand(elems[i]);
2809    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2810    vec->definitions[0] = Definition(res);
2811    ctx->block->instructions.emplace_back(std::move(vec));
2812    return res;
2813 }
2814
2815 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2816 {
2817    Builder bld(ctx->program, ctx->block);
2818    unsigned bytes_written = 0;
2819    while (bytes_written < total_size * 4) {
2820       unsigned todo = total_size * 4 - bytes_written;
2821       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2822       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2823
2824       aco_opcode op = aco_opcode::last_opcode;
2825       bool write2 = false;
2826       unsigned size = 0;
2827       if (todo >= 16 && aligned16) {
2828          op = aco_opcode::ds_write_b128;
2829          size = 4;
2830       } else if (todo >= 16 && aligned8) {
2831          op = aco_opcode::ds_write2_b64;
2832          write2 = true;
2833          size = 4;
2834       } else if (todo >= 12 && aligned16) {
2835          op = aco_opcode::ds_write_b96;
2836          size = 3;
2837       } else if (todo >= 8 && aligned8) {
2838          op = aco_opcode::ds_write_b64;
2839          size = 2;
2840       } else if (todo >= 8) {
2841          op = aco_opcode::ds_write2_b32;
2842          write2 = true;
2843          size = 2;
2844       } else if (todo >= 4) {
2845          op = aco_opcode::ds_write_b32;
2846          size = 1;
2847       } else {
2848          assert(false);
2849       }
2850
2851       unsigned offset = offset0 + offset1 + bytes_written;
2852       unsigned max_offset = write2 ? 1020 : 65535;
2853       Temp address_offset = address;
2854       if (offset > max_offset) {
2855          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2856          offset = offset1 + bytes_written;
2857       }
2858       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2859
2860       if (write2) {
2861          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2862          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2863          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2864       } else {
2865          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2866          bld.ds(op, address_offset, val, m, offset);
2867       }
2868
2869       bytes_written += size * 4;
2870    }
2871 }
2872
2873 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2874                Temp address, unsigned base_offset, unsigned align)
2875 {
2876    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2877
2878    Operand m = load_lds_size_m0(ctx);
2879
2880    /* we need at most two stores for 32bit variables */
2881    int start[2], count[2];
2882    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2883    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2884    assert(wrmask == 0);
2885
2886    /* one combined store is sufficient */
2887    if (count[0] == count[1]) {
2888       Builder bld(ctx->program, ctx->block);
2889
2890       Temp address_offset = address;
2891       if ((base_offset >> 2) + start[1] > 255) {
2892          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2893          base_offset = 0;
2894       }
2895
2896       assert(count[0] == 1);
2897       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2898       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2899       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2900       base_offset = base_offset / elem_size_bytes;
2901       bld.ds(op, address_offset, val0, val1, m,
2902              base_offset + start[0], base_offset + start[1]);
2903       return;
2904    }
2905
2906    for (unsigned i = 0; i < 2; i++) {
2907       if (count[i] == 0)
2908          continue;
2909
2910       unsigned elem_size_words = elem_size_bytes / 4;
2911       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2912                       base_offset, start[i] * elem_size_bytes, align);
2913    }
2914    return;
2915 }
2916
2917 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2918 {
2919    if (ctx->stage == vertex_vs) {
2920       visit_store_vs_output(ctx, instr);
2921    } else if (ctx->stage == fragment_fs) {
2922       visit_store_fs_output(ctx, instr);
2923    } else {
2924       unreachable("Shader stage not implemented");
2925    }
2926 }
2927
2928 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2929 {
2930    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2931    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2932
2933    Builder bld(ctx->program, ctx->block);
2934    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2935    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2936 }
2937
2938 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2939 {
2940    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2941    for (unsigned i = 0; i < num_components; i++)
2942       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
2943    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
2944       assert(num_components == 4);
2945       Builder bld(ctx->program, ctx->block);
2946       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
2947    }
2948
2949    for (Operand& op : vec->operands)
2950       op = op.isUndefined() ? Operand(0u) : op;
2951
2952    vec->definitions[0] = Definition(dst);
2953    ctx->block->instructions.emplace_back(std::move(vec));
2954    emit_split_vector(ctx, dst, num_components);
2955    return;
2956 }
2957
2958 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2959 {
2960    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2961    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2962    unsigned idx = nir_intrinsic_base(instr);
2963    unsigned component = nir_intrinsic_component(instr);
2964    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
2965
2966    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2967    if (offset) {
2968       assert(offset->u32 == 0);
2969    } else {
2970       /* the lower 15bit of the prim_mask contain the offset into LDS
2971        * while the upper bits contain the number of prims */
2972       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2973       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2974       Builder bld(ctx->program, ctx->block);
2975       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2976       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2977       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2978       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2979       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2980    }
2981
2982    if (instr->dest.ssa.num_components == 1) {
2983       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2984    } else {
2985       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2986       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2987       {
2988          Temp tmp = {ctx->program->allocateId(), v1};
2989          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2990          vec->operands[i] = Operand(tmp);
2991       }
2992       vec->definitions[0] = Definition(dst);
2993       ctx->block->instructions.emplace_back(std::move(vec));
2994    }
2995 }
2996
2997 unsigned get_num_channels_from_data_format(unsigned data_format)
2998 {
2999    switch (data_format) {
3000    case V_008F0C_BUF_DATA_FORMAT_8:
3001    case V_008F0C_BUF_DATA_FORMAT_16:
3002    case V_008F0C_BUF_DATA_FORMAT_32:
3003       return 1;
3004    case V_008F0C_BUF_DATA_FORMAT_8_8:
3005    case V_008F0C_BUF_DATA_FORMAT_16_16:
3006    case V_008F0C_BUF_DATA_FORMAT_32_32:
3007       return 2;
3008    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3009    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3010    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3011       return 3;
3012    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3013    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3014    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3015    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3016    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3017       return 4;
3018    default:
3019       break;
3020    }
3021
3022    return 4;
3023 }
3024
3025 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3026  * so we may need to fix it up. */
3027 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3028 {
3029    Builder bld(ctx->program, ctx->block);
3030
3031    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3032       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3033
3034    /* For the integer-like cases, do a natural sign extension.
3035     *
3036     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3037     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3038     * exponent.
3039     */
3040    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3041    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3042
3043    /* Convert back to the right type. */
3044    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3045       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3046       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3047       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3048    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3049       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3050    }
3051
3052    return alpha;
3053 }
3054
3055 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3056 {
3057    Builder bld(ctx->program, ctx->block);
3058    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3059    if (ctx->stage & sw_vs) {
3060
3061       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3062       if (off_instr->type != nir_instr_type_load_const) {
3063          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3064          nir_print_instr(off_instr, stderr);
3065          fprintf(stderr, "\n");
3066       }
3067       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3068
3069       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3070
3071       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3072       unsigned component = nir_intrinsic_component(instr);
3073       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3074       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3075       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3076       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3077
3078       unsigned dfmt = attrib_format & 0xf;
3079
3080       unsigned nfmt = (attrib_format >> 4) & 0x7;
3081       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3082       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3083       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3084       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3085       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3086       if (post_shuffle)
3087          num_channels = MAX2(num_channels, 3);
3088
3089       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3090
3091       Temp index;
3092       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3093          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3094          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3095          if (divisor) {
3096             ctx->needs_instance_id = true;
3097             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3098             if (divisor != 1) {
3099                Temp divided = bld.tmp(v1);
3100                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3101                index = bld.vadd32(bld.def(v1), start_instance, divided);
3102             } else {
3103                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3104             }
3105          } else {
3106             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3107          }
3108       } else {
3109          index = bld.vadd32(bld.def(v1),
3110                             get_arg(ctx, ctx->args->ac.base_vertex),
3111                             get_arg(ctx, ctx->args->ac.vertex_id));
3112       }
3113
3114       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3115          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3116          attrib_offset = attrib_offset % attrib_stride;
3117       }
3118
3119       Operand soffset(0u);
3120       if (attrib_offset >= 4096) {
3121          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3122          attrib_offset = 0;
3123       }
3124
3125       aco_opcode opcode;
3126       switch (num_channels) {
3127       case 1:
3128          opcode = aco_opcode::tbuffer_load_format_x;
3129          break;
3130       case 2:
3131          opcode = aco_opcode::tbuffer_load_format_xy;
3132          break;
3133       case 3:
3134          opcode = aco_opcode::tbuffer_load_format_xyz;
3135          break;
3136       case 4:
3137          opcode = aco_opcode::tbuffer_load_format_xyzw;
3138          break;
3139       default:
3140          unreachable("Unimplemented load_input vector size");
3141       }
3142
3143       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3144
3145       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3146       mubuf->operands[0] = Operand(index);
3147       mubuf->operands[1] = Operand(list);
3148       mubuf->operands[2] = soffset;
3149       mubuf->definitions[0] = Definition(tmp);
3150       mubuf->idxen = true;
3151       mubuf->can_reorder = true;
3152       mubuf->dfmt = dfmt;
3153       mubuf->nfmt = nfmt;
3154       assert(attrib_offset < 4096);
3155       mubuf->offset = attrib_offset;
3156       ctx->block->instructions.emplace_back(std::move(mubuf));
3157
3158       emit_split_vector(ctx, tmp, tmp.size());
3159
3160       if (tmp.id() != dst.id()) {
3161          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3162                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3163
3164          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3165          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3166          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3167
3168          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3169          for (unsigned i = 0; i < dst.size(); i++) {
3170             unsigned idx = i + component;
3171             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3172                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3173                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3174             } else if (idx < num_channels) {
3175                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3176             } else if (is_float && idx == 3) {
3177                vec->operands[i] = Operand(0x3f800000u);
3178             } else if (!is_float && idx == 3) {
3179                vec->operands[i] = Operand(1u);
3180             } else {
3181                vec->operands[i] = Operand(0u);
3182             }
3183          }
3184          vec->definitions[0] = Definition(dst);
3185          ctx->block->instructions.emplace_back(std::move(vec));
3186          emit_split_vector(ctx, dst, dst.size());
3187       }
3188
3189    } else if (ctx->stage == fragment_fs) {
3190       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3191       if (off_instr->type != nir_instr_type_load_const ||
3192           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3193          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3194          nir_print_instr(off_instr, stderr);
3195          fprintf(stderr, "\n");
3196       }
3197
3198       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3199       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3200       if (offset) {
3201          assert(offset->u32 == 0);
3202       } else {
3203          /* the lower 15bit of the prim_mask contain the offset into LDS
3204           * while the upper bits contain the number of prims */
3205          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3206          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3207          Builder bld(ctx->program, ctx->block);
3208          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3209          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3210          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3211          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3212          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3213       }
3214
3215       unsigned idx = nir_intrinsic_base(instr);
3216       unsigned component = nir_intrinsic_component(instr);
3217
3218       if (dst.size() == 1) {
3219          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3220       } else {
3221          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3222          for (unsigned i = 0; i < dst.size(); i++)
3223             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3224          vec->definitions[0] = Definition(dst);
3225          bld.insert(std::move(vec));
3226       }
3227
3228    } else {
3229       unreachable("Shader stage not implemented");
3230    }
3231 }
3232
3233 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3234 {
3235    if (ctx->program->info->need_indirect_descriptor_sets) {
3236       Builder bld(ctx->program, ctx->block);
3237       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3238       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3239    }
3240
3241    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3242 }
3243
3244
3245 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3246 {
3247    Builder bld(ctx->program, ctx->block);
3248    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3249    if (!ctx->divergent_vals[instr->dest.ssa.index])
3250       index = bld.as_uniform(index);
3251    unsigned desc_set = nir_intrinsic_desc_set(instr);
3252    unsigned binding = nir_intrinsic_binding(instr);
3253
3254    Temp desc_ptr;
3255    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3256    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3257    unsigned offset = layout->binding[binding].offset;
3258    unsigned stride;
3259    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3260        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3261       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3262       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3263       offset = pipeline_layout->push_constant_size + 16 * idx;
3264       stride = 16;
3265    } else {
3266       desc_ptr = load_desc_ptr(ctx, desc_set);
3267       stride = layout->binding[binding].size;
3268    }
3269
3270    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3271    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3272    if (stride != 1) {
3273       if (nir_const_index) {
3274          const_index = const_index * stride;
3275       } else if (index.type() == RegType::vgpr) {
3276          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3277          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3278       } else {
3279          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3280       }
3281    }
3282    if (offset) {
3283       if (nir_const_index) {
3284          const_index = const_index + offset;
3285       } else if (index.type() == RegType::vgpr) {
3286          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3287       } else {
3288          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3289       }
3290    }
3291
3292    if (nir_const_index && const_index == 0) {
3293       index = desc_ptr;
3294    } else if (index.type() == RegType::vgpr) {
3295       index = bld.vadd32(bld.def(v1),
3296                          nir_const_index ? Operand(const_index) : Operand(index),
3297                          Operand(desc_ptr));
3298    } else {
3299       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3300                        nir_const_index ? Operand(const_index) : Operand(index),
3301                        Operand(desc_ptr));
3302    }
3303
3304    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3305 }
3306
3307 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3308                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3309 {
3310    Builder bld(ctx->program, ctx->block);
3311
3312    unsigned num_bytes = dst.size() * 4;
3313    bool dlc = glc && ctx->options->chip_class >= GFX10;
3314
3315    aco_opcode op;
3316    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3317       if (ctx->options->chip_class < GFX8)
3318          offset = as_vgpr(ctx, offset);
3319
3320       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3321       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3322       unsigned const_offset = 0;
3323
3324       Temp lower = Temp();
3325       if (num_bytes > 16) {
3326          assert(num_components == 3 || num_components == 4);
3327          op = aco_opcode::buffer_load_dwordx4;
3328          lower = bld.tmp(v4);
3329          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3330          mubuf->definitions[0] = Definition(lower);
3331          mubuf->operands[0] = vaddr;
3332          mubuf->operands[1] = Operand(rsrc);
3333          mubuf->operands[2] = soffset;
3334          mubuf->offen = (offset.type() == RegType::vgpr);
3335          mubuf->glc = glc;
3336          mubuf->dlc = dlc;
3337          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3338          mubuf->can_reorder = readonly;
3339          bld.insert(std::move(mubuf));
3340          emit_split_vector(ctx, lower, 2);
3341          num_bytes -= 16;
3342          const_offset = 16;
3343       }
3344
3345       switch (num_bytes) {
3346          case 4:
3347             op = aco_opcode::buffer_load_dword;
3348             break;
3349          case 8:
3350             op = aco_opcode::buffer_load_dwordx2;
3351             break;
3352          case 12:
3353             op = aco_opcode::buffer_load_dwordx3;
3354             break;
3355          case 16:
3356             op = aco_opcode::buffer_load_dwordx4;
3357             break;
3358          default:
3359             unreachable("Load SSBO not implemented for this size.");
3360       }
3361       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3362       mubuf->operands[0] = vaddr;
3363       mubuf->operands[1] = Operand(rsrc);
3364       mubuf->operands[2] = soffset;
3365       mubuf->offen = (offset.type() == RegType::vgpr);
3366       mubuf->glc = glc;
3367       mubuf->dlc = dlc;
3368       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3369       mubuf->can_reorder = readonly;
3370       mubuf->offset = const_offset;
3371       aco_ptr<Instruction> instr = std::move(mubuf);
3372
3373       if (dst.size() > 4) {
3374          assert(lower != Temp());
3375          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3376          instr->definitions[0] = Definition(upper);
3377          bld.insert(std::move(instr));
3378          if (dst.size() == 8)
3379             emit_split_vector(ctx, upper, 2);
3380          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3381          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3382          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3383          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3384          if (dst.size() == 8)
3385             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3386       }
3387
3388       if (dst.type() == RegType::sgpr) {
3389          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3390          instr->definitions[0] = Definition(vec);
3391          bld.insert(std::move(instr));
3392          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3393       } else {
3394          instr->definitions[0] = Definition(dst);
3395          bld.insert(std::move(instr));
3396       }
3397    } else {
3398       switch (num_bytes) {
3399          case 4:
3400             op = aco_opcode::s_buffer_load_dword;
3401             break;
3402          case 8:
3403             op = aco_opcode::s_buffer_load_dwordx2;
3404             break;
3405          case 12:
3406          case 16:
3407             op = aco_opcode::s_buffer_load_dwordx4;
3408             break;
3409          case 24:
3410          case 32:
3411             op = aco_opcode::s_buffer_load_dwordx8;
3412             break;
3413          default:
3414             unreachable("Load SSBO not implemented for this size.");
3415       }
3416       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3417       load->operands[0] = Operand(rsrc);
3418       load->operands[1] = Operand(bld.as_uniform(offset));
3419       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3420       load->definitions[0] = Definition(dst);
3421       load->glc = glc;
3422       load->dlc = dlc;
3423       load->barrier = readonly ? barrier_none : barrier_buffer;
3424       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3425       assert(ctx->options->chip_class >= GFX8 || !glc);
3426
3427       /* trim vector */
3428       if (dst.size() == 3) {
3429          Temp vec = bld.tmp(s4);
3430          load->definitions[0] = Definition(vec);
3431          bld.insert(std::move(load));
3432          emit_split_vector(ctx, vec, 4);
3433
3434          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3435                     emit_extract_vector(ctx, vec, 0, s1),
3436                     emit_extract_vector(ctx, vec, 1, s1),
3437                     emit_extract_vector(ctx, vec, 2, s1));
3438       } else if (dst.size() == 6) {
3439          Temp vec = bld.tmp(s8);
3440          load->definitions[0] = Definition(vec);
3441          bld.insert(std::move(load));
3442          emit_split_vector(ctx, vec, 4);
3443
3444          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3445                     emit_extract_vector(ctx, vec, 0, s2),
3446                     emit_extract_vector(ctx, vec, 1, s2),
3447                     emit_extract_vector(ctx, vec, 2, s2));
3448       } else {
3449          bld.insert(std::move(load));
3450       }
3451
3452    }
3453    emit_split_vector(ctx, dst, num_components);
3454 }
3455
3456 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3457 {
3458    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3459    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3460
3461    Builder bld(ctx->program, ctx->block);
3462
3463    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3464    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3465    unsigned binding = nir_intrinsic_binding(idx_instr);
3466    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3467
3468    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3469       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3470                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3471                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3472                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3473       if (ctx->options->chip_class >= GFX10) {
3474          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3475                       S_008F0C_OOB_SELECT(3) |
3476                       S_008F0C_RESOURCE_LEVEL(1);
3477       } else {
3478          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3479                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3480       }
3481       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3482                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3483                                      Operand(0xFFFFFFFFu),
3484                                      Operand(desc_type));
3485       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3486                         rsrc, upper_dwords);
3487    } else {
3488       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3489       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3490    }
3491
3492    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3493 }
3494
3495 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3496 {
3497    Builder bld(ctx->program, ctx->block);
3498    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3499
3500    unsigned offset = nir_intrinsic_base(instr);
3501    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3502    if (index_cv && instr->dest.ssa.bit_size == 32) {
3503
3504       unsigned count = instr->dest.ssa.num_components;
3505       unsigned start = (offset + index_cv->u32) / 4u;
3506       start -= ctx->args->ac.base_inline_push_consts;
3507       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3508          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3509          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3510          for (unsigned i = 0; i < count; ++i) {
3511             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3512             vec->operands[i] = Operand{elems[i]};
3513          }
3514          vec->definitions[0] = Definition(dst);
3515          ctx->block->instructions.emplace_back(std::move(vec));
3516          ctx->allocated_vec.emplace(dst.id(), elems);
3517          return;
3518       }
3519    }
3520
3521    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3522    if (offset != 0) // TODO check if index != 0 as well
3523       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3524    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3525    Temp vec = dst;
3526    bool trim = false;
3527    aco_opcode op;
3528
3529    switch (dst.size()) {
3530    case 1:
3531       op = aco_opcode::s_load_dword;
3532       break;
3533    case 2:
3534       op = aco_opcode::s_load_dwordx2;
3535       break;
3536    case 3:
3537       vec = bld.tmp(s4);
3538       trim = true;
3539    case 4:
3540       op = aco_opcode::s_load_dwordx4;
3541       break;
3542    case 6:
3543       vec = bld.tmp(s8);
3544       trim = true;
3545    case 8:
3546       op = aco_opcode::s_load_dwordx8;
3547       break;
3548    default:
3549       unreachable("unimplemented or forbidden load_push_constant.");
3550    }
3551
3552    bld.smem(op, Definition(vec), ptr, index);
3553
3554    if (trim) {
3555       emit_split_vector(ctx, vec, 4);
3556       RegClass rc = dst.size() == 3 ? s1 : s2;
3557       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3558                  emit_extract_vector(ctx, vec, 0, rc),
3559                  emit_extract_vector(ctx, vec, 1, rc),
3560                  emit_extract_vector(ctx, vec, 2, rc));
3561
3562    }
3563    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3564 }
3565
3566 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3567 {
3568    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3569
3570    Builder bld(ctx->program, ctx->block);
3571
3572    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3573                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3574                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3575                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3576    if (ctx->options->chip_class >= GFX10) {
3577       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3578                    S_008F0C_OOB_SELECT(3) |
3579                    S_008F0C_RESOURCE_LEVEL(1);
3580    } else {
3581       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3582                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3583    }
3584
3585    unsigned base = nir_intrinsic_base(instr);
3586    unsigned range = nir_intrinsic_range(instr);
3587
3588    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3589    if (base && offset.type() == RegType::sgpr)
3590       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3591    else if (base && offset.type() == RegType::vgpr)
3592       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3593
3594    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3595                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3596                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3597                           Operand(desc_type));
3598
3599    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3600 }
3601
3602 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3603 {
3604    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3605       ctx->cf_info.exec_potentially_empty = true;
3606
3607    ctx->program->needs_exact = true;
3608
3609    // TODO: optimize uniform conditions
3610    Builder bld(ctx->program, ctx->block);
3611    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3612    assert(src.regClass() == bld.lm);
3613    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3614    bld.pseudo(aco_opcode::p_discard_if, src);
3615    ctx->block->kind |= block_kind_uses_discard_if;
3616    return;
3617 }
3618
3619 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3620 {
3621    Builder bld(ctx->program, ctx->block);
3622
3623    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3624       ctx->cf_info.exec_potentially_empty = true;
3625
3626    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3627                     ctx->cf_info.parent_loop.has_divergent_continue;
3628
3629    if (ctx->block->loop_nest_depth &&
3630        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3631       /* we handle discards the same way as jump instructions */
3632       append_logical_end(ctx->block);
3633
3634       /* in loops, discard behaves like break */
3635       Block *linear_target = ctx->cf_info.parent_loop.exit;
3636       ctx->block->kind |= block_kind_discard;
3637
3638       if (!divergent) {
3639          /* uniform discard - loop ends here */
3640          assert(nir_instr_is_last(&instr->instr));
3641          ctx->block->kind |= block_kind_uniform;
3642          ctx->cf_info.has_branch = true;
3643          bld.branch(aco_opcode::p_branch);
3644          add_linear_edge(ctx->block->index, linear_target);
3645          return;
3646       }
3647
3648       /* we add a break right behind the discard() instructions */
3649       ctx->block->kind |= block_kind_break;
3650       unsigned idx = ctx->block->index;
3651
3652       /* remove critical edges from linear CFG */
3653       bld.branch(aco_opcode::p_branch);
3654       Block* break_block = ctx->program->create_and_insert_block();
3655       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3656       break_block->kind |= block_kind_uniform;
3657       add_linear_edge(idx, break_block);
3658       add_linear_edge(break_block->index, linear_target);
3659       bld.reset(break_block);
3660       bld.branch(aco_opcode::p_branch);
3661
3662       Block* continue_block = ctx->program->create_and_insert_block();
3663       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3664       add_linear_edge(idx, continue_block);
3665       append_logical_start(continue_block);
3666       ctx->block = continue_block;
3667
3668       return;
3669    }
3670
3671    /* it can currently happen that NIR doesn't remove the unreachable code */
3672    if (!nir_instr_is_last(&instr->instr)) {
3673       ctx->program->needs_exact = true;
3674       /* save exec somewhere temporarily so that it doesn't get
3675        * overwritten before the discard from outer exec masks */
3676       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3677       bld.pseudo(aco_opcode::p_discard_if, cond);
3678       ctx->block->kind |= block_kind_uses_discard_if;
3679       return;
3680    }
3681
3682    /* This condition is incorrect for uniformly branched discards in a loop
3683     * predicated by a divergent condition, but the above code catches that case
3684     * and the discard would end up turning into a discard_if.
3685     * For example:
3686     * if (divergent) {
3687     *    while (...) {
3688     *       if (uniform) {
3689     *          discard;
3690     *       }
3691     *    }
3692     * }
3693     */
3694    if (!ctx->cf_info.parent_if.is_divergent) {
3695       /* program just ends here */
3696       ctx->block->kind |= block_kind_uniform;
3697       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3698               0 /* enabled mask */, 9 /* dest */,
3699               false /* compressed */, true/* done */, true /* valid mask */);
3700       bld.sopp(aco_opcode::s_endpgm);
3701       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3702    } else {
3703       ctx->block->kind |= block_kind_discard;
3704       /* branch and linear edge is added by visit_if() */
3705    }
3706 }
3707
3708 enum aco_descriptor_type {
3709    ACO_DESC_IMAGE,
3710    ACO_DESC_FMASK,
3711    ACO_DESC_SAMPLER,
3712    ACO_DESC_BUFFER,
3713    ACO_DESC_PLANE_0,
3714    ACO_DESC_PLANE_1,
3715    ACO_DESC_PLANE_2,
3716 };
3717
3718 static bool
3719 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3720    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3721       return false;
3722    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3723    return dim == ac_image_cube ||
3724           dim == ac_image_1darray ||
3725           dim == ac_image_2darray ||
3726           dim == ac_image_2darraymsaa;
3727 }
3728
3729 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3730                       enum aco_descriptor_type desc_type,
3731                       const nir_tex_instr *tex_instr, bool image, bool write)
3732 {
3733 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3734    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3735    if (it != ctx->tex_desc.end())
3736       return it->second;
3737 */
3738    Temp index = Temp();
3739    bool index_set = false;
3740    unsigned constant_index = 0;
3741    unsigned descriptor_set;
3742    unsigned base_index;
3743    Builder bld(ctx->program, ctx->block);
3744
3745    if (!deref_instr) {
3746       assert(tex_instr && !image);
3747       descriptor_set = 0;
3748       base_index = tex_instr->sampler_index;
3749    } else {
3750       while(deref_instr->deref_type != nir_deref_type_var) {
3751          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3752          if (!array_size)
3753             array_size = 1;
3754
3755          assert(deref_instr->deref_type == nir_deref_type_array);
3756          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3757          if (const_value) {
3758             constant_index += array_size * const_value->u32;
3759          } else {
3760             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3761             if (indirect.type() == RegType::vgpr)
3762                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3763
3764             if (array_size != 1)
3765                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3766
3767             if (!index_set) {
3768                index = indirect;
3769                index_set = true;
3770             } else {
3771                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3772             }
3773          }
3774
3775          deref_instr = nir_src_as_deref(deref_instr->parent);
3776       }
3777       descriptor_set = deref_instr->var->data.descriptor_set;
3778       base_index = deref_instr->var->data.binding;
3779    }
3780
3781    Temp list = load_desc_ptr(ctx, descriptor_set);
3782    list = convert_pointer_to_64_bit(ctx, list);
3783
3784    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3785    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3786    unsigned offset = binding->offset;
3787    unsigned stride = binding->size;
3788    aco_opcode opcode;
3789    RegClass type;
3790
3791    assert(base_index < layout->binding_count);
3792
3793    switch (desc_type) {
3794    case ACO_DESC_IMAGE:
3795       type = s8;
3796       opcode = aco_opcode::s_load_dwordx8;
3797       break;
3798    case ACO_DESC_FMASK:
3799       type = s8;
3800       opcode = aco_opcode::s_load_dwordx8;
3801       offset += 32;
3802       break;
3803    case ACO_DESC_SAMPLER:
3804       type = s4;
3805       opcode = aco_opcode::s_load_dwordx4;
3806       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3807          offset += radv_combined_image_descriptor_sampler_offset(binding);
3808       break;
3809    case ACO_DESC_BUFFER:
3810       type = s4;
3811       opcode = aco_opcode::s_load_dwordx4;
3812       break;
3813    case ACO_DESC_PLANE_0:
3814    case ACO_DESC_PLANE_1:
3815       type = s8;
3816       opcode = aco_opcode::s_load_dwordx8;
3817       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3818       break;
3819    case ACO_DESC_PLANE_2:
3820       type = s4;
3821       opcode = aco_opcode::s_load_dwordx4;
3822       offset += 64;
3823       break;
3824    default:
3825       unreachable("invalid desc_type\n");
3826    }
3827
3828    offset += constant_index * stride;
3829
3830    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3831       (!index_set || binding->immutable_samplers_equal)) {
3832       if (binding->immutable_samplers_equal)
3833          constant_index = 0;
3834
3835       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3836       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3837                         Operand(samplers[constant_index * 4 + 0]),
3838                         Operand(samplers[constant_index * 4 + 1]),
3839                         Operand(samplers[constant_index * 4 + 2]),
3840                         Operand(samplers[constant_index * 4 + 3]));
3841    }
3842
3843    Operand off;
3844    if (!index_set) {
3845       off = Operand(offset);
3846    } else {
3847       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3848                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3849    }
3850
3851    Temp res = bld.smem(opcode, bld.def(type), list, off);
3852
3853    if (desc_type == ACO_DESC_PLANE_2) {
3854       Temp components[8];
3855       for (unsigned i = 0; i < 8; i++)
3856          components[i] = bld.tmp(s1);
3857       bld.pseudo(aco_opcode::p_split_vector,
3858                  Definition(components[0]),
3859                  Definition(components[1]),
3860                  Definition(components[2]),
3861                  Definition(components[3]),
3862                  res);
3863
3864       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3865       bld.pseudo(aco_opcode::p_split_vector,
3866                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3867                  Definition(components[4]),
3868                  Definition(components[5]),
3869                  Definition(components[6]),
3870                  Definition(components[7]),
3871                  desc2);
3872
3873       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3874                        components[0], components[1], components[2], components[3],
3875                        components[4], components[5], components[6], components[7]);
3876    }
3877
3878    return res;
3879 }
3880
3881 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3882 {
3883    switch (dim) {
3884    case GLSL_SAMPLER_DIM_BUF:
3885       return 1;
3886    case GLSL_SAMPLER_DIM_1D:
3887       return array ? 2 : 1;
3888    case GLSL_SAMPLER_DIM_2D:
3889       return array ? 3 : 2;
3890    case GLSL_SAMPLER_DIM_MS:
3891       return array ? 4 : 3;
3892    case GLSL_SAMPLER_DIM_3D:
3893    case GLSL_SAMPLER_DIM_CUBE:
3894       return 3;
3895    case GLSL_SAMPLER_DIM_RECT:
3896    case GLSL_SAMPLER_DIM_SUBPASS:
3897       return 2;
3898    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3899       return 3;
3900    default:
3901       break;
3902    }
3903    return 0;
3904 }
3905
3906
3907 /* Adjust the sample index according to FMASK.
3908  *
3909  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3910  * which is the identity mapping. Each nibble says which physical sample
3911  * should be fetched to get that sample.
3912  *
3913  * For example, 0x11111100 means there are only 2 samples stored and
3914  * the second sample covers 3/4 of the pixel. When reading samples 0
3915  * and 1, return physical sample 0 (determined by the first two 0s
3916  * in FMASK), otherwise return physical sample 1.
3917  *
3918  * The sample index should be adjusted as follows:
3919  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3920  */
3921 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3922 {
3923    Builder bld(ctx->program, ctx->block);
3924    Temp fmask = bld.tmp(v1);
3925    unsigned dim = ctx->options->chip_class >= GFX10
3926                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3927                   : 0;
3928
3929    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3930    load->operands[0] = Operand(coords);
3931    load->operands[1] = Operand(fmask_desc_ptr);
3932    load->definitions[0] = Definition(fmask);
3933    load->glc = false;
3934    load->dlc = false;
3935    load->dmask = 0x1;
3936    load->unrm = true;
3937    load->da = da;
3938    load->dim = dim;
3939    load->can_reorder = true; /* fmask images shouldn't be modified */
3940    ctx->block->instructions.emplace_back(std::move(load));
3941
3942    Operand sample_index4;
3943    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3944       sample_index4 = Operand(sample_index.constantValue() << 2);
3945    } else if (sample_index.regClass() == s1) {
3946       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3947    } else {
3948       assert(sample_index.regClass() == v1);
3949       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3950    }
3951
3952    Temp final_sample;
3953    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3954       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3955    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3956       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3957    else
3958       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3959
3960    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3961     * resource descriptor is 0 (invalid),
3962     */
3963    Temp compare = bld.tmp(bld.lm);
3964    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3965                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3966
3967    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3968
3969    /* Replace the MSAA sample index. */
3970    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3971 }
3972
3973 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3974 {
3975
3976    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3977    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3978    bool is_array = glsl_sampler_type_is_array(type);
3979    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3980    assert(!add_frag_pos && "Input attachments should be lowered.");
3981    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3982    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3983    int count = image_type_to_components_count(dim, is_array);
3984    std::vector<Operand> coords(count);
3985
3986    if (is_ms) {
3987       Operand sample_index;
3988       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3989       if (sample_cv)
3990          sample_index = Operand(sample_cv->u32);
3991       else
3992          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3993
3994       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3995          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3996          for (unsigned i = 0; i < vec->operands.size(); i++)
3997             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3998          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3999          vec->definitions[0] = Definition(fmask_load_address);
4000          ctx->block->instructions.emplace_back(std::move(vec));
4001
4002          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4003          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4004       }
4005       count--;
4006       coords[count] = sample_index;
4007    }
4008
4009    if (count == 1 && !gfx9_1d)
4010       return emit_extract_vector(ctx, src0, 0, v1);
4011
4012    if (gfx9_1d) {
4013       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4014       coords.resize(coords.size() + 1);
4015       coords[1] = Operand((uint32_t) 0);
4016       if (is_array)
4017          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4018    } else {
4019       for (int i = 0; i < count; i++)
4020          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4021    }
4022
4023    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4024    for (unsigned i = 0; i < coords.size(); i++)
4025       vec->operands[i] = coords[i];
4026    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4027    vec->definitions[0] = Definition(res);
4028    ctx->block->instructions.emplace_back(std::move(vec));
4029    return res;
4030 }
4031
4032
4033 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4034 {
4035    Builder bld(ctx->program, ctx->block);
4036    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4037    const struct glsl_type *type = glsl_without_array(var->type);
4038    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4039    bool is_array = glsl_sampler_type_is_array(type);
4040    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4041
4042    if (dim == GLSL_SAMPLER_DIM_BUF) {
4043       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4044       unsigned num_channels = util_last_bit(mask);
4045       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4046       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4047
4048       aco_opcode opcode;
4049       switch (num_channels) {
4050       case 1:
4051          opcode = aco_opcode::buffer_load_format_x;
4052          break;
4053       case 2:
4054          opcode = aco_opcode::buffer_load_format_xy;
4055          break;
4056       case 3:
4057          opcode = aco_opcode::buffer_load_format_xyz;
4058          break;
4059       case 4:
4060          opcode = aco_opcode::buffer_load_format_xyzw;
4061          break;
4062       default:
4063          unreachable(">4 channel buffer image load");
4064       }
4065       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4066       load->operands[0] = Operand(vindex);
4067       load->operands[1] = Operand(rsrc);
4068       load->operands[2] = Operand((uint32_t) 0);
4069       Temp tmp;
4070       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4071          tmp = dst;
4072       else
4073          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4074       load->definitions[0] = Definition(tmp);
4075       load->idxen = true;
4076       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4077       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4078       load->barrier = barrier_image;
4079       ctx->block->instructions.emplace_back(std::move(load));
4080
4081       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4082       return;
4083    }
4084
4085    Temp coords = get_image_coords(ctx, instr, type);
4086    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4087
4088    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4089    unsigned num_components = util_bitcount(dmask);
4090    Temp tmp;
4091    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4092       tmp = dst;
4093    else
4094       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4095
4096    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4097    load->operands[0] = Operand(coords);
4098    load->operands[1] = Operand(resource);
4099    load->definitions[0] = Definition(tmp);
4100    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4101    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4102    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4103    load->dmask = dmask;
4104    load->unrm = true;
4105    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4106    load->barrier = barrier_image;
4107    ctx->block->instructions.emplace_back(std::move(load));
4108
4109    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4110    return;
4111 }
4112
4113 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4114 {
4115    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4116    const struct glsl_type *type = glsl_without_array(var->type);
4117    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4118    bool is_array = glsl_sampler_type_is_array(type);
4119    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4120
4121    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4122
4123    if (dim == GLSL_SAMPLER_DIM_BUF) {
4124       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4125       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4126       aco_opcode opcode;
4127       switch (data.size()) {
4128       case 1:
4129          opcode = aco_opcode::buffer_store_format_x;
4130          break;
4131       case 2:
4132          opcode = aco_opcode::buffer_store_format_xy;
4133          break;
4134       case 3:
4135          opcode = aco_opcode::buffer_store_format_xyz;
4136          break;
4137       case 4:
4138          opcode = aco_opcode::buffer_store_format_xyzw;
4139          break;
4140       default:
4141          unreachable(">4 channel buffer image store");
4142       }
4143       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4144       store->operands[0] = Operand(vindex);
4145       store->operands[1] = Operand(rsrc);
4146       store->operands[2] = Operand((uint32_t) 0);
4147       store->operands[3] = Operand(data);
4148       store->idxen = true;
4149       store->glc = glc;
4150       store->dlc = false;
4151       store->disable_wqm = true;
4152       store->barrier = barrier_image;
4153       ctx->program->needs_exact = true;
4154       ctx->block->instructions.emplace_back(std::move(store));
4155       return;
4156    }
4157
4158    assert(data.type() == RegType::vgpr);
4159    Temp coords = get_image_coords(ctx, instr, type);
4160    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4161
4162    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4163    store->operands[0] = Operand(coords);
4164    store->operands[1] = Operand(resource);
4165    store->operands[2] = Operand(s4);
4166    store->operands[3] = Operand(data);
4167    store->glc = glc;
4168    store->dlc = false;
4169    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4170    store->dmask = (1 << data.size()) - 1;
4171    store->unrm = true;
4172    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4173    store->disable_wqm = true;
4174    store->barrier = barrier_image;
4175    ctx->program->needs_exact = true;
4176    ctx->block->instructions.emplace_back(std::move(store));
4177    return;
4178 }
4179
4180 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4181 {
4182    /* return the previous value if dest is ever used */
4183    bool return_previous = false;
4184    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4185       return_previous = true;
4186       break;
4187    }
4188    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4189       return_previous = true;
4190       break;
4191    }
4192
4193    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4194    const struct glsl_type *type = glsl_without_array(var->type);
4195    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4196    bool is_array = glsl_sampler_type_is_array(type);
4197    Builder bld(ctx->program, ctx->block);
4198
4199    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4200    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4201
4202    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4203       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4204
4205    aco_opcode buf_op, image_op;
4206    switch (instr->intrinsic) {
4207       case nir_intrinsic_image_deref_atomic_add:
4208          buf_op = aco_opcode::buffer_atomic_add;
4209          image_op = aco_opcode::image_atomic_add;
4210          break;
4211       case nir_intrinsic_image_deref_atomic_umin:
4212          buf_op = aco_opcode::buffer_atomic_umin;
4213          image_op = aco_opcode::image_atomic_umin;
4214          break;
4215       case nir_intrinsic_image_deref_atomic_imin:
4216          buf_op = aco_opcode::buffer_atomic_smin;
4217          image_op = aco_opcode::image_atomic_smin;
4218          break;
4219       case nir_intrinsic_image_deref_atomic_umax:
4220          buf_op = aco_opcode::buffer_atomic_umax;
4221          image_op = aco_opcode::image_atomic_umax;
4222          break;
4223       case nir_intrinsic_image_deref_atomic_imax:
4224          buf_op = aco_opcode::buffer_atomic_smax;
4225          image_op = aco_opcode::image_atomic_smax;
4226          break;
4227       case nir_intrinsic_image_deref_atomic_and:
4228          buf_op = aco_opcode::buffer_atomic_and;
4229          image_op = aco_opcode::image_atomic_and;
4230          break;
4231       case nir_intrinsic_image_deref_atomic_or:
4232          buf_op = aco_opcode::buffer_atomic_or;
4233          image_op = aco_opcode::image_atomic_or;
4234          break;
4235       case nir_intrinsic_image_deref_atomic_xor:
4236          buf_op = aco_opcode::buffer_atomic_xor;
4237          image_op = aco_opcode::image_atomic_xor;
4238          break;
4239       case nir_intrinsic_image_deref_atomic_exchange:
4240          buf_op = aco_opcode::buffer_atomic_swap;
4241          image_op = aco_opcode::image_atomic_swap;
4242          break;
4243       case nir_intrinsic_image_deref_atomic_comp_swap:
4244          buf_op = aco_opcode::buffer_atomic_cmpswap;
4245          image_op = aco_opcode::image_atomic_cmpswap;
4246          break;
4247       default:
4248          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4249    }
4250
4251    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4252
4253    if (dim == GLSL_SAMPLER_DIM_BUF) {
4254       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4255       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4256       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4257       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4258       mubuf->operands[0] = Operand(vindex);
4259       mubuf->operands[1] = Operand(resource);
4260       mubuf->operands[2] = Operand((uint32_t)0);
4261       mubuf->operands[3] = Operand(data);
4262       if (return_previous)
4263          mubuf->definitions[0] = Definition(dst);
4264       mubuf->offset = 0;
4265       mubuf->idxen = true;
4266       mubuf->glc = return_previous;
4267       mubuf->dlc = false; /* Not needed for atomics */
4268       mubuf->disable_wqm = true;
4269       mubuf->barrier = barrier_image;
4270       ctx->program->needs_exact = true;
4271       ctx->block->instructions.emplace_back(std::move(mubuf));
4272       return;
4273    }
4274
4275    Temp coords = get_image_coords(ctx, instr, type);
4276    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4277    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4278    mimg->operands[0] = Operand(coords);
4279    mimg->operands[1] = Operand(resource);
4280    mimg->operands[2] = Operand(s4); /* no sampler */
4281    mimg->operands[3] = Operand(data);
4282    if (return_previous)
4283       mimg->definitions[0] = Definition(dst);
4284    mimg->glc = return_previous;
4285    mimg->dlc = false; /* Not needed for atomics */
4286    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4287    mimg->dmask = (1 << data.size()) - 1;
4288    mimg->unrm = true;
4289    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4290    mimg->disable_wqm = true;
4291    mimg->barrier = barrier_image;
4292    ctx->program->needs_exact = true;
4293    ctx->block->instructions.emplace_back(std::move(mimg));
4294    return;
4295 }
4296
4297 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4298 {
4299    if (in_elements && ctx->options->chip_class == GFX8) {
4300       Builder bld(ctx->program, ctx->block);
4301
4302       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4303       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4304       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4305       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4306
4307       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4308       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4309
4310       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4311       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4312       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4313
4314       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4315       /* idea
4316        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4317        * in case 12 (or 3?), we have to divide by 3:
4318        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4319        * use v_mul_hi_u32 with magic number to divide
4320        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4321        * disable v_skip
4322        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4323        */
4324
4325    } else {
4326       emit_extract_vector(ctx, desc, 2, dst);
4327    }
4328 }
4329
4330 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4331 {
4332    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4333    const struct glsl_type *type = glsl_without_array(var->type);
4334    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4335    bool is_array = glsl_sampler_type_is_array(type);
4336    Builder bld(ctx->program, ctx->block);
4337
4338    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4339       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4340       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4341    }
4342
4343    /* LOD */
4344    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4345
4346    /* Resource */
4347    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4348
4349    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4350
4351    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4352    mimg->operands[0] = Operand(lod);
4353    mimg->operands[1] = Operand(resource);
4354    unsigned& dmask = mimg->dmask;
4355    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4356    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4357    mimg->da = glsl_sampler_type_is_array(type);
4358    mimg->can_reorder = true;
4359    Definition& def = mimg->definitions[0];
4360    ctx->block->instructions.emplace_back(std::move(mimg));
4361
4362    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4363        glsl_sampler_type_is_array(type)) {
4364
4365       assert(instr->dest.ssa.num_components == 3);
4366       Temp tmp = {ctx->program->allocateId(), v3};
4367       def = Definition(tmp);
4368       emit_split_vector(ctx, tmp, 3);
4369
4370       /* divide 3rd value by 6 by multiplying with magic number */
4371       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4372       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4373
4374       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4375                  emit_extract_vector(ctx, tmp, 0, v1),
4376                  emit_extract_vector(ctx, tmp, 1, v1),
4377                  by_6);
4378
4379    } else if (ctx->options->chip_class == GFX9 &&
4380               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4381               glsl_sampler_type_is_array(type)) {
4382       assert(instr->dest.ssa.num_components == 2);
4383       def = Definition(dst);
4384       dmask = 0x5;
4385    } else {
4386       def = Definition(dst);
4387    }
4388
4389    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4390 }
4391
4392 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4393 {
4394    Builder bld(ctx->program, ctx->block);
4395    unsigned num_components = instr->num_components;
4396
4397    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4398    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4399    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4400
4401    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4402    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4403 }
4404
4405 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4406 {
4407    Builder bld(ctx->program, ctx->block);
4408    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4409    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4410    unsigned writemask = nir_intrinsic_write_mask(instr);
4411
4412    Temp offset;
4413    if (ctx->options->chip_class < GFX8)
4414       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4415    else
4416       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4417
4418    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4419    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4420
4421    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4422                ctx->options->chip_class >= GFX8;
4423    if (smem)
4424       offset = bld.as_uniform(offset);
4425    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4426
4427    while (writemask) {
4428       int start, count;
4429       u_bit_scan_consecutive_range(&writemask, &start, &count);
4430       if (count == 3 && smem) {
4431          writemask |= 1u << (start + 2);
4432          count = 2;
4433       }
4434       int num_bytes = count * elem_size_bytes;
4435
4436       if (num_bytes > 16) {
4437          assert(elem_size_bytes == 8);
4438          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4439          count = 2;
4440          num_bytes = 16;
4441       }
4442
4443       // TODO: check alignment of sub-dword stores
4444       // TODO: split 3 bytes. there is no store instruction for that
4445
4446       Temp write_data;
4447       if (count != instr->num_components) {
4448          emit_split_vector(ctx, data, instr->num_components);
4449          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4450          for (int i = 0; i < count; i++) {
4451             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4452             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4453          }
4454          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4455          vec->definitions[0] = Definition(write_data);
4456          ctx->block->instructions.emplace_back(std::move(vec));
4457       } else if (!smem && data.type() != RegType::vgpr) {
4458          assert(num_bytes % 4 == 0);
4459          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4460       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4461          assert(num_bytes % 4 == 0);
4462          write_data = bld.as_uniform(data);
4463       } else {
4464          write_data = data;
4465       }
4466
4467       aco_opcode vmem_op, smem_op;
4468       switch (num_bytes) {
4469          case 4:
4470             vmem_op = aco_opcode::buffer_store_dword;
4471             smem_op = aco_opcode::s_buffer_store_dword;
4472             break;
4473          case 8:
4474             vmem_op = aco_opcode::buffer_store_dwordx2;
4475             smem_op = aco_opcode::s_buffer_store_dwordx2;
4476             break;
4477          case 12:
4478             vmem_op = aco_opcode::buffer_store_dwordx3;
4479             smem_op = aco_opcode::last_opcode;
4480             assert(!smem);
4481             break;
4482          case 16:
4483             vmem_op = aco_opcode::buffer_store_dwordx4;
4484             smem_op = aco_opcode::s_buffer_store_dwordx4;
4485             break;
4486          default:
4487             unreachable("Store SSBO not implemented for this size.");
4488       }
4489       if (ctx->stage == fragment_fs)
4490          smem_op = aco_opcode::p_fs_buffer_store_smem;
4491
4492       if (smem) {
4493          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4494          store->operands[0] = Operand(rsrc);
4495          if (start) {
4496             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4497                                 offset, Operand(start * elem_size_bytes));
4498             store->operands[1] = Operand(off);
4499          } else {
4500             store->operands[1] = Operand(offset);
4501          }
4502          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4503             store->operands[1].setFixed(m0);
4504          store->operands[2] = Operand(write_data);
4505          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4506          store->dlc = false;
4507          store->disable_wqm = true;
4508          store->barrier = barrier_buffer;
4509          ctx->block->instructions.emplace_back(std::move(store));
4510          ctx->program->wb_smem_l1_on_end = true;
4511          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4512             ctx->block->kind |= block_kind_needs_lowering;
4513             ctx->program->needs_exact = true;
4514          }
4515       } else {
4516          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4517          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4518          store->operands[1] = Operand(rsrc);
4519          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4520          store->operands[3] = Operand(write_data);
4521          store->offset = start * elem_size_bytes;
4522          store->offen = (offset.type() == RegType::vgpr);
4523          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4524          store->dlc = false;
4525          store->disable_wqm = true;
4526          store->barrier = barrier_buffer;
4527          ctx->program->needs_exact = true;
4528          ctx->block->instructions.emplace_back(std::move(store));
4529       }
4530    }
4531 }
4532
4533 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4534 {
4535    /* return the previous value if dest is ever used */
4536    bool return_previous = false;
4537    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4538       return_previous = true;
4539       break;
4540    }
4541    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4542       return_previous = true;
4543       break;
4544    }
4545
4546    Builder bld(ctx->program, ctx->block);
4547    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4548
4549    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4550       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4551                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4552
4553    Temp offset;
4554    if (ctx->options->chip_class < GFX8)
4555       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4556    else
4557       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4558
4559    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4560    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4561
4562    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4563
4564    aco_opcode op32, op64;
4565    switch (instr->intrinsic) {
4566       case nir_intrinsic_ssbo_atomic_add:
4567          op32 = aco_opcode::buffer_atomic_add;
4568          op64 = aco_opcode::buffer_atomic_add_x2;
4569          break;
4570       case nir_intrinsic_ssbo_atomic_imin:
4571          op32 = aco_opcode::buffer_atomic_smin;
4572          op64 = aco_opcode::buffer_atomic_smin_x2;
4573          break;
4574       case nir_intrinsic_ssbo_atomic_umin:
4575          op32 = aco_opcode::buffer_atomic_umin;
4576          op64 = aco_opcode::buffer_atomic_umin_x2;
4577          break;
4578       case nir_intrinsic_ssbo_atomic_imax:
4579          op32 = aco_opcode::buffer_atomic_smax;
4580          op64 = aco_opcode::buffer_atomic_smax_x2;
4581          break;
4582       case nir_intrinsic_ssbo_atomic_umax:
4583          op32 = aco_opcode::buffer_atomic_umax;
4584          op64 = aco_opcode::buffer_atomic_umax_x2;
4585          break;
4586       case nir_intrinsic_ssbo_atomic_and:
4587          op32 = aco_opcode::buffer_atomic_and;
4588          op64 = aco_opcode::buffer_atomic_and_x2;
4589          break;
4590       case nir_intrinsic_ssbo_atomic_or:
4591          op32 = aco_opcode::buffer_atomic_or;
4592          op64 = aco_opcode::buffer_atomic_or_x2;
4593          break;
4594       case nir_intrinsic_ssbo_atomic_xor:
4595          op32 = aco_opcode::buffer_atomic_xor;
4596          op64 = aco_opcode::buffer_atomic_xor_x2;
4597          break;
4598       case nir_intrinsic_ssbo_atomic_exchange:
4599          op32 = aco_opcode::buffer_atomic_swap;
4600          op64 = aco_opcode::buffer_atomic_swap_x2;
4601          break;
4602       case nir_intrinsic_ssbo_atomic_comp_swap:
4603          op32 = aco_opcode::buffer_atomic_cmpswap;
4604          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4605          break;
4606       default:
4607          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4608    }
4609    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4610    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4611    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4612    mubuf->operands[1] = Operand(rsrc);
4613    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4614    mubuf->operands[3] = Operand(data);
4615    if (return_previous)
4616       mubuf->definitions[0] = Definition(dst);
4617    mubuf->offset = 0;
4618    mubuf->offen = (offset.type() == RegType::vgpr);
4619    mubuf->glc = return_previous;
4620    mubuf->dlc = false; /* Not needed for atomics */
4621    mubuf->disable_wqm = true;
4622    mubuf->barrier = barrier_buffer;
4623    ctx->program->needs_exact = true;
4624    ctx->block->instructions.emplace_back(std::move(mubuf));
4625 }
4626
4627 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4628
4629    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4630    Builder bld(ctx->program, ctx->block);
4631    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4632    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4633 }
4634
4635 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4636 {
4637    Builder bld(ctx->program, ctx->block);
4638    unsigned num_components = instr->num_components;
4639    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4640
4641    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4642    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4643
4644    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4645    bool dlc = glc && ctx->options->chip_class >= GFX10;
4646    aco_opcode op;
4647    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4648       bool global = ctx->options->chip_class >= GFX9;
4649       aco_opcode op;
4650       switch (num_bytes) {
4651       case 4:
4652          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4653          break;
4654       case 8:
4655          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4656          break;
4657       case 12:
4658          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4659          break;
4660       case 16:
4661          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4662          break;
4663       default:
4664          unreachable("load_global not implemented for this size.");
4665       }
4666       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4667       flat->operands[0] = Operand(addr);
4668       flat->operands[1] = Operand(s1);
4669       flat->glc = glc;
4670       flat->dlc = dlc;
4671       flat->barrier = barrier_buffer;
4672
4673       if (dst.type() == RegType::sgpr) {
4674          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4675          flat->definitions[0] = Definition(vec);
4676          ctx->block->instructions.emplace_back(std::move(flat));
4677          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4678       } else {
4679          flat->definitions[0] = Definition(dst);
4680          ctx->block->instructions.emplace_back(std::move(flat));
4681       }
4682       emit_split_vector(ctx, dst, num_components);
4683    } else {
4684       switch (num_bytes) {
4685          case 4:
4686             op = aco_opcode::s_load_dword;
4687             break;
4688          case 8:
4689             op = aco_opcode::s_load_dwordx2;
4690             break;
4691          case 12:
4692          case 16:
4693             op = aco_opcode::s_load_dwordx4;
4694             break;
4695          default:
4696             unreachable("load_global not implemented for this size.");
4697       }
4698       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4699       load->operands[0] = Operand(addr);
4700       load->operands[1] = Operand(0u);
4701       load->definitions[0] = Definition(dst);
4702       load->glc = glc;
4703       load->dlc = dlc;
4704       load->barrier = barrier_buffer;
4705       assert(ctx->options->chip_class >= GFX8 || !glc);
4706
4707       if (dst.size() == 3) {
4708          /* trim vector */
4709          Temp vec = bld.tmp(s4);
4710          load->definitions[0] = Definition(vec);
4711          ctx->block->instructions.emplace_back(std::move(load));
4712          emit_split_vector(ctx, vec, 4);
4713
4714          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4715                     emit_extract_vector(ctx, vec, 0, s1),
4716                     emit_extract_vector(ctx, vec, 1, s1),
4717                     emit_extract_vector(ctx, vec, 2, s1));
4718       } else {
4719          ctx->block->instructions.emplace_back(std::move(load));
4720       }
4721    }
4722 }
4723
4724 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4725 {
4726    Builder bld(ctx->program, ctx->block);
4727    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4728
4729    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4730    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4731
4732    unsigned writemask = nir_intrinsic_write_mask(instr);
4733    while (writemask) {
4734       int start, count;
4735       u_bit_scan_consecutive_range(&writemask, &start, &count);
4736       unsigned num_bytes = count * elem_size_bytes;
4737
4738       Temp write_data = data;
4739       if (count != instr->num_components) {
4740          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4741          for (int i = 0; i < count; i++)
4742             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4743          write_data = bld.tmp(RegType::vgpr, count);
4744          vec->definitions[0] = Definition(write_data);
4745          ctx->block->instructions.emplace_back(std::move(vec));
4746       }
4747
4748       unsigned offset = start * elem_size_bytes;
4749       if (offset > 0 && ctx->options->chip_class < GFX9) {
4750          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4751          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4752          Temp carry = bld.tmp(bld.lm);
4753          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4754
4755          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4756                   Operand(offset), addr0);
4757          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4758                   Operand(0u), addr1,
4759                   carry).def(1).setHint(vcc);
4760
4761          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4762
4763          offset = 0;
4764       }
4765
4766       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4767       bool global = ctx->options->chip_class >= GFX9;
4768       aco_opcode op;
4769       switch (num_bytes) {
4770       case 4:
4771          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4772          break;
4773       case 8:
4774          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4775          break;
4776       case 12:
4777          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4778          break;
4779       case 16:
4780          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4781          break;
4782       default:
4783          unreachable("store_global not implemented for this size.");
4784       }
4785       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4786       flat->operands[0] = Operand(addr);
4787       flat->operands[1] = Operand(s1);
4788       flat->operands[2] = Operand(data);
4789       flat->glc = glc;
4790       flat->dlc = false;
4791       flat->offset = offset;
4792       flat->disable_wqm = true;
4793       flat->barrier = barrier_buffer;
4794       ctx->program->needs_exact = true;
4795       ctx->block->instructions.emplace_back(std::move(flat));
4796    }
4797 }
4798
4799 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4800 {
4801    /* return the previous value if dest is ever used */
4802    bool return_previous = false;
4803    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4804       return_previous = true;
4805       break;
4806    }
4807    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4808       return_previous = true;
4809       break;
4810    }
4811
4812    Builder bld(ctx->program, ctx->block);
4813    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4814    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4815
4816    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4817       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4818                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4819
4820    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4821
4822    bool global = ctx->options->chip_class >= GFX9;
4823    aco_opcode op32, op64;
4824    switch (instr->intrinsic) {
4825       case nir_intrinsic_global_atomic_add:
4826          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4827          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4828          break;
4829       case nir_intrinsic_global_atomic_imin:
4830          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4831          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4832          break;
4833       case nir_intrinsic_global_atomic_umin:
4834          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4835          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4836          break;
4837       case nir_intrinsic_global_atomic_imax:
4838          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4839          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4840          break;
4841       case nir_intrinsic_global_atomic_umax:
4842          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4843          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4844          break;
4845       case nir_intrinsic_global_atomic_and:
4846          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4847          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4848          break;
4849       case nir_intrinsic_global_atomic_or:
4850          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4851          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4852          break;
4853       case nir_intrinsic_global_atomic_xor:
4854          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4855          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4856          break;
4857       case nir_intrinsic_global_atomic_exchange:
4858          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4859          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4860          break;
4861       case nir_intrinsic_global_atomic_comp_swap:
4862          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4863          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4864          break;
4865       default:
4866          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4867    }
4868    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4869    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4870    flat->operands[0] = Operand(addr);
4871    flat->operands[1] = Operand(s1);
4872    flat->operands[2] = Operand(data);
4873    if (return_previous)
4874       flat->definitions[0] = Definition(dst);
4875    flat->glc = return_previous;
4876    flat->dlc = false; /* Not needed for atomics */
4877    flat->offset = 0;
4878    flat->disable_wqm = true;
4879    flat->barrier = barrier_buffer;
4880    ctx->program->needs_exact = true;
4881    ctx->block->instructions.emplace_back(std::move(flat));
4882 }
4883
4884 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4885    Builder bld(ctx->program, ctx->block);
4886    switch(instr->intrinsic) {
4887       case nir_intrinsic_group_memory_barrier:
4888       case nir_intrinsic_memory_barrier:
4889          bld.barrier(aco_opcode::p_memory_barrier_all);
4890          break;
4891       case nir_intrinsic_memory_barrier_atomic_counter:
4892          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4893          break;
4894       case nir_intrinsic_memory_barrier_buffer:
4895          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4896          break;
4897       case nir_intrinsic_memory_barrier_image:
4898          bld.barrier(aco_opcode::p_memory_barrier_image);
4899          break;
4900       case nir_intrinsic_memory_barrier_shared:
4901          bld.barrier(aco_opcode::p_memory_barrier_shared);
4902          break;
4903       default:
4904          unreachable("Unimplemented memory barrier intrinsic");
4905          break;
4906    }
4907 }
4908
4909 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4910 {
4911    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4912    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4913    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4914    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4915    Builder bld(ctx->program, ctx->block);
4916
4917    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4918    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4919    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4920 }
4921
4922 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4923 {
4924    unsigned writemask = nir_intrinsic_write_mask(instr);
4925    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4926    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4927    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4928    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4929
4930    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4931    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4932 }
4933
4934 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4935 {
4936    unsigned offset = nir_intrinsic_base(instr);
4937    Operand m = load_lds_size_m0(ctx);
4938    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4939    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4940
4941    unsigned num_operands = 3;
4942    aco_opcode op32, op64, op32_rtn, op64_rtn;
4943    switch(instr->intrinsic) {
4944       case nir_intrinsic_shared_atomic_add:
4945          op32 = aco_opcode::ds_add_u32;
4946          op64 = aco_opcode::ds_add_u64;
4947          op32_rtn = aco_opcode::ds_add_rtn_u32;
4948          op64_rtn = aco_opcode::ds_add_rtn_u64;
4949          break;
4950       case nir_intrinsic_shared_atomic_imin:
4951          op32 = aco_opcode::ds_min_i32;
4952          op64 = aco_opcode::ds_min_i64;
4953          op32_rtn = aco_opcode::ds_min_rtn_i32;
4954          op64_rtn = aco_opcode::ds_min_rtn_i64;
4955          break;
4956       case nir_intrinsic_shared_atomic_umin:
4957          op32 = aco_opcode::ds_min_u32;
4958          op64 = aco_opcode::ds_min_u64;
4959          op32_rtn = aco_opcode::ds_min_rtn_u32;
4960          op64_rtn = aco_opcode::ds_min_rtn_u64;
4961          break;
4962       case nir_intrinsic_shared_atomic_imax:
4963          op32 = aco_opcode::ds_max_i32;
4964          op64 = aco_opcode::ds_max_i64;
4965          op32_rtn = aco_opcode::ds_max_rtn_i32;
4966          op64_rtn = aco_opcode::ds_max_rtn_i64;
4967          break;
4968       case nir_intrinsic_shared_atomic_umax:
4969          op32 = aco_opcode::ds_max_u32;
4970          op64 = aco_opcode::ds_max_u64;
4971          op32_rtn = aco_opcode::ds_max_rtn_u32;
4972          op64_rtn = aco_opcode::ds_max_rtn_u64;
4973          break;
4974       case nir_intrinsic_shared_atomic_and:
4975          op32 = aco_opcode::ds_and_b32;
4976          op64 = aco_opcode::ds_and_b64;
4977          op32_rtn = aco_opcode::ds_and_rtn_b32;
4978          op64_rtn = aco_opcode::ds_and_rtn_b64;
4979          break;
4980       case nir_intrinsic_shared_atomic_or:
4981          op32 = aco_opcode::ds_or_b32;
4982          op64 = aco_opcode::ds_or_b64;
4983          op32_rtn = aco_opcode::ds_or_rtn_b32;
4984          op64_rtn = aco_opcode::ds_or_rtn_b64;
4985          break;
4986       case nir_intrinsic_shared_atomic_xor:
4987          op32 = aco_opcode::ds_xor_b32;
4988          op64 = aco_opcode::ds_xor_b64;
4989          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4990          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4991          break;
4992       case nir_intrinsic_shared_atomic_exchange:
4993          op32 = aco_opcode::ds_write_b32;
4994          op64 = aco_opcode::ds_write_b64;
4995          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4996          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4997          break;
4998       case nir_intrinsic_shared_atomic_comp_swap:
4999          op32 = aco_opcode::ds_cmpst_b32;
5000          op64 = aco_opcode::ds_cmpst_b64;
5001          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5002          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5003          num_operands = 4;
5004          break;
5005       default:
5006          unreachable("Unhandled shared atomic intrinsic");
5007    }
5008
5009    /* return the previous value if dest is ever used */
5010    bool return_previous = false;
5011    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5012       return_previous = true;
5013       break;
5014    }
5015    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5016       return_previous = true;
5017       break;
5018    }
5019
5020    aco_opcode op;
5021    if (data.size() == 1) {
5022       assert(instr->dest.ssa.bit_size == 32);
5023       op = return_previous ? op32_rtn : op32;
5024    } else {
5025       assert(instr->dest.ssa.bit_size == 64);
5026       op = return_previous ? op64_rtn : op64;
5027    }
5028
5029    if (offset > 65535) {
5030       Builder bld(ctx->program, ctx->block);
5031       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5032       offset = 0;
5033    }
5034
5035    aco_ptr<DS_instruction> ds;
5036    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5037    ds->operands[0] = Operand(address);
5038    ds->operands[1] = Operand(data);
5039    if (num_operands == 4)
5040       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5041    ds->operands[num_operands - 1] = m;
5042    ds->offset0 = offset;
5043    if (return_previous)
5044       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5045    ctx->block->instructions.emplace_back(std::move(ds));
5046 }
5047
5048 Temp get_scratch_resource(isel_context *ctx)
5049 {
5050    Builder bld(ctx->program, ctx->block);
5051    Temp scratch_addr = ctx->program->private_segment_buffer;
5052    if (ctx->stage != compute_cs)
5053       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5054
5055    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5056                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5057
5058    if (ctx->program->chip_class >= GFX10) {
5059       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5060                    S_008F0C_OOB_SELECT(3) |
5061                    S_008F0C_RESOURCE_LEVEL(1);
5062    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5063       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5064                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5065    }
5066
5067    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5068    if (ctx->program->chip_class <= GFX8)
5069       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5070
5071    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5072 }
5073
5074 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5075    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5076    Builder bld(ctx->program, ctx->block);
5077    Temp rsrc = get_scratch_resource(ctx);
5078    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5079    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5080
5081    aco_opcode op;
5082    switch (dst.size()) {
5083       case 1:
5084          op = aco_opcode::buffer_load_dword;
5085          break;
5086       case 2:
5087          op = aco_opcode::buffer_load_dwordx2;
5088          break;
5089       case 3:
5090          op = aco_opcode::buffer_load_dwordx3;
5091          break;
5092       case 4:
5093          op = aco_opcode::buffer_load_dwordx4;
5094          break;
5095       case 6:
5096       case 8: {
5097          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5098          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5099                                 bld.def(v4), offset, rsrc,
5100                                 ctx->program->scratch_offset, 0, true);
5101          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5102                                                   aco_opcode::buffer_load_dwordx4,
5103                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5104                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5105          emit_split_vector(ctx, lower, 2);
5106          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5107          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5108          if (dst.size() == 8) {
5109             emit_split_vector(ctx, upper, 2);
5110             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5111             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5112          } else {
5113             elems[2] = upper;
5114          }
5115
5116          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5117                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5118          for (unsigned i = 0; i < dst.size() / 2; i++)
5119             vec->operands[i] = Operand(elems[i]);
5120          vec->definitions[0] = Definition(dst);
5121          bld.insert(std::move(vec));
5122          ctx->allocated_vec.emplace(dst.id(), elems);
5123          return;
5124       }
5125       default:
5126          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5127    }
5128
5129    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5130    emit_split_vector(ctx, dst, instr->num_components);
5131 }
5132
5133 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5134    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5135    Builder bld(ctx->program, ctx->block);
5136    Temp rsrc = get_scratch_resource(ctx);
5137    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5138    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5139
5140    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5141    unsigned writemask = nir_intrinsic_write_mask(instr);
5142
5143    while (writemask) {
5144       int start, count;
5145       u_bit_scan_consecutive_range(&writemask, &start, &count);
5146       int num_bytes = count * elem_size_bytes;
5147
5148       if (num_bytes > 16) {
5149          assert(elem_size_bytes == 8);
5150          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5151          count = 2;
5152          num_bytes = 16;
5153       }
5154
5155       // TODO: check alignment of sub-dword stores
5156       // TODO: split 3 bytes. there is no store instruction for that
5157
5158       Temp write_data;
5159       if (count != instr->num_components) {
5160          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5161          for (int i = 0; i < count; i++) {
5162             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5163             vec->operands[i] = Operand(elem);
5164          }
5165          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5166          vec->definitions[0] = Definition(write_data);
5167          ctx->block->instructions.emplace_back(std::move(vec));
5168       } else {
5169          write_data = data;
5170       }
5171
5172       aco_opcode op;
5173       switch (num_bytes) {
5174          case 4:
5175             op = aco_opcode::buffer_store_dword;
5176             break;
5177          case 8:
5178             op = aco_opcode::buffer_store_dwordx2;
5179             break;
5180          case 12:
5181             op = aco_opcode::buffer_store_dwordx3;
5182             break;
5183          case 16:
5184             op = aco_opcode::buffer_store_dwordx4;
5185             break;
5186          default:
5187             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5188       }
5189
5190       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5191    }
5192 }
5193
5194 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5195    uint8_t log2_ps_iter_samples;
5196    if (ctx->program->info->ps.force_persample) {
5197       log2_ps_iter_samples =
5198          util_logbase2(ctx->options->key.fs.num_samples);
5199    } else {
5200       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5201    }
5202
5203    /* The bit pattern matches that used by fixed function fragment
5204     * processing. */
5205    static const unsigned ps_iter_masks[] = {
5206       0xffff, /* not used */
5207       0x5555,
5208       0x1111,
5209       0x0101,
5210       0x0001,
5211    };
5212    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5213
5214    Builder bld(ctx->program, ctx->block);
5215
5216    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5217                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5218    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5219    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5220    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5221    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5222 }
5223
5224 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5225 {
5226    Builder bld(ctx->program, ctx->block);
5227
5228    if (cluster_size == 1) {
5229       return src;
5230    } if (op == nir_op_iand && cluster_size == 4) {
5231       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5232       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5233       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5234                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5235    } else if (op == nir_op_ior && cluster_size == 4) {
5236       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5237       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5238                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5239    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5240       //subgroupAnd(val) -> (exec & ~val) == 0
5241       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5242       return bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5243    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5244       //subgroupOr(val) -> (val & exec) != 0
5245       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5246       return bool_to_vector_condition(ctx, tmp);
5247    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5248       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5249       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5250       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5251       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5252       return bool_to_vector_condition(ctx, tmp);
5253    } else {
5254       //subgroupClustered{And,Or,Xor}(val, n) ->
5255       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5256       //cluster_offset = ~(n - 1) & lane_id
5257       //cluster_mask = ((1 << n) - 1)
5258       //subgroupClusteredAnd():
5259       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5260       //subgroupClusteredOr():
5261       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5262       //subgroupClusteredXor():
5263       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5264       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5265       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5266
5267       Temp tmp;
5268       if (op == nir_op_iand)
5269          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5270       else
5271          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5272
5273       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5274       if (ctx->program->wave_size == 64)
5275          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5276       else
5277          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5278       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5279       if (cluster_mask != 0xffffffff)
5280          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5281
5282       Definition cmp_def = Definition();
5283       if (op == nir_op_iand) {
5284          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5285       } else if (op == nir_op_ior) {
5286          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5287       } else if (op == nir_op_ixor) {
5288          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5289                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5290          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5291       }
5292       cmp_def.setHint(vcc);
5293       return cmp_def.getTemp();
5294    }
5295 }
5296
5297 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5298 {
5299    Builder bld(ctx->program, ctx->block);
5300
5301    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5302    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5303    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5304    Temp tmp;
5305    if (op == nir_op_iand)
5306       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5307    else
5308       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5309
5310    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5311    Temp lo = lohi.def(0).getTemp();
5312    Temp hi = lohi.def(1).getTemp();
5313    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5314
5315    Definition cmp_def = Definition();
5316    if (op == nir_op_iand)
5317       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5318    else if (op == nir_op_ior)
5319       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5320    else if (op == nir_op_ixor)
5321       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5322                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5323    cmp_def.setHint(vcc);
5324    return cmp_def.getTemp();
5325 }
5326
5327 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5328 {
5329    Builder bld(ctx->program, ctx->block);
5330
5331    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5332    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5333    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5334    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5335    if (op == nir_op_iand)
5336       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5337    else if (op == nir_op_ior)
5338       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5339    else if (op == nir_op_ixor)
5340       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5341
5342    assert(false);
5343    return Temp();
5344 }
5345
5346 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5347 {
5348    Builder bld(ctx->program, ctx->block);
5349    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5350    if (src.regClass().type() == RegType::vgpr) {
5351       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5352    } else if (src.regClass() == s1) {
5353       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5354    } else if (src.regClass() == s2) {
5355       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5356    } else {
5357       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5358       nir_print_instr(&instr->instr, stderr);
5359       fprintf(stderr, "\n");
5360    }
5361 }
5362
5363 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5364 {
5365    Builder bld(ctx->program, ctx->block);
5366    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5367    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5368    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5369
5370    /* Build DD X/Y */
5371    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5372    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5373    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5374    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5375    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5376    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5377
5378    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5379    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5380    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5381    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5382    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5383    Temp wqm1 = bld.tmp(v1);
5384    emit_wqm(ctx, tmp1, wqm1, true);
5385    Temp wqm2 = bld.tmp(v1);
5386    emit_wqm(ctx, tmp2, wqm2, true);
5387    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5388    return;
5389 }
5390
5391 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5392 {
5393    Builder bld(ctx->program, ctx->block);
5394    switch(instr->intrinsic) {
5395    case nir_intrinsic_load_barycentric_sample:
5396    case nir_intrinsic_load_barycentric_pixel:
5397    case nir_intrinsic_load_barycentric_centroid: {
5398       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5399       Temp bary = Temp(0, s2);
5400       switch (mode) {
5401       case INTERP_MODE_SMOOTH:
5402       case INTERP_MODE_NONE:
5403          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5404             bary = get_arg(ctx, ctx->args->ac.persp_center);
5405          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5406             bary = ctx->persp_centroid;
5407          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5408             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5409          break;
5410       case INTERP_MODE_NOPERSPECTIVE:
5411          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5412             bary = get_arg(ctx, ctx->args->ac.linear_center);
5413          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5414             bary = ctx->linear_centroid;
5415          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5416             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5417          break;
5418       default:
5419          break;
5420       }
5421       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5422       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5423       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5424       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5425                  Operand(p1), Operand(p2));
5426       emit_split_vector(ctx, dst, 2);
5427       break;
5428    }
5429    case nir_intrinsic_load_barycentric_at_sample: {
5430       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5431       switch (ctx->options->key.fs.num_samples) {
5432          case 2: sample_pos_offset += 1 << 3; break;
5433          case 4: sample_pos_offset += 3 << 3; break;
5434          case 8: sample_pos_offset += 7 << 3; break;
5435          default: break;
5436       }
5437       Temp sample_pos;
5438       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5439       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5440       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5441       if (addr.type() == RegType::sgpr) {
5442          Operand offset;
5443          if (const_addr) {
5444             sample_pos_offset += const_addr->u32 << 3;
5445             offset = Operand(sample_pos_offset);
5446          } else if (ctx->options->chip_class >= GFX9) {
5447             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5448          } else {
5449             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5450             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5451          }
5452          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5453
5454       } else if (ctx->options->chip_class >= GFX9) {
5455          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5456          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5457       } else {
5458          /* addr += private_segment_buffer + sample_pos_offset */
5459          Temp tmp0 = bld.tmp(s1);
5460          Temp tmp1 = bld.tmp(s1);
5461          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5462          Definition scc_tmp = bld.def(s1, scc);
5463          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5464          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5465          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5466          Temp pck0 = bld.tmp(v1);
5467          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5468          tmp1 = as_vgpr(ctx, tmp1);
5469          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5470          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5471
5472          /* sample_pos = flat_load_dwordx2 addr */
5473          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5474       }
5475
5476       /* sample_pos -= 0.5 */
5477       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5478       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5479       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5480       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5481       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5482
5483       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5484       break;
5485    }
5486    case nir_intrinsic_load_barycentric_at_offset: {
5487       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5488       RegClass rc = RegClass(offset.type(), 1);
5489       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5490       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5491       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5492       break;
5493    }
5494    case nir_intrinsic_load_front_face: {
5495       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5496                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5497       break;
5498    }
5499    case nir_intrinsic_load_view_index:
5500    case nir_intrinsic_load_layer_id: {
5501       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5502          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5503          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5504          break;
5505       }
5506
5507       unsigned idx = nir_intrinsic_base(instr);
5508       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5509                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5510       break;
5511    }
5512    case nir_intrinsic_load_frag_coord: {
5513       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5514       break;
5515    }
5516    case nir_intrinsic_load_sample_pos: {
5517       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5518       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5519       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5520                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5521                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5522       break;
5523    }
5524    case nir_intrinsic_load_interpolated_input:
5525       visit_load_interpolated_input(ctx, instr);
5526       break;
5527    case nir_intrinsic_store_output:
5528       visit_store_output(ctx, instr);
5529       break;
5530    case nir_intrinsic_load_input:
5531       visit_load_input(ctx, instr);
5532       break;
5533    case nir_intrinsic_load_ubo:
5534       visit_load_ubo(ctx, instr);
5535       break;
5536    case nir_intrinsic_load_push_constant:
5537       visit_load_push_constant(ctx, instr);
5538       break;
5539    case nir_intrinsic_load_constant:
5540       visit_load_constant(ctx, instr);
5541       break;
5542    case nir_intrinsic_vulkan_resource_index:
5543       visit_load_resource(ctx, instr);
5544       break;
5545    case nir_intrinsic_discard:
5546       visit_discard(ctx, instr);
5547       break;
5548    case nir_intrinsic_discard_if:
5549       visit_discard_if(ctx, instr);
5550       break;
5551    case nir_intrinsic_load_shared:
5552       visit_load_shared(ctx, instr);
5553       break;
5554    case nir_intrinsic_store_shared:
5555       visit_store_shared(ctx, instr);
5556       break;
5557    case nir_intrinsic_shared_atomic_add:
5558    case nir_intrinsic_shared_atomic_imin:
5559    case nir_intrinsic_shared_atomic_umin:
5560    case nir_intrinsic_shared_atomic_imax:
5561    case nir_intrinsic_shared_atomic_umax:
5562    case nir_intrinsic_shared_atomic_and:
5563    case nir_intrinsic_shared_atomic_or:
5564    case nir_intrinsic_shared_atomic_xor:
5565    case nir_intrinsic_shared_atomic_exchange:
5566    case nir_intrinsic_shared_atomic_comp_swap:
5567       visit_shared_atomic(ctx, instr);
5568       break;
5569    case nir_intrinsic_image_deref_load:
5570       visit_image_load(ctx, instr);
5571       break;
5572    case nir_intrinsic_image_deref_store:
5573       visit_image_store(ctx, instr);
5574       break;
5575    case nir_intrinsic_image_deref_atomic_add:
5576    case nir_intrinsic_image_deref_atomic_umin:
5577    case nir_intrinsic_image_deref_atomic_imin:
5578    case nir_intrinsic_image_deref_atomic_umax:
5579    case nir_intrinsic_image_deref_atomic_imax:
5580    case nir_intrinsic_image_deref_atomic_and:
5581    case nir_intrinsic_image_deref_atomic_or:
5582    case nir_intrinsic_image_deref_atomic_xor:
5583    case nir_intrinsic_image_deref_atomic_exchange:
5584    case nir_intrinsic_image_deref_atomic_comp_swap:
5585       visit_image_atomic(ctx, instr);
5586       break;
5587    case nir_intrinsic_image_deref_size:
5588       visit_image_size(ctx, instr);
5589       break;
5590    case nir_intrinsic_load_ssbo:
5591       visit_load_ssbo(ctx, instr);
5592       break;
5593    case nir_intrinsic_store_ssbo:
5594       visit_store_ssbo(ctx, instr);
5595       break;
5596    case nir_intrinsic_load_global:
5597       visit_load_global(ctx, instr);
5598       break;
5599    case nir_intrinsic_store_global:
5600       visit_store_global(ctx, instr);
5601       break;
5602    case nir_intrinsic_global_atomic_add:
5603    case nir_intrinsic_global_atomic_imin:
5604    case nir_intrinsic_global_atomic_umin:
5605    case nir_intrinsic_global_atomic_imax:
5606    case nir_intrinsic_global_atomic_umax:
5607    case nir_intrinsic_global_atomic_and:
5608    case nir_intrinsic_global_atomic_or:
5609    case nir_intrinsic_global_atomic_xor:
5610    case nir_intrinsic_global_atomic_exchange:
5611    case nir_intrinsic_global_atomic_comp_swap:
5612       visit_global_atomic(ctx, instr);
5613       break;
5614    case nir_intrinsic_ssbo_atomic_add:
5615    case nir_intrinsic_ssbo_atomic_imin:
5616    case nir_intrinsic_ssbo_atomic_umin:
5617    case nir_intrinsic_ssbo_atomic_imax:
5618    case nir_intrinsic_ssbo_atomic_umax:
5619    case nir_intrinsic_ssbo_atomic_and:
5620    case nir_intrinsic_ssbo_atomic_or:
5621    case nir_intrinsic_ssbo_atomic_xor:
5622    case nir_intrinsic_ssbo_atomic_exchange:
5623    case nir_intrinsic_ssbo_atomic_comp_swap:
5624       visit_atomic_ssbo(ctx, instr);
5625       break;
5626    case nir_intrinsic_load_scratch:
5627       visit_load_scratch(ctx, instr);
5628       break;
5629    case nir_intrinsic_store_scratch:
5630       visit_store_scratch(ctx, instr);
5631       break;
5632    case nir_intrinsic_get_buffer_size:
5633       visit_get_buffer_size(ctx, instr);
5634       break;
5635    case nir_intrinsic_barrier: {
5636       unsigned* bsize = ctx->program->info->cs.block_size;
5637       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5638       if (workgroup_size > ctx->program->wave_size)
5639          bld.sopp(aco_opcode::s_barrier);
5640       break;
5641    }
5642    case nir_intrinsic_group_memory_barrier:
5643    case nir_intrinsic_memory_barrier:
5644    case nir_intrinsic_memory_barrier_atomic_counter:
5645    case nir_intrinsic_memory_barrier_buffer:
5646    case nir_intrinsic_memory_barrier_image:
5647    case nir_intrinsic_memory_barrier_shared:
5648       emit_memory_barrier(ctx, instr);
5649       break;
5650    case nir_intrinsic_load_num_work_groups: {
5651       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5652       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5653       emit_split_vector(ctx, dst, 3);
5654       break;
5655    }
5656    case nir_intrinsic_load_local_invocation_id: {
5657       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5658       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5659       emit_split_vector(ctx, dst, 3);
5660       break;
5661    }
5662    case nir_intrinsic_load_work_group_id: {
5663       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5664       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5665       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5666                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5667                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5668                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5669       emit_split_vector(ctx, dst, 3);
5670       break;
5671    }
5672    case nir_intrinsic_load_local_invocation_index: {
5673       Temp id = emit_mbcnt(ctx, bld.def(v1));
5674
5675       /* The tg_size bits [6:11] contain the subgroup id,
5676        * we need this multiplied by the wave size, and then OR the thread id to it.
5677        */
5678       if (ctx->program->wave_size == 64) {
5679          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5680          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5681                                 get_arg(ctx, ctx->args->ac.tg_size));
5682          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5683       } else {
5684          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5685          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5686                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5687          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5688       }
5689       break;
5690    }
5691    case nir_intrinsic_load_subgroup_id: {
5692       if (ctx->stage == compute_cs) {
5693          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5694                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5695       } else {
5696          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5697       }
5698       break;
5699    }
5700    case nir_intrinsic_load_subgroup_invocation: {
5701       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5702       break;
5703    }
5704    case nir_intrinsic_load_num_subgroups: {
5705       if (ctx->stage == compute_cs)
5706          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5707                   get_arg(ctx, ctx->args->ac.tg_size));
5708       else
5709          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5710       break;
5711    }
5712    case nir_intrinsic_ballot: {
5713       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5714       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5715       Definition tmp = bld.def(dst.regClass());
5716       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5717       if (instr->src[0].ssa->bit_size == 1) {
5718          assert(src.regClass() == bld.lm);
5719          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5720       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5721          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5722       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5723          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5724       } else {
5725          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5726          nir_print_instr(&instr->instr, stderr);
5727          fprintf(stderr, "\n");
5728       }
5729       if (dst.size() != bld.lm.size()) {
5730          /* Wave32 with ballot size set to 64 */
5731          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5732       }
5733       emit_wqm(ctx, tmp.getTemp(), dst);
5734       break;
5735    }
5736    case nir_intrinsic_shuffle:
5737    case nir_intrinsic_read_invocation: {
5738       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5739       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5740          emit_uniform_subgroup(ctx, instr, src);
5741       } else {
5742          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5743          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5744             tid = bld.as_uniform(tid);
5745          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5746          if (src.regClass() == v1) {
5747             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5748          } else if (src.regClass() == v2) {
5749             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5750             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5751             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5752             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5753             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5754             emit_split_vector(ctx, dst, 2);
5755          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5756             assert(src.regClass() == bld.lm);
5757             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5758             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5759          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5760             assert(src.regClass() == bld.lm);
5761             Temp tmp;
5762             if (ctx->program->wave_size == 64)
5763                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5764             else
5765                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5766             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5767             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5768             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5769          } else {
5770             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5771             nir_print_instr(&instr->instr, stderr);
5772             fprintf(stderr, "\n");
5773          }
5774       }
5775       break;
5776    }
5777    case nir_intrinsic_load_sample_id: {
5778       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5779                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5780       break;
5781    }
5782    case nir_intrinsic_load_sample_mask_in: {
5783       visit_load_sample_mask_in(ctx, instr);
5784       break;
5785    }
5786    case nir_intrinsic_read_first_invocation: {
5787       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5788       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5789       if (src.regClass() == v1) {
5790          emit_wqm(ctx,
5791                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5792                   dst);
5793       } else if (src.regClass() == v2) {
5794          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5795          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5796          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5797          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5798          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5799          emit_split_vector(ctx, dst, 2);
5800       } else if (instr->dest.ssa.bit_size == 1) {
5801          assert(src.regClass() == bld.lm);
5802          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5803                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5804          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5805       } else if (src.regClass() == s1) {
5806          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5807       } else if (src.regClass() == s2) {
5808          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5809       } else {
5810          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5811          nir_print_instr(&instr->instr, stderr);
5812          fprintf(stderr, "\n");
5813       }
5814       break;
5815    }
5816    case nir_intrinsic_vote_all: {
5817       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5818       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5819       assert(src.regClass() == bld.lm);
5820       assert(dst.regClass() == bld.lm);
5821
5822       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5823       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5824       emit_wqm(ctx, val, dst);
5825       break;
5826    }
5827    case nir_intrinsic_vote_any: {
5828       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5829       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5830       assert(src.regClass() == bld.lm);
5831       assert(dst.regClass() == bld.lm);
5832
5833       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5834       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), Operand(0u), bld.scc(tmp));
5835       emit_wqm(ctx, val, dst);
5836       break;
5837    }
5838    case nir_intrinsic_reduce:
5839    case nir_intrinsic_inclusive_scan:
5840    case nir_intrinsic_exclusive_scan: {
5841       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5842       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5843       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5844       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5845          nir_intrinsic_cluster_size(instr) : 0;
5846       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5847
5848       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5849          emit_uniform_subgroup(ctx, instr, src);
5850       } else if (instr->dest.ssa.bit_size == 1) {
5851          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5852             op = nir_op_iand;
5853          else if (op == nir_op_iadd)
5854             op = nir_op_ixor;
5855          else if (op == nir_op_umax || op == nir_op_imax)
5856             op = nir_op_ior;
5857          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5858
5859          switch (instr->intrinsic) {
5860          case nir_intrinsic_reduce:
5861             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5862             break;
5863          case nir_intrinsic_exclusive_scan:
5864             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5865             break;
5866          case nir_intrinsic_inclusive_scan:
5867             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5868             break;
5869          default:
5870             assert(false);
5871          }
5872       } else if (cluster_size == 1) {
5873          bld.copy(Definition(dst), src);
5874       } else {
5875          src = as_vgpr(ctx, src);
5876
5877          ReduceOp reduce_op;
5878          switch (op) {
5879          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5880             CASE(iadd)
5881             CASE(imul)
5882             CASE(fadd)
5883             CASE(fmul)
5884             CASE(imin)
5885             CASE(umin)
5886             CASE(fmin)
5887             CASE(imax)
5888             CASE(umax)
5889             CASE(fmax)
5890             CASE(iand)
5891             CASE(ior)
5892             CASE(ixor)
5893             default:
5894                unreachable("unknown reduction op");
5895          #undef CASE
5896          }
5897
5898          aco_opcode aco_op;
5899          switch (instr->intrinsic) {
5900             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5901             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5902             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5903             default:
5904                unreachable("unknown reduce intrinsic");
5905          }
5906
5907          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5908          reduce->operands[0] = Operand(src);
5909          // filled in by aco_reduce_assign.cpp, used internally as part of the
5910          // reduce sequence
5911          assert(dst.size() == 1 || dst.size() == 2);
5912          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5913          reduce->operands[2] = Operand(v1.as_linear());
5914
5915          Temp tmp_dst = bld.tmp(dst.regClass());
5916          reduce->definitions[0] = Definition(tmp_dst);
5917          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
5918          reduce->definitions[2] = Definition();
5919          reduce->definitions[3] = Definition(scc, s1);
5920          reduce->definitions[4] = Definition();
5921          reduce->reduce_op = reduce_op;
5922          reduce->cluster_size = cluster_size;
5923          ctx->block->instructions.emplace_back(std::move(reduce));
5924
5925          emit_wqm(ctx, tmp_dst, dst);
5926       }
5927       break;
5928    }
5929    case nir_intrinsic_quad_broadcast: {
5930       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5931       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5932          emit_uniform_subgroup(ctx, instr, src);
5933       } else {
5934          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5935          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5936          if (instr->dest.ssa.bit_size == 1) {
5937             assert(src.regClass() == bld.lm);
5938             assert(dst.regClass() == bld.lm);
5939             uint32_t half_mask = 0x11111111u << lane;
5940             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5941             Temp tmp = bld.tmp(bld.lm);
5942             bld.sop1(Builder::s_wqm, Definition(tmp),
5943                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
5944                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
5945             emit_wqm(ctx, tmp, dst);
5946          } else if (instr->dest.ssa.bit_size == 32) {
5947             emit_wqm(ctx,
5948                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5949                                   dpp_quad_perm(lane, lane, lane, lane)),
5950                      dst);
5951          } else if (instr->dest.ssa.bit_size == 64) {
5952             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5953             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5954             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5955             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5956             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5957             emit_split_vector(ctx, dst, 2);
5958          } else {
5959             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5960             nir_print_instr(&instr->instr, stderr);
5961             fprintf(stderr, "\n");
5962          }
5963       }
5964       break;
5965    }
5966    case nir_intrinsic_quad_swap_horizontal:
5967    case nir_intrinsic_quad_swap_vertical:
5968    case nir_intrinsic_quad_swap_diagonal:
5969    case nir_intrinsic_quad_swizzle_amd: {
5970       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5971       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5972          emit_uniform_subgroup(ctx, instr, src);
5973          break;
5974       }
5975       uint16_t dpp_ctrl = 0;
5976       switch (instr->intrinsic) {
5977       case nir_intrinsic_quad_swap_horizontal:
5978          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5979          break;
5980       case nir_intrinsic_quad_swap_vertical:
5981          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5982          break;
5983       case nir_intrinsic_quad_swap_diagonal:
5984          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5985          break;
5986       case nir_intrinsic_quad_swizzle_amd: {
5987          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5988          break;
5989       }
5990       default:
5991          break;
5992       }
5993
5994       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5995       if (instr->dest.ssa.bit_size == 1) {
5996          assert(src.regClass() == bld.lm);
5997          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5998          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5999          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6000          emit_wqm(ctx, tmp, dst);
6001       } else if (instr->dest.ssa.bit_size == 32) {
6002          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6003          emit_wqm(ctx, tmp, dst);
6004       } else if (instr->dest.ssa.bit_size == 64) {
6005          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6006          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6007          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6008          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6009          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6010          emit_split_vector(ctx, dst, 2);
6011       } else {
6012          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6013          nir_print_instr(&instr->instr, stderr);
6014          fprintf(stderr, "\n");
6015       }
6016       break;
6017    }
6018    case nir_intrinsic_masked_swizzle_amd: {
6019       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6020       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6021          emit_uniform_subgroup(ctx, instr, src);
6022          break;
6023       }
6024       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6025       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6026       if (dst.regClass() == v1) {
6027          emit_wqm(ctx,
6028                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6029                   dst);
6030       } else if (dst.regClass() == v2) {
6031          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6032          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6033          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6034          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6035          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6036          emit_split_vector(ctx, dst, 2);
6037       } else {
6038          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6039          nir_print_instr(&instr->instr, stderr);
6040          fprintf(stderr, "\n");
6041       }
6042       break;
6043    }
6044    case nir_intrinsic_write_invocation_amd: {
6045       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6046       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6047       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6048       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6049       if (dst.regClass() == v1) {
6050          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6051          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
6052       } else if (dst.regClass() == v2) {
6053          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6054          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6055          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6056          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6057          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
6058          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
6059          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6060          emit_split_vector(ctx, dst, 2);
6061       } else {
6062          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6063          nir_print_instr(&instr->instr, stderr);
6064          fprintf(stderr, "\n");
6065       }
6066       break;
6067    }
6068    case nir_intrinsic_mbcnt_amd: {
6069       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6070       RegClass rc = RegClass(src.type(), 1);
6071       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6072       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6073       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6074       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6075       emit_wqm(ctx, wqm_tmp, dst);
6076       break;
6077    }
6078    case nir_intrinsic_load_helper_invocation: {
6079       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6080       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6081       ctx->block->kind |= block_kind_needs_lowering;
6082       ctx->program->needs_exact = true;
6083       break;
6084    }
6085    case nir_intrinsic_is_helper_invocation: {
6086       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6087       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6088       ctx->block->kind |= block_kind_needs_lowering;
6089       ctx->program->needs_exact = true;
6090       break;
6091    }
6092    case nir_intrinsic_demote:
6093       bld.pseudo(aco_opcode::p_demote_to_helper);
6094       ctx->block->kind |= block_kind_uses_demote;
6095       ctx->program->needs_exact = true;
6096       break;
6097    case nir_intrinsic_demote_if: {
6098       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6099       assert(src.regClass() == bld.lm);
6100       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6101       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6102       ctx->block->kind |= block_kind_uses_demote;
6103       ctx->program->needs_exact = true;
6104       break;
6105    }
6106    case nir_intrinsic_first_invocation: {
6107       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6108                get_ssa_temp(ctx, &instr->dest.ssa));
6109       break;
6110    }
6111    case nir_intrinsic_shader_clock:
6112       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6113       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6114       break;
6115    case nir_intrinsic_load_vertex_id_zero_base: {
6116       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6117       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6118       break;
6119    }
6120    case nir_intrinsic_load_first_vertex: {
6121       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6122       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6123       break;
6124    }
6125    case nir_intrinsic_load_base_instance: {
6126       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6127       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6128       break;
6129    }
6130    case nir_intrinsic_load_instance_id: {
6131       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6132       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6133       break;
6134    }
6135    case nir_intrinsic_load_draw_id: {
6136       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6137       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6138       break;
6139    }
6140    default:
6141       fprintf(stderr, "Unimplemented intrinsic instr: ");
6142       nir_print_instr(&instr->instr, stderr);
6143       fprintf(stderr, "\n");
6144       abort();
6145
6146       break;
6147    }
6148 }
6149
6150
6151 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6152                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6153                     enum glsl_base_type *stype)
6154 {
6155    nir_deref_instr *texture_deref_instr = NULL;
6156    nir_deref_instr *sampler_deref_instr = NULL;
6157    int plane = -1;
6158
6159    for (unsigned i = 0; i < instr->num_srcs; i++) {
6160       switch (instr->src[i].src_type) {
6161       case nir_tex_src_texture_deref:
6162          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6163          break;
6164       case nir_tex_src_sampler_deref:
6165          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6166          break;
6167       case nir_tex_src_plane:
6168          plane = nir_src_as_int(instr->src[i].src);
6169          break;
6170       default:
6171          break;
6172       }
6173    }
6174
6175    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6176
6177    if (!sampler_deref_instr)
6178       sampler_deref_instr = texture_deref_instr;
6179
6180    if (plane >= 0) {
6181       assert(instr->op != nir_texop_txf_ms &&
6182              instr->op != nir_texop_samples_identical);
6183       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6184       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6185    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6186       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6187    } else {
6188       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6189    }
6190    if (samp_ptr) {
6191       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6192
6193       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6194          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6195          Builder bld(ctx->program, ctx->block);
6196
6197          /* to avoid unnecessary moves, we split and recombine sampler and image */
6198          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6199                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6200          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6201          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6202                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6203                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6204          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6205                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6206
6207          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6208          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6209                                img[0], img[1], img[2], img[3],
6210                                img[4], img[5], img[6], img[7]);
6211          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6212                                 samp[0], samp[1], samp[2], samp[3]);
6213       }
6214    }
6215    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6216                      instr->op == nir_texop_samples_identical))
6217       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6218 }
6219
6220 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6221                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6222 {
6223    Builder bld(ctx->program, ctx->block);
6224
6225    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6226    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6227    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6228
6229    Operand neg_one(0xbf800000u);
6230    Operand one(0x3f800000u);
6231    Operand two(0x40000000u);
6232    Operand four(0x40800000u);
6233
6234    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6235    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6236    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6237
6238    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6239    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6240    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6241    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6242
6243    // select sc
6244    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6245    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6246                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6247                        one, is_ma_y);
6248    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6249
6250    // select tc
6251    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6252    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6253    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6254
6255    // select ma
6256    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6257                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6258                   deriv_z, is_ma_z);
6259    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6260    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6261 }
6262
6263 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6264 {
6265    Builder bld(ctx->program, ctx->block);
6266    Temp coord_args[4], ma, tc, sc, id;
6267    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6268       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6269
6270    if (is_array) {
6271       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6272
6273       // see comment in ac_prepare_cube_coords()
6274       if (ctx->options->chip_class <= GFX8)
6275          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6276    }
6277
6278    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6279
6280    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6281    vop3a->operands[0] = Operand(ma);
6282    vop3a->abs[0] = true;
6283    Temp invma = bld.tmp(v1);
6284    vop3a->definitions[0] = Definition(invma);
6285    ctx->block->instructions.emplace_back(std::move(vop3a));
6286
6287    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6288    if (!is_deriv)
6289       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6290
6291    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6292    if (!is_deriv)
6293       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6294
6295    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6296
6297    if (is_deriv) {
6298       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6299       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6300
6301       for (unsigned i = 0; i < 2; i++) {
6302          // see comment in ac_prepare_cube_coords()
6303          Temp deriv_ma;
6304          Temp deriv_sc, deriv_tc;
6305          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6306                            &deriv_ma, &deriv_sc, &deriv_tc);
6307
6308          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6309
6310          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6311                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6312                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6313          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6314                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6315                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6316          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6317       }
6318
6319       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6320       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6321    }
6322
6323    if (is_array)
6324       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6325    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6326
6327 }
6328
6329 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6330 {
6331    Temp coord_vec[3];
6332    for (unsigned i = 0; i < coords.size(); i++)
6333       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6334
6335    Builder bld(ctx->program, ctx->block);
6336    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6337
6338    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6339    for (unsigned i = 0; i < coords.size(); i++)
6340       vec->operands[i] = Operand(coord_vec[i]);
6341    Temp res = bld.tmp(RegType::vgpr, coords.size());
6342    vec->definitions[0] = Definition(res);
6343    ctx->block->instructions.emplace_back(std::move(vec));
6344    return res;
6345 }
6346
6347 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6348 {
6349    if (vec->parent_instr->type != nir_instr_type_alu)
6350       return;
6351    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6352    if (vec_instr->op != nir_op_vec(vec->num_components))
6353       return;
6354
6355    for (unsigned i = 0; i < vec->num_components; i++) {
6356       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6357               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6358    }
6359 }
6360
6361 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6362 {
6363    Builder bld(ctx->program, ctx->block);
6364    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6365         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6366    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6367         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6368    nir_const_value *sample_index_cv = NULL;
6369    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6370    enum glsl_base_type stype;
6371    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6372
6373    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6374                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6375    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6376                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6377
6378    for (unsigned i = 0; i < instr->num_srcs; i++) {
6379       switch (instr->src[i].src_type) {
6380       case nir_tex_src_coord:
6381          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6382          break;
6383       case nir_tex_src_bias:
6384          if (instr->op == nir_texop_txb) {
6385             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6386             has_bias = true;
6387          }
6388          break;
6389       case nir_tex_src_lod: {
6390          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6391
6392          if (val && val->f32 <= 0.0) {
6393             level_zero = true;
6394          } else {
6395             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6396             has_lod = true;
6397          }
6398          break;
6399       }
6400       case nir_tex_src_comparator:
6401          if (instr->is_shadow) {
6402             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6403             has_compare = true;
6404          }
6405          break;
6406       case nir_tex_src_offset:
6407          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6408          get_const_vec(instr->src[i].src.ssa, const_offset);
6409          has_offset = true;
6410          break;
6411       case nir_tex_src_ddx:
6412          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6413          has_ddx = true;
6414          break;
6415       case nir_tex_src_ddy:
6416          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6417          has_ddy = true;
6418          break;
6419       case nir_tex_src_ms_index:
6420          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6421          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6422          has_sample_index = true;
6423          break;
6424       case nir_tex_src_texture_offset:
6425       case nir_tex_src_sampler_offset:
6426       default:
6427          break;
6428       }
6429    }
6430 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6431    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6432       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6433
6434    if (instr->op == nir_texop_texture_samples) {
6435       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6436
6437       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6438       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6439       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6440       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6441
6442       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6443                samples, Operand(1u), bld.scc(is_msaa));
6444       return;
6445    }
6446
6447    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6448       aco_ptr<Instruction> tmp_instr;
6449       Temp acc, pack = Temp();
6450
6451       uint32_t pack_const = 0;
6452       for (unsigned i = 0; i < offset.size(); i++) {
6453          if (!const_offset[i])
6454             continue;
6455          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6456       }
6457
6458       if (offset.type() == RegType::sgpr) {
6459          for (unsigned i = 0; i < offset.size(); i++) {
6460             if (const_offset[i])
6461                continue;
6462
6463             acc = emit_extract_vector(ctx, offset, i, s1);
6464             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6465
6466             if (i) {
6467                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6468             }
6469
6470             if (pack == Temp()) {
6471                pack = acc;
6472             } else {
6473                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6474             }
6475          }
6476
6477          if (pack_const && pack != Temp())
6478             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6479       } else {
6480          for (unsigned i = 0; i < offset.size(); i++) {
6481             if (const_offset[i])
6482                continue;
6483
6484             acc = emit_extract_vector(ctx, offset, i, v1);
6485             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6486
6487             if (i) {
6488                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6489             }
6490
6491             if (pack == Temp()) {
6492                pack = acc;
6493             } else {
6494                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6495             }
6496          }
6497
6498          if (pack_const && pack != Temp())
6499             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6500       }
6501       if (pack_const && pack == Temp())
6502          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6503       else if (pack == Temp())
6504          has_offset = false;
6505       else
6506          offset = pack;
6507    }
6508
6509    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6510       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6511
6512    /* pack derivatives */
6513    if (has_ddx || has_ddy) {
6514       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6515          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6516                              ddx, Operand(0u), ddy, Operand(0u));
6517       } else {
6518          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6519       }
6520       has_derivs = true;
6521    }
6522
6523    if (instr->coord_components > 1 &&
6524        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6525        instr->is_array &&
6526        instr->op != nir_texop_txf)
6527       coords = apply_round_slice(ctx, coords, 1);
6528
6529    if (instr->coord_components > 2 &&
6530       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6531        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6532        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6533        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6534        instr->is_array &&
6535        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6536       coords = apply_round_slice(ctx, coords, 2);
6537
6538    if (ctx->options->chip_class == GFX9 &&
6539        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6540        instr->op != nir_texop_lod && instr->coord_components) {
6541       assert(coords.size() > 0 && coords.size() < 3);
6542
6543       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6544       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6545       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6546       if (coords.size() > 1)
6547          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6548       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6549       vec->definitions[0] = Definition(coords);
6550       ctx->block->instructions.emplace_back(std::move(vec));
6551    }
6552
6553    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6554
6555    if (instr->op == nir_texop_samples_identical)
6556       resource = fmask_ptr;
6557
6558    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6559              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6560             instr->op != nir_texop_txs) {
6561       assert(has_sample_index);
6562       Operand op(sample_index);
6563       if (sample_index_cv)
6564          op = Operand(sample_index_cv->u32);
6565       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6566    }
6567
6568    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6569       Temp split_coords[coords.size()];
6570       emit_split_vector(ctx, coords, coords.size());
6571       for (unsigned i = 0; i < coords.size(); i++)
6572          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6573
6574       unsigned i = 0;
6575       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6576          Temp off = emit_extract_vector(ctx, offset, i, v1);
6577          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6578       }
6579
6580       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6581       for (unsigned i = 0; i < coords.size(); i++)
6582          vec->operands[i] = Operand(split_coords[i]);
6583       coords = bld.tmp(coords.regClass());
6584       vec->definitions[0] = Definition(coords);
6585       ctx->block->instructions.emplace_back(std::move(vec));
6586
6587       has_offset = false;
6588    }
6589
6590    /* Build tex instruction */
6591    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6592    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6593                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6594                   : 0;
6595    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6596    Temp tmp_dst = dst;
6597
6598    /* gather4 selects the component by dmask and always returns vec4 */
6599    if (instr->op == nir_texop_tg4) {
6600       assert(instr->dest.ssa.num_components == 4);
6601       if (instr->is_shadow)
6602          dmask = 1;
6603       else
6604          dmask = 1 << instr->component;
6605       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6606          tmp_dst = bld.tmp(v4);
6607    } else if (instr->op == nir_texop_samples_identical) {
6608       tmp_dst = bld.tmp(v1);
6609    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6610       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6611    }
6612
6613    aco_ptr<MIMG_instruction> tex;
6614    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6615       if (!has_lod)
6616          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6617
6618       bool div_by_6 = instr->op == nir_texop_txs &&
6619                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6620                       instr->is_array &&
6621                       (dmask & (1 << 2));
6622       if (tmp_dst.id() == dst.id() && div_by_6)
6623          tmp_dst = bld.tmp(tmp_dst.regClass());
6624
6625       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6626       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6627       tex->operands[1] = Operand(resource);
6628       if (ctx->options->chip_class == GFX9 &&
6629           instr->op == nir_texop_txs &&
6630           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6631           instr->is_array) {
6632          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6633       } else if (instr->op == nir_texop_query_levels) {
6634          tex->dmask = 1 << 3;
6635       } else {
6636          tex->dmask = dmask;
6637       }
6638       tex->da = da;
6639       tex->definitions[0] = Definition(tmp_dst);
6640       tex->dim = dim;
6641       tex->can_reorder = true;
6642       ctx->block->instructions.emplace_back(std::move(tex));
6643
6644       if (div_by_6) {
6645          /* divide 3rd value by 6 by multiplying with magic number */
6646          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6647          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6648          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6649          assert(instr->dest.ssa.num_components == 3);
6650          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6651          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6652                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6653                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6654                               by_6);
6655
6656       }
6657
6658       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6659       return;
6660    }
6661
6662    Temp tg4_compare_cube_wa64 = Temp();
6663
6664    if (tg4_integer_workarounds) {
6665       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6666       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6667       tex->operands[1] = Operand(resource);
6668       tex->dim = dim;
6669       tex->dmask = 0x3;
6670       tex->da = da;
6671       Temp size = bld.tmp(v2);
6672       tex->definitions[0] = Definition(size);
6673       tex->can_reorder = true;
6674       ctx->block->instructions.emplace_back(std::move(tex));
6675       emit_split_vector(ctx, size, size.size());
6676
6677       Temp half_texel[2];
6678       for (unsigned i = 0; i < 2; i++) {
6679          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6680          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6681          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6682          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6683       }
6684
6685       Temp orig_coords[2] = {
6686          emit_extract_vector(ctx, coords, 0, v1),
6687          emit_extract_vector(ctx, coords, 1, v1)};
6688       Temp new_coords[2] = {
6689          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6690          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6691       };
6692
6693       if (tg4_integer_cube_workaround) {
6694          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6695          Temp desc[resource.size()];
6696          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6697                                                                            Format::PSEUDO, 1, resource.size())};
6698          split->operands[0] = Operand(resource);
6699          for (unsigned i = 0; i < resource.size(); i++) {
6700             desc[i] = bld.tmp(s1);
6701             split->definitions[i] = Definition(desc[i]);
6702          }
6703          ctx->block->instructions.emplace_back(std::move(split));
6704
6705          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6706          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6707                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6708
6709          Temp nfmt;
6710          if (stype == GLSL_TYPE_UINT) {
6711             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6712                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6713                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6714                             bld.scc(compare_cube_wa));
6715          } else {
6716             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6717                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6718                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6719                             bld.scc(compare_cube_wa));
6720          }
6721          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6722          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6723
6724          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6725
6726          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6727                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6728          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6729
6730          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6731                                                                          Format::PSEUDO, resource.size(), 1)};
6732          for (unsigned i = 0; i < resource.size(); i++)
6733             vec->operands[i] = Operand(desc[i]);
6734          resource = bld.tmp(resource.regClass());
6735          vec->definitions[0] = Definition(resource);
6736          ctx->block->instructions.emplace_back(std::move(vec));
6737
6738          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6739                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6740          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6741                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6742       }
6743
6744       if (coords.size() == 3) {
6745          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6746                              new_coords[0], new_coords[1],
6747                              emit_extract_vector(ctx, coords, 2, v1));
6748       } else {
6749          assert(coords.size() == 2);
6750          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6751                              new_coords[0], new_coords[1]);
6752       }
6753    }
6754
6755    std::vector<Operand> args;
6756    if (has_offset)
6757       args.emplace_back(Operand(offset));
6758    if (has_bias)
6759       args.emplace_back(Operand(bias));
6760    if (has_compare)
6761       args.emplace_back(Operand(compare));
6762    if (has_derivs)
6763       args.emplace_back(Operand(derivs));
6764    args.emplace_back(Operand(coords));
6765    if (has_sample_index)
6766       args.emplace_back(Operand(sample_index));
6767    if (has_lod)
6768       args.emplace_back(lod);
6769
6770    Temp arg;
6771    if (args.size() > 1) {
6772       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6773       unsigned size = 0;
6774       for (unsigned i = 0; i < args.size(); i++) {
6775          size += args[i].size();
6776          vec->operands[i] = args[i];
6777       }
6778       RegClass rc = RegClass(RegType::vgpr, size);
6779       Temp tmp = bld.tmp(rc);
6780       vec->definitions[0] = Definition(tmp);
6781       ctx->block->instructions.emplace_back(std::move(vec));
6782       arg = tmp;
6783    } else {
6784       assert(args[0].isTemp());
6785       arg = as_vgpr(ctx, args[0].getTemp());
6786    }
6787
6788    /* we don't need the bias, sample index, compare value or offset to be
6789     * computed in WQM but if the p_create_vector copies the coordinates, then it
6790     * needs to be in WQM */
6791    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6792        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6793        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6794       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6795
6796    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6797       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6798
6799       assert(coords.size() == 1);
6800       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6801       aco_opcode op;
6802       switch (last_bit) {
6803       case 1:
6804          op = aco_opcode::buffer_load_format_x; break;
6805       case 2:
6806          op = aco_opcode::buffer_load_format_xy; break;
6807       case 3:
6808          op = aco_opcode::buffer_load_format_xyz; break;
6809       case 4:
6810          op = aco_opcode::buffer_load_format_xyzw; break;
6811       default:
6812          unreachable("Tex instruction loads more than 4 components.");
6813       }
6814
6815       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6816       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6817          tmp_dst = dst;
6818       else
6819          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6820
6821       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6822       mubuf->operands[0] = Operand(coords);
6823       mubuf->operands[1] = Operand(resource);
6824       mubuf->operands[2] = Operand((uint32_t) 0);
6825       mubuf->definitions[0] = Definition(tmp_dst);
6826       mubuf->idxen = true;
6827       mubuf->can_reorder = true;
6828       ctx->block->instructions.emplace_back(std::move(mubuf));
6829
6830       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6831       return;
6832    }
6833
6834
6835    if (instr->op == nir_texop_txf ||
6836        instr->op == nir_texop_txf_ms ||
6837        instr->op == nir_texop_samples_identical) {
6838       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6839       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6840       tex->operands[0] = Operand(arg);
6841       tex->operands[1] = Operand(resource);
6842       tex->dim = dim;
6843       tex->dmask = dmask;
6844       tex->unrm = true;
6845       tex->da = da;
6846       tex->definitions[0] = Definition(tmp_dst);
6847       tex->can_reorder = true;
6848       ctx->block->instructions.emplace_back(std::move(tex));
6849
6850       if (instr->op == nir_texop_samples_identical) {
6851          assert(dmask == 1 && dst.regClass() == v1);
6852          assert(dst.id() != tmp_dst.id());
6853
6854          Temp tmp = bld.tmp(bld.lm);
6855          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6856          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6857
6858       } else {
6859          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6860       }
6861       return;
6862    }
6863
6864    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6865    aco_opcode opcode = aco_opcode::image_sample;
6866    if (has_offset) { /* image_sample_*_o */
6867       if (has_compare) {
6868          opcode = aco_opcode::image_sample_c_o;
6869          if (has_derivs)
6870             opcode = aco_opcode::image_sample_c_d_o;
6871          if (has_bias)
6872             opcode = aco_opcode::image_sample_c_b_o;
6873          if (level_zero)
6874             opcode = aco_opcode::image_sample_c_lz_o;
6875          if (has_lod)
6876             opcode = aco_opcode::image_sample_c_l_o;
6877       } else {
6878          opcode = aco_opcode::image_sample_o;
6879          if (has_derivs)
6880             opcode = aco_opcode::image_sample_d_o;
6881          if (has_bias)
6882             opcode = aco_opcode::image_sample_b_o;
6883          if (level_zero)
6884             opcode = aco_opcode::image_sample_lz_o;
6885          if (has_lod)
6886             opcode = aco_opcode::image_sample_l_o;
6887       }
6888    } else { /* no offset */
6889       if (has_compare) {
6890          opcode = aco_opcode::image_sample_c;
6891          if (has_derivs)
6892             opcode = aco_opcode::image_sample_c_d;
6893          if (has_bias)
6894             opcode = aco_opcode::image_sample_c_b;
6895          if (level_zero)
6896             opcode = aco_opcode::image_sample_c_lz;
6897          if (has_lod)
6898             opcode = aco_opcode::image_sample_c_l;
6899       } else {
6900          opcode = aco_opcode::image_sample;
6901          if (has_derivs)
6902             opcode = aco_opcode::image_sample_d;
6903          if (has_bias)
6904             opcode = aco_opcode::image_sample_b;
6905          if (level_zero)
6906             opcode = aco_opcode::image_sample_lz;
6907          if (has_lod)
6908             opcode = aco_opcode::image_sample_l;
6909       }
6910    }
6911
6912    if (instr->op == nir_texop_tg4) {
6913       if (has_offset) {
6914          opcode = aco_opcode::image_gather4_lz_o;
6915          if (has_compare)
6916             opcode = aco_opcode::image_gather4_c_lz_o;
6917       } else {
6918          opcode = aco_opcode::image_gather4_lz;
6919          if (has_compare)
6920             opcode = aco_opcode::image_gather4_c_lz;
6921       }
6922    } else if (instr->op == nir_texop_lod) {
6923       opcode = aco_opcode::image_get_lod;
6924    }
6925
6926    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6927    tex->operands[0] = Operand(arg);
6928    tex->operands[1] = Operand(resource);
6929    tex->operands[2] = Operand(sampler);
6930    tex->dim = dim;
6931    tex->dmask = dmask;
6932    tex->da = da;
6933    tex->definitions[0] = Definition(tmp_dst);
6934    tex->can_reorder = true;
6935    ctx->block->instructions.emplace_back(std::move(tex));
6936
6937    if (tg4_integer_cube_workaround) {
6938       assert(tmp_dst.id() != dst.id());
6939       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6940
6941       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6942       Temp val[4];
6943       for (unsigned i = 0; i < dst.size(); i++) {
6944          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6945          Temp cvt_val;
6946          if (stype == GLSL_TYPE_UINT)
6947             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6948          else
6949             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6950          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6951       }
6952       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6953       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6954                            val[0], val[1], val[2], val[3]);
6955    }
6956    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6957    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6958
6959 }
6960
6961
6962 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6963 {
6964    Temp tmp = get_ssa_temp(ctx, ssa);
6965    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6966       return Operand(tmp.regClass());
6967    else
6968       return Operand(tmp);
6969 }
6970
6971 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6972 {
6973    aco_ptr<Pseudo_instruction> phi;
6974    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6975    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
6976
6977    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
6978    logical |= ctx->block->kind & block_kind_merge;
6979    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6980
6981    /* we want a sorted list of sources, since the predecessor list is also sorted */
6982    std::map<unsigned, nir_ssa_def*> phi_src;
6983    nir_foreach_phi_src(src, instr)
6984       phi_src[src->pred->index] = src->src.ssa;
6985
6986    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
6987    unsigned num_operands = 0;
6988    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
6989    unsigned num_defined = 0;
6990    unsigned cur_pred_idx = 0;
6991    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
6992       if (cur_pred_idx < preds.size()) {
6993          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
6994          unsigned block = ctx->cf_info.nir_to_aco[src.first];
6995          unsigned skipped = 0;
6996          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
6997             skipped++;
6998          if (cur_pred_idx + skipped < preds.size()) {
6999             for (unsigned i = 0; i < skipped; i++)
7000                operands[num_operands++] = Operand(dst.regClass());
7001             cur_pred_idx += skipped;
7002          } else {
7003             continue;
7004          }
7005       }
7006       cur_pred_idx++;
7007       Operand op = get_phi_operand(ctx, src.second);
7008       operands[num_operands++] = op;
7009       num_defined += !op.isUndefined();
7010    }
7011    /* handle block_kind_continue_or_break at loop exit blocks */
7012    while (cur_pred_idx++ < preds.size())
7013       operands[num_operands++] = Operand(dst.regClass());
7014
7015    if (num_defined == 0) {
7016       Builder bld(ctx->program, ctx->block);
7017       if (dst.regClass() == s1) {
7018          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7019       } else if (dst.regClass() == v1) {
7020          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7021       } else {
7022          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7023          for (unsigned i = 0; i < dst.size(); i++)
7024             vec->operands[i] = Operand(0u);
7025          vec->definitions[0] = Definition(dst);
7026          ctx->block->instructions.emplace_back(std::move(vec));
7027       }
7028       return;
7029    }
7030
7031    /* we can use a linear phi in some cases if one src is undef */
7032    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7033       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7034
7035       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7036       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7037       assert(invert->kind & block_kind_invert);
7038
7039       unsigned then_block = invert->linear_preds[0];
7040
7041       Block* insert_block = NULL;
7042       for (unsigned i = 0; i < num_operands; i++) {
7043          Operand op = operands[i];
7044          if (op.isUndefined())
7045             continue;
7046          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7047          phi->operands[0] = op;
7048          break;
7049       }
7050       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7051       phi->operands[1] = Operand(dst.regClass());
7052       phi->definitions[0] = Definition(dst);
7053       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7054       return;
7055    }
7056
7057    /* try to scalarize vector phis */
7058    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7059       // TODO: scalarize linear phis on divergent ifs
7060       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7061       std::array<Temp, 4> new_vec;
7062       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7063          Operand src = operands[i];
7064          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7065             can_scalarize = false;
7066       }
7067       if (can_scalarize) {
7068          unsigned num_components = instr->dest.ssa.num_components;
7069          assert(dst.size() % num_components == 0);
7070          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7071
7072          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7073          for (unsigned k = 0; k < num_components; k++) {
7074             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7075             for (unsigned i = 0; i < num_operands; i++) {
7076                Operand src = operands[i];
7077                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7078             }
7079             Temp phi_dst = {ctx->program->allocateId(), rc};
7080             phi->definitions[0] = Definition(phi_dst);
7081             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7082             new_vec[k] = phi_dst;
7083             vec->operands[k] = Operand(phi_dst);
7084          }
7085          vec->definitions[0] = Definition(dst);
7086          ctx->block->instructions.emplace_back(std::move(vec));
7087          ctx->allocated_vec.emplace(dst.id(), new_vec);
7088          return;
7089       }
7090    }
7091
7092    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7093    for (unsigned i = 0; i < num_operands; i++)
7094       phi->operands[i] = operands[i];
7095    phi->definitions[0] = Definition(dst);
7096    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7097 }
7098
7099
7100 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7101 {
7102    Temp dst = get_ssa_temp(ctx, &instr->def);
7103
7104    assert(dst.type() == RegType::sgpr);
7105
7106    if (dst.size() == 1) {
7107       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7108    } else {
7109       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7110       for (unsigned i = 0; i < dst.size(); i++)
7111          vec->operands[i] = Operand(0u);
7112       vec->definitions[0] = Definition(dst);
7113       ctx->block->instructions.emplace_back(std::move(vec));
7114    }
7115 }
7116
7117 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7118 {
7119    Builder bld(ctx->program, ctx->block);
7120    Block *logical_target;
7121    append_logical_end(ctx->block);
7122    unsigned idx = ctx->block->index;
7123
7124    switch (instr->type) {
7125    case nir_jump_break:
7126       logical_target = ctx->cf_info.parent_loop.exit;
7127       add_logical_edge(idx, logical_target);
7128       ctx->block->kind |= block_kind_break;
7129
7130       if (!ctx->cf_info.parent_if.is_divergent &&
7131           !ctx->cf_info.parent_loop.has_divergent_continue) {
7132          /* uniform break - directly jump out of the loop */
7133          ctx->block->kind |= block_kind_uniform;
7134          ctx->cf_info.has_branch = true;
7135          bld.branch(aco_opcode::p_branch);
7136          add_linear_edge(idx, logical_target);
7137          return;
7138       }
7139       ctx->cf_info.parent_loop.has_divergent_branch = true;
7140       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7141       break;
7142    case nir_jump_continue:
7143       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7144       add_logical_edge(idx, logical_target);
7145       ctx->block->kind |= block_kind_continue;
7146
7147       if (ctx->cf_info.parent_if.is_divergent) {
7148          /* for potential uniform breaks after this continue,
7149             we must ensure that they are handled correctly */
7150          ctx->cf_info.parent_loop.has_divergent_continue = true;
7151          ctx->cf_info.parent_loop.has_divergent_branch = true;
7152          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7153       } else {
7154          /* uniform continue - directly jump to the loop header */
7155          ctx->block->kind |= block_kind_uniform;
7156          ctx->cf_info.has_branch = true;
7157          bld.branch(aco_opcode::p_branch);
7158          add_linear_edge(idx, logical_target);
7159          return;
7160       }
7161       break;
7162    default:
7163       fprintf(stderr, "Unknown NIR jump instr: ");
7164       nir_print_instr(&instr->instr, stderr);
7165       fprintf(stderr, "\n");
7166       abort();
7167    }
7168
7169    /* remove critical edges from linear CFG */
7170    bld.branch(aco_opcode::p_branch);
7171    Block* break_block = ctx->program->create_and_insert_block();
7172    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7173    break_block->kind |= block_kind_uniform;
7174    add_linear_edge(idx, break_block);
7175    /* the loop_header pointer might be invalidated by this point */
7176    if (instr->type == nir_jump_continue)
7177       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7178    add_linear_edge(break_block->index, logical_target);
7179    bld.reset(break_block);
7180    bld.branch(aco_opcode::p_branch);
7181
7182    Block* continue_block = ctx->program->create_and_insert_block();
7183    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7184    add_linear_edge(idx, continue_block);
7185    append_logical_start(continue_block);
7186    ctx->block = continue_block;
7187    return;
7188 }
7189
7190 void visit_block(isel_context *ctx, nir_block *block)
7191 {
7192    nir_foreach_instr(instr, block) {
7193       switch (instr->type) {
7194       case nir_instr_type_alu:
7195          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7196          break;
7197       case nir_instr_type_load_const:
7198          visit_load_const(ctx, nir_instr_as_load_const(instr));
7199          break;
7200       case nir_instr_type_intrinsic:
7201          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7202          break;
7203       case nir_instr_type_tex:
7204          visit_tex(ctx, nir_instr_as_tex(instr));
7205          break;
7206       case nir_instr_type_phi:
7207          visit_phi(ctx, nir_instr_as_phi(instr));
7208          break;
7209       case nir_instr_type_ssa_undef:
7210          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7211          break;
7212       case nir_instr_type_deref:
7213          break;
7214       case nir_instr_type_jump:
7215          visit_jump(ctx, nir_instr_as_jump(instr));
7216          break;
7217       default:
7218          fprintf(stderr, "Unknown NIR instr type: ");
7219          nir_print_instr(instr, stderr);
7220          fprintf(stderr, "\n");
7221          //abort();
7222       }
7223    }
7224
7225    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7226       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7227 }
7228
7229
7230
7231 static void visit_loop(isel_context *ctx, nir_loop *loop)
7232 {
7233    append_logical_end(ctx->block);
7234    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7235    Builder bld(ctx->program, ctx->block);
7236    bld.branch(aco_opcode::p_branch);
7237    unsigned loop_preheader_idx = ctx->block->index;
7238
7239    Block loop_exit = Block();
7240    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7241    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7242
7243    Block* loop_header = ctx->program->create_and_insert_block();
7244    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7245    loop_header->kind |= block_kind_loop_header;
7246    add_edge(loop_preheader_idx, loop_header);
7247    ctx->block = loop_header;
7248
7249    /* emit loop body */
7250    unsigned loop_header_idx = loop_header->index;
7251    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7252    append_logical_start(ctx->block);
7253    visit_cf_list(ctx, &loop->body);
7254
7255    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7256    if (!ctx->cf_info.has_branch) {
7257       append_logical_end(ctx->block);
7258       if (ctx->cf_info.exec_potentially_empty) {
7259          /* Discards can result in code running with an empty exec mask.
7260           * This would result in divergent breaks not ever being taken. As a
7261           * workaround, break the loop when the loop mask is empty instead of
7262           * always continuing. */
7263          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7264          unsigned block_idx = ctx->block->index;
7265
7266          /* create helper blocks to avoid critical edges */
7267          Block *break_block = ctx->program->create_and_insert_block();
7268          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7269          break_block->kind = block_kind_uniform;
7270          bld.reset(break_block);
7271          bld.branch(aco_opcode::p_branch);
7272          add_linear_edge(block_idx, break_block);
7273          add_linear_edge(break_block->index, &loop_exit);
7274
7275          Block *continue_block = ctx->program->create_and_insert_block();
7276          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7277          continue_block->kind = block_kind_uniform;
7278          bld.reset(continue_block);
7279          bld.branch(aco_opcode::p_branch);
7280          add_linear_edge(block_idx, continue_block);
7281          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7282
7283          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7284          ctx->block = &ctx->program->blocks[block_idx];
7285       } else {
7286          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7287          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7288             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7289          else
7290             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7291       }
7292
7293       bld.reset(ctx->block);
7294       bld.branch(aco_opcode::p_branch);
7295    }
7296
7297    /* fixup phis in loop header from unreachable blocks */
7298    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7299       bool linear = ctx->cf_info.has_branch;
7300       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7301       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7302          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7303              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7304             /* the last operand should be the one that needs to be removed */
7305             instr->operands.pop_back();
7306          } else if (!is_phi(instr)) {
7307             break;
7308          }
7309       }
7310    }
7311
7312    ctx->cf_info.has_branch = false;
7313
7314    // TODO: if the loop has not a single exit, we must add one °°
7315    /* emit loop successor block */
7316    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7317    append_logical_start(ctx->block);
7318
7319    #if 0
7320    // TODO: check if it is beneficial to not branch on continues
7321    /* trim linear phis in loop header */
7322    for (auto&& instr : loop_entry->instructions) {
7323       if (instr->opcode == aco_opcode::p_linear_phi) {
7324          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7325          new_phi->definitions[0] = instr->definitions[0];
7326          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7327             new_phi->operands[i] = instr->operands[i];
7328          /* check that the remaining operands are all the same */
7329          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7330             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7331          instr.swap(new_phi);
7332       } else if (instr->opcode == aco_opcode::p_phi) {
7333          continue;
7334       } else {
7335          break;
7336       }
7337    }
7338    #endif
7339 }
7340
7341 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7342 {
7343    ic->cond = cond;
7344
7345    append_logical_end(ctx->block);
7346    ctx->block->kind |= block_kind_branch;
7347
7348    /* branch to linear then block */
7349    assert(cond.regClass() == ctx->program->lane_mask);
7350    aco_ptr<Pseudo_branch_instruction> branch;
7351    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7352    branch->operands[0] = Operand(cond);
7353    ctx->block->instructions.push_back(std::move(branch));
7354
7355    ic->BB_if_idx = ctx->block->index;
7356    ic->BB_invert = Block();
7357    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7358    /* Invert blocks are intentionally not marked as top level because they
7359     * are not part of the logical cfg. */
7360    ic->BB_invert.kind |= block_kind_invert;
7361    ic->BB_endif = Block();
7362    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7363    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7364
7365    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7366    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7367    ctx->cf_info.parent_if.is_divergent = true;
7368    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7369
7370    /** emit logical then block */
7371    Block* BB_then_logical = ctx->program->create_and_insert_block();
7372    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7373    add_edge(ic->BB_if_idx, BB_then_logical);
7374    ctx->block = BB_then_logical;
7375    append_logical_start(BB_then_logical);
7376 }
7377
7378 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7379 {
7380    Block *BB_then_logical = ctx->block;
7381    append_logical_end(BB_then_logical);
7382     /* branch from logical then block to invert block */
7383    aco_ptr<Pseudo_branch_instruction> branch;
7384    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7385    BB_then_logical->instructions.emplace_back(std::move(branch));
7386    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7387    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7388       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7389    BB_then_logical->kind |= block_kind_uniform;
7390    assert(!ctx->cf_info.has_branch);
7391    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7392    ctx->cf_info.parent_loop.has_divergent_branch = false;
7393
7394    /** emit linear then block */
7395    Block* BB_then_linear = ctx->program->create_and_insert_block();
7396    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7397    BB_then_linear->kind |= block_kind_uniform;
7398    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7399    /* branch from linear then block to invert block */
7400    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7401    BB_then_linear->instructions.emplace_back(std::move(branch));
7402    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7403
7404    /** emit invert merge block */
7405    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7406    ic->invert_idx = ctx->block->index;
7407
7408    /* branch to linear else block (skip else) */
7409    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7410    branch->operands[0] = Operand(ic->cond);
7411    ctx->block->instructions.push_back(std::move(branch));
7412
7413    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7414    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7415
7416    /** emit logical else block */
7417    Block* BB_else_logical = ctx->program->create_and_insert_block();
7418    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7419    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7420    add_linear_edge(ic->invert_idx, BB_else_logical);
7421    ctx->block = BB_else_logical;
7422    append_logical_start(BB_else_logical);
7423 }
7424
7425 static void end_divergent_if(isel_context *ctx, if_context *ic)
7426 {
7427    Block *BB_else_logical = ctx->block;
7428    append_logical_end(BB_else_logical);
7429
7430    /* branch from logical else block to endif block */
7431    aco_ptr<Pseudo_branch_instruction> branch;
7432    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7433    BB_else_logical->instructions.emplace_back(std::move(branch));
7434    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7435    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7436       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7437    BB_else_logical->kind |= block_kind_uniform;
7438
7439    assert(!ctx->cf_info.has_branch);
7440    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7441
7442
7443    /** emit linear else block */
7444    Block* BB_else_linear = ctx->program->create_and_insert_block();
7445    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7446    BB_else_linear->kind |= block_kind_uniform;
7447    add_linear_edge(ic->invert_idx, BB_else_linear);
7448
7449    /* branch from linear else block to endif block */
7450    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7451    BB_else_linear->instructions.emplace_back(std::move(branch));
7452    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7453
7454
7455    /** emit endif merge block */
7456    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7457    append_logical_start(ctx->block);
7458
7459
7460    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7461    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7462    /* uniform control flow never has an empty exec-mask */
7463    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7464       ctx->cf_info.exec_potentially_empty = false;
7465 }
7466
7467 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7468 {
7469    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7470    Builder bld(ctx->program, ctx->block);
7471    aco_ptr<Pseudo_branch_instruction> branch;
7472
7473    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7474       /**
7475        * Uniform conditionals are represented in the following way*) :
7476        *
7477        * The linear and logical CFG:
7478        *                        BB_IF
7479        *                        /    \
7480        *       BB_THEN (logical)      BB_ELSE (logical)
7481        *                        \    /
7482        *                        BB_ENDIF
7483        *
7484        * *) Exceptions may be due to break and continue statements within loops
7485        *    If a break/continue happens within uniform control flow, it branches
7486        *    to the loop exit/entry block. Otherwise, it branches to the next
7487        *    merge block.
7488        **/
7489       append_logical_end(ctx->block);
7490       ctx->block->kind |= block_kind_uniform;
7491
7492       /* emit branch */
7493       assert(cond.regClass() == bld.lm);
7494       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7495       cond = bool_to_scalar_condition(ctx, cond);
7496
7497       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7498       branch->operands[0] = Operand(cond);
7499       branch->operands[0].setFixed(scc);
7500       ctx->block->instructions.emplace_back(std::move(branch));
7501
7502       unsigned BB_if_idx = ctx->block->index;
7503       Block BB_endif = Block();
7504       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7505       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7506
7507       /** emit then block */
7508       Block* BB_then = ctx->program->create_and_insert_block();
7509       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7510       add_edge(BB_if_idx, BB_then);
7511       append_logical_start(BB_then);
7512       ctx->block = BB_then;
7513       visit_cf_list(ctx, &if_stmt->then_list);
7514       BB_then = ctx->block;
7515       bool then_branch = ctx->cf_info.has_branch;
7516       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7517
7518       if (!then_branch) {
7519          append_logical_end(BB_then);
7520          /* branch from then block to endif block */
7521          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7522          BB_then->instructions.emplace_back(std::move(branch));
7523          add_linear_edge(BB_then->index, &BB_endif);
7524          if (!then_branch_divergent)
7525             add_logical_edge(BB_then->index, &BB_endif);
7526          BB_then->kind |= block_kind_uniform;
7527       }
7528
7529       ctx->cf_info.has_branch = false;
7530       ctx->cf_info.parent_loop.has_divergent_branch = false;
7531
7532       /** emit else block */
7533       Block* BB_else = ctx->program->create_and_insert_block();
7534       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7535       add_edge(BB_if_idx, BB_else);
7536       append_logical_start(BB_else);
7537       ctx->block = BB_else;
7538       visit_cf_list(ctx, &if_stmt->else_list);
7539       BB_else = ctx->block;
7540
7541       if (!ctx->cf_info.has_branch) {
7542          append_logical_end(BB_else);
7543          /* branch from then block to endif block */
7544          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7545          BB_else->instructions.emplace_back(std::move(branch));
7546          add_linear_edge(BB_else->index, &BB_endif);
7547          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7548             add_logical_edge(BB_else->index, &BB_endif);
7549          BB_else->kind |= block_kind_uniform;
7550       }
7551
7552       ctx->cf_info.has_branch &= then_branch;
7553       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7554
7555       /** emit endif merge block */
7556       if (!ctx->cf_info.has_branch) {
7557          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7558          append_logical_start(ctx->block);
7559       }
7560    } else { /* non-uniform condition */
7561       /**
7562        * To maintain a logical and linear CFG without critical edges,
7563        * non-uniform conditionals are represented in the following way*) :
7564        *
7565        * The linear CFG:
7566        *                        BB_IF
7567        *                        /    \
7568        *       BB_THEN (logical)      BB_THEN (linear)
7569        *                        \    /
7570        *                        BB_INVERT (linear)
7571        *                        /    \
7572        *       BB_ELSE (logical)      BB_ELSE (linear)
7573        *                        \    /
7574        *                        BB_ENDIF
7575        *
7576        * The logical CFG:
7577        *                        BB_IF
7578        *                        /    \
7579        *       BB_THEN (logical)      BB_ELSE (logical)
7580        *                        \    /
7581        *                        BB_ENDIF
7582        *
7583        * *) Exceptions may be due to break and continue statements within loops
7584        **/
7585
7586       if_context ic;
7587
7588       begin_divergent_if_then(ctx, &ic, cond);
7589       visit_cf_list(ctx, &if_stmt->then_list);
7590
7591       begin_divergent_if_else(ctx, &ic);
7592       visit_cf_list(ctx, &if_stmt->else_list);
7593
7594       end_divergent_if(ctx, &ic);
7595    }
7596 }
7597
7598 static void visit_cf_list(isel_context *ctx,
7599                           struct exec_list *list)
7600 {
7601    foreach_list_typed(nir_cf_node, node, node, list) {
7602       switch (node->type) {
7603       case nir_cf_node_block:
7604          visit_block(ctx, nir_cf_node_as_block(node));
7605          break;
7606       case nir_cf_node_if:
7607          visit_if(ctx, nir_cf_node_as_if(node));
7608          break;
7609       case nir_cf_node_loop:
7610          visit_loop(ctx, nir_cf_node_as_loop(node));
7611          break;
7612       default:
7613          unreachable("unimplemented cf list type");
7614       }
7615    }
7616 }
7617
7618 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7619 {
7620    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7621    uint64_t mask = ctx->vs_output.mask[slot];
7622    if (!is_pos && !mask)
7623       return;
7624    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7625       return;
7626    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7627    exp->enabled_mask = mask;
7628    for (unsigned i = 0; i < 4; ++i) {
7629       if (mask & (1 << i))
7630          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7631       else
7632          exp->operands[i] = Operand(v1);
7633    }
7634    exp->valid_mask = false;
7635    exp->done = false;
7636    exp->compressed = false;
7637    if (is_pos)
7638       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7639    else
7640       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7641    ctx->block->instructions.emplace_back(std::move(exp));
7642 }
7643
7644 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7645 {
7646    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7647    exp->enabled_mask = 0;
7648    for (unsigned i = 0; i < 4; ++i)
7649       exp->operands[i] = Operand(v1);
7650    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7651       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7652       exp->enabled_mask |= 0x1;
7653    }
7654    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7655       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7656       exp->enabled_mask |= 0x4;
7657    }
7658    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7659       if (ctx->options->chip_class < GFX9) {
7660          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7661          exp->enabled_mask |= 0x8;
7662       } else {
7663          Builder bld(ctx->program, ctx->block);
7664
7665          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7666                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7667          if (exp->operands[2].isTemp())
7668             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7669
7670          exp->operands[2] = Operand(out);
7671          exp->enabled_mask |= 0x4;
7672       }
7673    }
7674    exp->valid_mask = false;
7675    exp->done = false;
7676    exp->compressed = false;
7677    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7678    ctx->block->instructions.emplace_back(std::move(exp));
7679 }
7680
7681 static void create_vs_exports(isel_context *ctx)
7682 {
7683    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7684
7685    if (outinfo->export_prim_id) {
7686       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7687       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7688    }
7689
7690    if (ctx->options->key.has_multiview_view_index) {
7691       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7692       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7693    }
7694
7695    /* the order these position exports are created is important */
7696    int next_pos = 0;
7697    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7698    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7699       export_vs_psiz_layer_viewport(ctx, &next_pos);
7700    }
7701    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7702       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7703    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7704       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7705
7706    if (ctx->options->key.vs_common_out.export_clip_dists) {
7707       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7708          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7709       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7710          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7711    }
7712
7713    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7714       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7715           i != VARYING_SLOT_PRIMITIVE_ID)
7716          continue;
7717
7718       export_vs_varying(ctx, i, false, NULL);
7719    }
7720 }
7721
7722 static void emit_stream_output(isel_context *ctx,
7723                                Temp const *so_buffers,
7724                                Temp const *so_write_offset,
7725                                const struct radv_stream_output *output)
7726 {
7727    unsigned num_comps = util_bitcount(output->component_mask);
7728    unsigned loc = output->location;
7729    unsigned buf = output->buffer;
7730    unsigned offset = output->offset;
7731
7732    assert(num_comps && num_comps <= 4);
7733    if (!num_comps || num_comps > 4)
7734       return;
7735
7736    unsigned start = ffs(output->component_mask) - 1;
7737
7738    Temp out[4];
7739    bool all_undef = true;
7740    assert(ctx->stage == vertex_vs);
7741    for (unsigned i = 0; i < num_comps; i++) {
7742       out[i] = ctx->vs_output.outputs[loc][start + i];
7743       all_undef = all_undef && !out[i].id();
7744    }
7745    if (all_undef)
7746       return;
7747
7748    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7749    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7750    for (unsigned i = 0; i < num_comps; ++i)
7751       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7752    vec->definitions[0] = Definition(write_data);
7753    ctx->block->instructions.emplace_back(std::move(vec));
7754
7755    aco_opcode opcode;
7756    switch (num_comps) {
7757    case 1:
7758       opcode = aco_opcode::buffer_store_dword;
7759       break;
7760    case 2:
7761       opcode = aco_opcode::buffer_store_dwordx2;
7762       break;
7763    case 3:
7764       opcode = aco_opcode::buffer_store_dwordx3;
7765       break;
7766    case 4:
7767       opcode = aco_opcode::buffer_store_dwordx4;
7768       break;
7769    }
7770
7771    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7772    store->operands[0] = Operand(so_write_offset[buf]);
7773    store->operands[1] = Operand(so_buffers[buf]);
7774    store->operands[2] = Operand((uint32_t) 0);
7775    store->operands[3] = Operand(write_data);
7776    if (offset > 4095) {
7777       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7778       Builder bld(ctx->program, ctx->block);
7779       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7780    } else {
7781       store->offset = offset;
7782    }
7783    store->offen = true;
7784    store->glc = true;
7785    store->dlc = false;
7786    store->slc = true;
7787    store->can_reorder = true;
7788    ctx->block->instructions.emplace_back(std::move(store));
7789 }
7790
7791 static void emit_streamout(isel_context *ctx, unsigned stream)
7792 {
7793    Builder bld(ctx->program, ctx->block);
7794
7795    Temp so_buffers[4];
7796    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7797    for (unsigned i = 0; i < 4; i++) {
7798       unsigned stride = ctx->program->info->so.strides[i];
7799       if (!stride)
7800          continue;
7801
7802       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7803    }
7804
7805    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7806                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7807
7808    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7809
7810    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7811
7812    if_context ic;
7813    begin_divergent_if_then(ctx, &ic, can_emit);
7814
7815    bld.reset(ctx->block);
7816
7817    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7818
7819    Temp so_write_offset[4];
7820
7821    for (unsigned i = 0; i < 4; i++) {
7822       unsigned stride = ctx->program->info->so.strides[i];
7823       if (!stride)
7824          continue;
7825
7826       if (stride == 1) {
7827          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7828                                 get_arg(ctx, ctx->args->streamout_write_idx),
7829                                 get_arg(ctx, ctx->args->streamout_offset[i]));
7830          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7831
7832          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7833       } else {
7834          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7835          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
7836                                  get_arg(ctx, ctx->args->streamout_offset[i]));
7837          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7838       }
7839    }
7840
7841    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7842       struct radv_stream_output *output =
7843          &ctx->program->info->so.outputs[i];
7844       if (stream != output->stream)
7845          continue;
7846
7847       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7848    }
7849
7850    begin_divergent_if_else(ctx, &ic);
7851    end_divergent_if(ctx, &ic);
7852 }
7853
7854 } /* end namespace */
7855
7856 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
7857 {
7858    /* Split all arguments except for the first (ring_offsets) and the last
7859     * (exec) so that the dead channels don't stay live throughout the program.
7860     */
7861    for (unsigned i = 1; i < startpgm->definitions.size() - 1; i++) {
7862       if (startpgm->definitions[i].regClass().size() > 1) {
7863          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
7864                            startpgm->definitions[i].regClass().size());
7865       }
7866    }
7867 }
7868
7869 void handle_bc_optimize(isel_context *ctx)
7870 {
7871    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7872    Builder bld(ctx->program, ctx->block);
7873    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7874    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7875    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7876    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
7877    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
7878    if (uses_center && uses_centroid) {
7879       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
7880                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
7881
7882       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7883          Temp new_coord[2];
7884          for (unsigned i = 0; i < 2; i++) {
7885             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
7886             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
7887             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7888                                     persp_centroid, persp_center, sel);
7889          }
7890          ctx->persp_centroid = bld.tmp(v2);
7891          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
7892                     Operand(new_coord[0]), Operand(new_coord[1]));
7893          emit_split_vector(ctx, ctx->persp_centroid, 2);
7894       }
7895
7896       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7897          Temp new_coord[2];
7898          for (unsigned i = 0; i < 2; i++) {
7899             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
7900             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
7901             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7902                                     linear_centroid, linear_center, sel);
7903          }
7904          ctx->linear_centroid = bld.tmp(v2);
7905          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
7906                     Operand(new_coord[0]), Operand(new_coord[1]));
7907          emit_split_vector(ctx, ctx->linear_centroid, 2);
7908       }
7909    }
7910 }
7911
7912 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
7913 {
7914    Program *program = ctx->program;
7915
7916    unsigned float_controls = shader->info.float_controls_execution_mode;
7917
7918    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
7919       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
7920    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
7921       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
7922                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
7923
7924    program->next_fp_mode.must_flush_denorms32 =
7925       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
7926    program->next_fp_mode.must_flush_denorms16_64 =
7927       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
7928                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
7929
7930    program->next_fp_mode.care_about_round32 =
7931       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
7932
7933    program->next_fp_mode.care_about_round16_64 =
7934       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
7935                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
7936
7937    /* default to preserving fp16 and fp64 denorms, since it's free */
7938    if (program->next_fp_mode.must_flush_denorms16_64)
7939       program->next_fp_mode.denorm16_64 = 0;
7940    else
7941       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
7942
7943    /* preserving fp32 denorms is expensive, so only do it if asked */
7944    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
7945       program->next_fp_mode.denorm32 = fp_denorm_keep;
7946    else
7947       program->next_fp_mode.denorm32 = 0;
7948
7949    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
7950       program->next_fp_mode.round32 = fp_round_tz;
7951    else
7952       program->next_fp_mode.round32 = fp_round_ne;
7953
7954    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
7955       program->next_fp_mode.round16_64 = fp_round_tz;
7956    else
7957       program->next_fp_mode.round16_64 = fp_round_ne;
7958
7959    ctx->block->fp_mode = program->next_fp_mode;
7960 }
7961
7962 void select_program(Program *program,
7963                     unsigned shader_count,
7964                     struct nir_shader *const *shaders,
7965                     ac_shader_config* config,
7966                     struct radv_shader_args *args)
7967 {
7968    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
7969
7970    for (unsigned i = 0; i < shader_count; i++) {
7971       nir_shader *nir = shaders[i];
7972       init_context(&ctx, nir);
7973
7974       setup_fp_mode(&ctx, nir);
7975
7976       if (!i) {
7977          /* needs to be after init_context() for FS */
7978          Pseudo_instruction *startpgm = add_startpgm(&ctx);
7979          append_logical_start(ctx.block);
7980          split_arguments(&ctx, startpgm);
7981       }
7982
7983       if_context ic;
7984       if (shader_count >= 2) {
7985          Builder bld(ctx.program, ctx.block);
7986          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7987          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
7988          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
7989
7990          begin_divergent_if_then(&ctx, &ic, cond);
7991       }
7992
7993       if (i) {
7994          Builder bld(ctx.program, ctx.block);
7995          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7996          bld.sopp(aco_opcode::s_barrier);
7997       }
7998
7999       if (ctx.stage == fragment_fs)
8000          handle_bc_optimize(&ctx);
8001
8002       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8003       visit_cf_list(&ctx, &func->body);
8004
8005       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8006          emit_streamout(&ctx, 0);
8007
8008       if (ctx.stage == vertex_vs)
8009          create_vs_exports(&ctx);
8010
8011       if (shader_count >= 2) {
8012          begin_divergent_if_else(&ctx, &ic);
8013          end_divergent_if(&ctx, &ic);
8014       }
8015
8016       ralloc_free(ctx.divergent_vals);
8017    }
8018
8019    program->config->float_mode = program->blocks[0].fp_mode.val;
8020
8021    append_logical_end(ctx.block);
8022    ctx.block->kind |= block_kind_uniform;
8023    Builder bld(ctx.program, ctx.block);
8024    if (ctx.program->wb_smem_l1_on_end)
8025       bld.smem(aco_opcode::s_dcache_wb, false);
8026    bld.sopp(aco_opcode::s_endpgm);
8027
8028    /* cleanup CFG */
8029    for (Block& BB : program->blocks) {
8030       for (unsigned idx : BB.linear_preds)
8031          program->blocks[idx].linear_succs.emplace_back(BB.index);
8032       for (unsigned idx : BB.logical_preds)
8033          program->blocks[idx].logical_succs.emplace_back(BB.index);
8034    }
8035 }
8036 }