src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.readlane(bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 439                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 440 {
 441    Builder bld(ctx->program, ctx->block);
 442    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 443    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 444    if (src1.type() == RegType::sgpr) {
 445       if (commutative && src0.type() == RegType::vgpr) {
 446          Temp t = src0;
 447          src0 = src1;
 448          src1 = t;
 449       } else if (src0.type() == RegType::vgpr &&
 450                  op != aco_opcode::v_madmk_f32 &&
 451                  op != aco_opcode::v_madak_f32 &&
 452                  op != aco_opcode::v_madmk_f16 &&
 453                  op != aco_opcode::v_madak_f16) {
 454          /* If the instruction is not commutative, we emit a VOP3A instruction */
 455          bld.vop2_e64(op, Definition(dst), src0, src1);
 456          return;
 457       } else {
 458          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 459       }
 460    }
 461
 462    if (flush_denorms && ctx->program->chip_class < GFX9) {
 463       assert(dst.size() == 1);
 464       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 465       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 466    } else {
 467       bld.vop2(op, Definition(dst), src0, src1);
 468    }
 469 }
 470
 471 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 472                             bool flush_denorms = false)
 473 {
 474    Temp src0 = get_alu_src(ctx, instr->src[0]);
 475    Temp src1 = get_alu_src(ctx, instr->src[1]);
 476    Temp src2 = get_alu_src(ctx, instr->src[2]);
 477
 478    /* ensure that the instruction has at most 1 sgpr operand
 479     * The optimizer will inline constants for us */
 480    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 481       src0 = as_vgpr(ctx, src0);
 482    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 483       src1 = as_vgpr(ctx, src1);
 484    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 485       src2 = as_vgpr(ctx, src2);
 486
 487    Builder bld(ctx->program, ctx->block);
 488    if (flush_denorms && ctx->program->chip_class < GFX9) {
 489       assert(dst.size() == 1);
 490       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 491       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 492    } else {
 493       bld.vop3(op, Definition(dst), src0, src1, src2);
 494    }
 495 }
 496
 497 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 498 {
 499    Builder bld(ctx->program, ctx->block);
 500    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 501 }
 502
 503 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 504 {
 505    Temp src0 = get_alu_src(ctx, instr->src[0]);
 506    Temp src1 = get_alu_src(ctx, instr->src[1]);
 507    assert(src0.size() == src1.size());
 508
 509    aco_ptr<Instruction> vopc;
 510    if (src1.type() == RegType::sgpr) {
 511       if (src0.type() == RegType::vgpr) {
 512          /* to swap the operands, we might also have to change the opcode */
 513          switch (op) {
 514             case aco_opcode::v_cmp_lt_f32:
 515                op = aco_opcode::v_cmp_gt_f32;
 516                break;
 517             case aco_opcode::v_cmp_ge_f32:
 518                op = aco_opcode::v_cmp_le_f32;
 519                break;
 520             case aco_opcode::v_cmp_lt_i32:
 521                op = aco_opcode::v_cmp_gt_i32;
 522                break;
 523             case aco_opcode::v_cmp_ge_i32:
 524                op = aco_opcode::v_cmp_le_i32;
 525                break;
 526             case aco_opcode::v_cmp_lt_u32:
 527                op = aco_opcode::v_cmp_gt_u32;
 528                break;
 529             case aco_opcode::v_cmp_ge_u32:
 530                op = aco_opcode::v_cmp_le_u32;
 531                break;
 532             case aco_opcode::v_cmp_lt_f64:
 533                op = aco_opcode::v_cmp_gt_f64;
 534                break;
 535             case aco_opcode::v_cmp_ge_f64:
 536                op = aco_opcode::v_cmp_le_f64;
 537                break;
 538             case aco_opcode::v_cmp_lt_i64:
 539                op = aco_opcode::v_cmp_gt_i64;
 540                break;
 541             case aco_opcode::v_cmp_ge_i64:
 542                op = aco_opcode::v_cmp_le_i64;
 543                break;
 544             case aco_opcode::v_cmp_lt_u64:
 545                op = aco_opcode::v_cmp_gt_u64;
 546                break;
 547             case aco_opcode::v_cmp_ge_u64:
 548                op = aco_opcode::v_cmp_le_u64;
 549                break;
 550             default: /* eq and ne are commutative */
 551                break;
 552          }
 553          Temp t = src0;
 554          src0 = src1;
 555          src1 = t;
 556       } else {
 557          src1 = as_vgpr(ctx, src1);
 558       }
 559    }
 560
 561    Builder bld(ctx->program, ctx->block);
 562    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 563 }
 564
 565 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 566 {
 567    Temp src0 = get_alu_src(ctx, instr->src[0]);
 568    Temp src1 = get_alu_src(ctx, instr->src[1]);
 569    Builder bld(ctx->program, ctx->block);
 570
 571    assert(dst.regClass() == bld.lm);
 572    assert(src0.type() == RegType::sgpr);
 573    assert(src1.type() == RegType::sgpr);
 574    assert(src0.regClass() == src1.regClass());
 575
 576    /* Emit the SALU comparison instruction */
 577    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 578    /* Turn the result into a per-lane bool */
 579    bool_to_vector_condition(ctx, cmp, dst);
 580 }
 581
 582 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 583                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 584 {
 585    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 586    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 587    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 588    bool use_valu = s_op == aco_opcode::num_opcodes ||
 589                    divergent_vals ||
 590                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 591                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 592    aco_opcode op = use_valu ? v_op : s_op;
 593    assert(op != aco_opcode::num_opcodes);
 594    assert(dst.regClass() == ctx->program->lane_mask);
 595
 596    if (use_valu)
 597       emit_vopc_instruction(ctx, instr, op, dst);
 598    else
 599       emit_sopc_instruction(ctx, instr, op, dst);
 600 }
 601
 602 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 603 {
 604    Builder bld(ctx->program, ctx->block);
 605    Temp src0 = get_alu_src(ctx, instr->src[0]);
 606    Temp src1 = get_alu_src(ctx, instr->src[1]);
 607
 608    assert(dst.regClass() == bld.lm);
 609    assert(src0.regClass() == bld.lm);
 610    assert(src1.regClass() == bld.lm);
 611
 612    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 613 }
 614
 615 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 616 {
 617    Builder bld(ctx->program, ctx->block);
 618    Temp cond = get_alu_src(ctx, instr->src[0]);
 619    Temp then = get_alu_src(ctx, instr->src[1]);
 620    Temp els = get_alu_src(ctx, instr->src[2]);
 621
 622    assert(cond.regClass() == bld.lm);
 623
 624    if (dst.type() == RegType::vgpr) {
 625       aco_ptr<Instruction> bcsel;
 626       if (dst.size() == 1) {
 627          then = as_vgpr(ctx, then);
 628          els = as_vgpr(ctx, els);
 629
 630          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 631       } else if (dst.size() == 2) {
 632          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 633          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 634          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 635          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 636
 637          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 638          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 639
 640          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 641       } else {
 642          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 643          nir_print_instr(&instr->instr, stderr);
 644          fprintf(stderr, "\n");
 645       }
 646       return;
 647    }
 648
 649    if (instr->dest.dest.ssa.bit_size == 1) {
 650       assert(dst.regClass() == bld.lm);
 651       assert(then.regClass() == bld.lm);
 652       assert(els.regClass() == bld.lm);
 653    }
 654
 655    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 656       if (dst.regClass() == s1 || dst.regClass() == s2) {
 657          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 658          assert(dst.size() == then.size());
 659          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 660          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 661       } else {
 662          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 663          nir_print_instr(&instr->instr, stderr);
 664          fprintf(stderr, "\n");
 665       }
 666       return;
 667    }
 668
 669    /* divergent boolean bcsel
 670     * this implements bcsel on bools: dst = s0 ? s1 : s2
 671     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 672    assert(instr->dest.dest.ssa.bit_size == 1);
 673
 674    if (cond.id() != then.id())
 675       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 676
 677    if (cond.id() == els.id())
 678       bld.sop1(Builder::s_mov, Definition(dst), then);
 679    else
 680       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 681                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 682 }
 683
 684 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 685                     aco_opcode op, uint32_t undo)
 686 {
 687    /* multiply by 16777216 to handle denormals */
 688    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 689                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 690    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 691    scaled = bld.vop1(op, bld.def(v1), scaled);
 692    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 693
 694    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 695
 696    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 697 }
 698
 699 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 700 {
 701    if (ctx->block->fp_mode.denorm32 == 0) {
 702       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 703       return;
 704    }
 705
 706    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 707 }
 708
 709 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 710 {
 711    if (ctx->block->fp_mode.denorm32 == 0) {
 712       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 713       return;
 714    }
 715
 716    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 717 }
 718
 719 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 720 {
 721    if (ctx->block->fp_mode.denorm32 == 0) {
 722       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 723       return;
 724    }
 725
 726    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 727 }
 728
 729 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 730 {
 731    if (ctx->block->fp_mode.denorm32 == 0) {
 732       bld.vop1(aco_opcode::v_log_f32, dst, val);
 733       return;
 734    }
 735
 736    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 737 }
 738
 739 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 740 {
 741    if (ctx->options->chip_class >= GFX7)
 742       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 743
 744    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 745    /* TODO: create more efficient code! */
 746    if (val.type() == RegType::sgpr)
 747       val = as_vgpr(ctx, val);
 748
 749    /* Split the input value. */
 750    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 751    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 752
 753    /* Extract the exponent and compute the unbiased value. */
 754    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 755
 756    /* Extract the fractional part. */
 757    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 758    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 759
 760    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 761    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 762
 763    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 764    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 765    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 766    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 767    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 768
 769    /* Get the sign bit. */
 770    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 771
 772    /* Decide the operation to apply depending on the unbiased exponent. */
 773    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 774    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 775    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 776    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 777    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 778    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 779
 780    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 781 }
 782
 783 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 784 {
 785    if (!instr->dest.dest.is_ssa) {
 786       fprintf(stderr, "nir alu dst not in ssa: ");
 787       nir_print_instr(&instr->instr, stderr);
 788       fprintf(stderr, "\n");
 789       abort();
 790    }
 791    Builder bld(ctx->program, ctx->block);
 792    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 793    switch(instr->op) {
 794    case nir_op_vec2:
 795    case nir_op_vec3:
 796    case nir_op_vec4: {
 797       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 798       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 799       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 800          elems[i] = get_alu_src(ctx, instr->src[i]);
 801          vec->operands[i] = Operand{elems[i]};
 802       }
 803       vec->definitions[0] = Definition(dst);
 804       ctx->block->instructions.emplace_back(std::move(vec));
 805       ctx->allocated_vec.emplace(dst.id(), elems);
 806       break;
 807    }
 808    case nir_op_mov: {
 809       Temp src = get_alu_src(ctx, instr->src[0]);
 810       aco_ptr<Instruction> mov;
 811       if (dst.type() == RegType::sgpr) {
 812          if (src.type() == RegType::vgpr)
 813             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 814          else if (src.regClass() == s1)
 815             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 816          else if (src.regClass() == s2)
 817             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 818          else
 819             unreachable("wrong src register class for nir_op_imov");
 820       } else if (dst.regClass() == v1) {
 821          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 822       } else if (dst.regClass() == v2) {
 823          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 824       } else {
 825          nir_print_instr(&instr->instr, stderr);
 826          unreachable("Should have been lowered to scalar.");
 827       }
 828       break;
 829    }
 830    case nir_op_inot: {
 831       Temp src = get_alu_src(ctx, instr->src[0]);
 832       if (instr->dest.dest.ssa.bit_size == 1) {
 833          assert(src.regClass() == bld.lm);
 834          assert(dst.regClass() == bld.lm);
 835          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 836          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 837          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 838       } else if (dst.regClass() == v1) {
 839          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 840       } else if (dst.type() == RegType::sgpr) {
 841          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 842          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 843       } else {
 844          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 845          nir_print_instr(&instr->instr, stderr);
 846          fprintf(stderr, "\n");
 847       }
 848       break;
 849    }
 850    case nir_op_ineg: {
 851       Temp src = get_alu_src(ctx, instr->src[0]);
 852       if (dst.regClass() == v1) {
 853          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 854       } else if (dst.regClass() == s1) {
 855          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 856       } else if (dst.size() == 2) {
 857          Temp src0 = bld.tmp(dst.type(), 1);
 858          Temp src1 = bld.tmp(dst.type(), 1);
 859          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 860
 861          if (dst.regClass() == s2) {
 862             Temp carry = bld.tmp(s1);
 863             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 864             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 865             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 866          } else {
 867             Temp lower = bld.tmp(v1);
 868             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 869             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 870             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 871          }
 872       } else {
 873          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 874          nir_print_instr(&instr->instr, stderr);
 875          fprintf(stderr, "\n");
 876       }
 877       break;
 878    }
 879    case nir_op_iabs: {
 880       if (dst.regClass() == s1) {
 881          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 882       } else if (dst.regClass() == v1) {
 883          Temp src = get_alu_src(ctx, instr->src[0]);
 884          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 885       } else {
 886          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 887          nir_print_instr(&instr->instr, stderr);
 888          fprintf(stderr, "\n");
 889       }
 890       break;
 891    }
 892    case nir_op_isign: {
 893       Temp src = get_alu_src(ctx, instr->src[0]);
 894       if (dst.regClass() == s1) {
 895          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 896          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 897          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 898       } else if (dst.regClass() == s2) {
 899          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 900          Temp neqz;
 901          if (ctx->program->chip_class >= GFX8)
 902             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 903          else
 904             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 905          /* SCC gets zero-extended to 64 bit */
 906          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 907       } else if (dst.regClass() == v1) {
 908          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 909          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 910          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 911       } else if (dst.regClass() == v2) {
 912          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 913          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 914          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 915          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 916          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 917          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 918       } else {
 919          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 920          nir_print_instr(&instr->instr, stderr);
 921          fprintf(stderr, "\n");
 922       }
 923       break;
 924    }
 925    case nir_op_imax: {
 926       if (dst.regClass() == v1) {
 927          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 928       } else if (dst.regClass() == s1) {
 929          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 930       } else {
 931          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 932          nir_print_instr(&instr->instr, stderr);
 933          fprintf(stderr, "\n");
 934       }
 935       break;
 936    }
 937    case nir_op_umax: {
 938       if (dst.regClass() == v1) {
 939          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 940       } else if (dst.regClass() == s1) {
 941          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 942       } else {
 943          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 944          nir_print_instr(&instr->instr, stderr);
 945          fprintf(stderr, "\n");
 946       }
 947       break;
 948    }
 949    case nir_op_imin: {
 950       if (dst.regClass() == v1) {
 951          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 952       } else if (dst.regClass() == s1) {
 953          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 954       } else {
 955          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 956          nir_print_instr(&instr->instr, stderr);
 957          fprintf(stderr, "\n");
 958       }
 959       break;
 960    }
 961    case nir_op_umin: {
 962       if (dst.regClass() == v1) {
 963          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 964       } else if (dst.regClass() == s1) {
 965          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 966       } else {
 967          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 968          nir_print_instr(&instr->instr, stderr);
 969          fprintf(stderr, "\n");
 970       }
 971       break;
 972    }
 973    case nir_op_ior: {
 974       if (instr->dest.dest.ssa.bit_size == 1) {
 975          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 976       } else if (dst.regClass() == v1) {
 977          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 978       } else if (dst.regClass() == s1) {
 979          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 980       } else if (dst.regClass() == s2) {
 981          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 982       } else {
 983          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 984          nir_print_instr(&instr->instr, stderr);
 985          fprintf(stderr, "\n");
 986       }
 987       break;
 988    }
 989    case nir_op_iand: {
 990       if (instr->dest.dest.ssa.bit_size == 1) {
 991          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 992       } else if (dst.regClass() == v1) {
 993          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 994       } else if (dst.regClass() == s1) {
 995          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 996       } else if (dst.regClass() == s2) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_ixor: {
1006       if (instr->dest.dest.ssa.bit_size == 1) {
1007          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1008       } else if (dst.regClass() == v1) {
1009          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1010       } else if (dst.regClass() == s1) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1012       } else if (dst.regClass() == s2) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1014       } else {
1015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1016          nir_print_instr(&instr->instr, stderr);
1017          fprintf(stderr, "\n");
1018       }
1019       break;
1020    }
1021    case nir_op_ushr: {
1022       if (dst.regClass() == v1) {
1023          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1024       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1025          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1026                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1027       } else if (dst.regClass() == v2) {
1028          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1029                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1030       } else if (dst.regClass() == s2) {
1031          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1032       } else if (dst.regClass() == s1) {
1033          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1034       } else {
1035          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1036          nir_print_instr(&instr->instr, stderr);
1037          fprintf(stderr, "\n");
1038       }
1039       break;
1040    }
1041    case nir_op_ishl: {
1042       if (dst.regClass() == v1) {
1043          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1044       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1045          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1046                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1047       } else if (dst.regClass() == v2) {
1048          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1049                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1050       } else if (dst.regClass() == s1) {
1051          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1052       } else if (dst.regClass() == s2) {
1053          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1054       } else {
1055          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1056          nir_print_instr(&instr->instr, stderr);
1057          fprintf(stderr, "\n");
1058       }
1059       break;
1060    }
1061    case nir_op_ishr: {
1062       if (dst.regClass() == v1) {
1063          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1064       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1065          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1066                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1067       } else if (dst.regClass() == v2) {
1068          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1069                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1070       } else if (dst.regClass() == s1) {
1071          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1072       } else if (dst.regClass() == s2) {
1073          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1074       } else {
1075          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1076          nir_print_instr(&instr->instr, stderr);
1077          fprintf(stderr, "\n");
1078       }
1079       break;
1080    }
1081    case nir_op_find_lsb: {
1082       Temp src = get_alu_src(ctx, instr->src[0]);
1083       if (src.regClass() == s1) {
1084          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1085       } else if (src.regClass() == v1) {
1086          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1087       } else if (src.regClass() == s2) {
1088          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1089       } else {
1090          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1091          nir_print_instr(&instr->instr, stderr);
1092          fprintf(stderr, "\n");
1093       }
1094       break;
1095    }
1096    case nir_op_ufind_msb:
1097    case nir_op_ifind_msb: {
1098       Temp src = get_alu_src(ctx, instr->src[0]);
1099       if (src.regClass() == s1 || src.regClass() == s2) {
1100          aco_opcode op = src.regClass() == s2 ?
1101                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1102                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1103          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1104
1105          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1106                                         Operand(src.size() * 32u - 1u), msb_rev);
1107          Temp msb = sub.def(0).getTemp();
1108          Temp carry = sub.def(1).getTemp();
1109
1110          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1111       } else if (src.regClass() == v1) {
1112          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1113          Temp msb_rev = bld.tmp(v1);
1114          emit_vop1_instruction(ctx, instr, op, msb_rev);
1115          Temp msb = bld.tmp(v1);
1116          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1117          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1118       } else {
1119          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1120          nir_print_instr(&instr->instr, stderr);
1121          fprintf(stderr, "\n");
1122       }
1123       break;
1124    }
1125    case nir_op_bitfield_reverse: {
1126       if (dst.regClass() == s1) {
1127          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1128       } else if (dst.regClass() == v1) {
1129          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1130       } else {
1131          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1132          nir_print_instr(&instr->instr, stderr);
1133          fprintf(stderr, "\n");
1134       }
1135       break;
1136    }
1137    case nir_op_iadd: {
1138       if (dst.regClass() == s1) {
1139          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1140          break;
1141       }
1142
1143       Temp src0 = get_alu_src(ctx, instr->src[0]);
1144       Temp src1 = get_alu_src(ctx, instr->src[1]);
1145       if (dst.regClass() == v1) {
1146          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1147          break;
1148       }
1149
1150       assert(src0.size() == 2 && src1.size() == 2);
1151       Temp src00 = bld.tmp(src0.type(), 1);
1152       Temp src01 = bld.tmp(dst.type(), 1);
1153       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1154       Temp src10 = bld.tmp(src1.type(), 1);
1155       Temp src11 = bld.tmp(dst.type(), 1);
1156       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1157
1158       if (dst.regClass() == s2) {
1159          Temp carry = bld.tmp(s1);
1160          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1161          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1162          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1163       } else if (dst.regClass() == v2) {
1164          Temp dst0 = bld.tmp(v1);
1165          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1166          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1167          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1168       } else {
1169          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1170          nir_print_instr(&instr->instr, stderr);
1171          fprintf(stderr, "\n");
1172       }
1173       break;
1174    }
1175    case nir_op_uadd_sat: {
1176       Temp src0 = get_alu_src(ctx, instr->src[0]);
1177       Temp src1 = get_alu_src(ctx, instr->src[1]);
1178       if (dst.regClass() == s1) {
1179          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1180          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1181                   src0, src1);
1182          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1183       } else if (dst.regClass() == v1) {
1184          if (ctx->options->chip_class >= GFX9) {
1185             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1186             add->operands[0] = Operand(src0);
1187             add->operands[1] = Operand(src1);
1188             add->definitions[0] = Definition(dst);
1189             add->clamp = 1;
1190             ctx->block->instructions.emplace_back(std::move(add));
1191          } else {
1192             if (src1.regClass() != v1)
1193                std::swap(src0, src1);
1194             assert(src1.regClass() == v1);
1195             Temp tmp = bld.tmp(v1);
1196             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1197             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1198          }
1199       } else {
1200          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1201          nir_print_instr(&instr->instr, stderr);
1202          fprintf(stderr, "\n");
1203       }
1204       break;
1205    }
1206    case nir_op_uadd_carry: {
1207       Temp src0 = get_alu_src(ctx, instr->src[0]);
1208       Temp src1 = get_alu_src(ctx, instr->src[1]);
1209       if (dst.regClass() == s1) {
1210          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1211          break;
1212       }
1213       if (dst.regClass() == v1) {
1214          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1215          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1216          break;
1217       }
1218
1219       Temp src00 = bld.tmp(src0.type(), 1);
1220       Temp src01 = bld.tmp(dst.type(), 1);
1221       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1222       Temp src10 = bld.tmp(src1.type(), 1);
1223       Temp src11 = bld.tmp(dst.type(), 1);
1224       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1225       if (dst.regClass() == s2) {
1226          Temp carry = bld.tmp(s1);
1227          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1228          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1230       } else if (dst.regClass() == v2) {
1231          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1232          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1233          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1234          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1235       } else {
1236          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1237          nir_print_instr(&instr->instr, stderr);
1238          fprintf(stderr, "\n");
1239       }
1240       break;
1241    }
1242    case nir_op_isub: {
1243       if (dst.regClass() == s1) {
1244          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1245          break;
1246       }
1247
1248       Temp src0 = get_alu_src(ctx, instr->src[0]);
1249       Temp src1 = get_alu_src(ctx, instr->src[1]);
1250       if (dst.regClass() == v1) {
1251          bld.vsub32(Definition(dst), src0, src1);
1252          break;
1253       }
1254
1255       Temp src00 = bld.tmp(src0.type(), 1);
1256       Temp src01 = bld.tmp(dst.type(), 1);
1257       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1258       Temp src10 = bld.tmp(src1.type(), 1);
1259       Temp src11 = bld.tmp(dst.type(), 1);
1260       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1261       if (dst.regClass() == s2) {
1262          Temp carry = bld.tmp(s1);
1263          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1264          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1265          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1266       } else if (dst.regClass() == v2) {
1267          Temp lower = bld.tmp(v1);
1268          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1269          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1270          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1271       } else {
1272          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1273          nir_print_instr(&instr->instr, stderr);
1274          fprintf(stderr, "\n");
1275       }
1276       break;
1277    }
1278    case nir_op_usub_borrow: {
1279       Temp src0 = get_alu_src(ctx, instr->src[0]);
1280       Temp src1 = get_alu_src(ctx, instr->src[1]);
1281       if (dst.regClass() == s1) {
1282          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1283          break;
1284       } else if (dst.regClass() == v1) {
1285          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1286          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1287          break;
1288       }
1289
1290       Temp src00 = bld.tmp(src0.type(), 1);
1291       Temp src01 = bld.tmp(dst.type(), 1);
1292       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1293       Temp src10 = bld.tmp(src1.type(), 1);
1294       Temp src11 = bld.tmp(dst.type(), 1);
1295       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1296       if (dst.regClass() == s2) {
1297          Temp borrow = bld.tmp(s1);
1298          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1299          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1300          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1301       } else if (dst.regClass() == v2) {
1302          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1303          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1304          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1305          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1306       } else {
1307          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1308          nir_print_instr(&instr->instr, stderr);
1309          fprintf(stderr, "\n");
1310       }
1311       break;
1312    }
1313    case nir_op_imul: {
1314       if (dst.regClass() == v1) {
1315          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1316                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1317       } else if (dst.regClass() == s1) {
1318          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1319       } else {
1320          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1321          nir_print_instr(&instr->instr, stderr);
1322          fprintf(stderr, "\n");
1323       }
1324       break;
1325    }
1326    case nir_op_umul_high: {
1327       if (dst.regClass() == v1) {
1328          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1329       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1330          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1331       } else if (dst.regClass() == s1) {
1332          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1333                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1334          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1335       } else {
1336          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1337          nir_print_instr(&instr->instr, stderr);
1338          fprintf(stderr, "\n");
1339       }
1340       break;
1341    }
1342    case nir_op_imul_high: {
1343       if (dst.regClass() == v1) {
1344          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1345       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1346          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1347       } else if (dst.regClass() == s1) {
1348          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1349                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1350          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_fmul: {
1359       if (dst.size() == 1) {
1360          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1361       } else if (dst.size() == 2) {
1362          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1363                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1364       } else {
1365          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1366          nir_print_instr(&instr->instr, stderr);
1367          fprintf(stderr, "\n");
1368       }
1369       break;
1370    }
1371    case nir_op_fadd: {
1372       if (dst.size() == 1) {
1373          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1374       } else if (dst.size() == 2) {
1375          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1376                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1377       } else {
1378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1379          nir_print_instr(&instr->instr, stderr);
1380          fprintf(stderr, "\n");
1381       }
1382       break;
1383    }
1384    case nir_op_fsub: {
1385       Temp src0 = get_alu_src(ctx, instr->src[0]);
1386       Temp src1 = get_alu_src(ctx, instr->src[1]);
1387       if (dst.size() == 1) {
1388          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1389             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1390          else
1391             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1392       } else if (dst.size() == 2) {
1393          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1394                                      get_alu_src(ctx, instr->src[0]),
1395                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1396          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1397          sub->neg[1] = true;
1398       } else {
1399          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1400          nir_print_instr(&instr->instr, stderr);
1401          fprintf(stderr, "\n");
1402       }
1403       break;
1404    }
1405    case nir_op_fmax: {
1406       if (dst.size() == 1) {
1407          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1408       } else if (dst.size() == 2) {
1409          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1410             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1411                                 get_alu_src(ctx, instr->src[0]),
1412                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1413             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1414          } else {
1415             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1416                      get_alu_src(ctx, instr->src[0]),
1417                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1418          }
1419       } else {
1420          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1421          nir_print_instr(&instr->instr, stderr);
1422          fprintf(stderr, "\n");
1423       }
1424       break;
1425    }
1426    case nir_op_fmin: {
1427       if (dst.size() == 1) {
1428          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1429       } else if (dst.size() == 2) {
1430          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1431             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1432                                 get_alu_src(ctx, instr->src[0]),
1433                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1434             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1435          } else {
1436             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1437                      get_alu_src(ctx, instr->src[0]),
1438                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1439          }
1440       } else {
1441          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1442          nir_print_instr(&instr->instr, stderr);
1443          fprintf(stderr, "\n");
1444       }
1445       break;
1446    }
1447    case nir_op_fmax3: {
1448       if (dst.size() == 1) {
1449          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1450       } else {
1451          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1452          nir_print_instr(&instr->instr, stderr);
1453          fprintf(stderr, "\n");
1454       }
1455       break;
1456    }
1457    case nir_op_fmin3: {
1458       if (dst.size() == 1) {
1459          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1460       } else {
1461          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1462          nir_print_instr(&instr->instr, stderr);
1463          fprintf(stderr, "\n");
1464       }
1465       break;
1466    }
1467    case nir_op_fmed3: {
1468       if (dst.size() == 1) {
1469          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1470       } else {
1471          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1472          nir_print_instr(&instr->instr, stderr);
1473          fprintf(stderr, "\n");
1474       }
1475       break;
1476    }
1477    case nir_op_umax3: {
1478       if (dst.size() == 1) {
1479          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1480       } else {
1481          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1482          nir_print_instr(&instr->instr, stderr);
1483          fprintf(stderr, "\n");
1484       }
1485       break;
1486    }
1487    case nir_op_umin3: {
1488       if (dst.size() == 1) {
1489          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1490       } else {
1491          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1492          nir_print_instr(&instr->instr, stderr);
1493          fprintf(stderr, "\n");
1494       }
1495       break;
1496    }
1497    case nir_op_umed3: {
1498       if (dst.size() == 1) {
1499          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1500       } else {
1501          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1502          nir_print_instr(&instr->instr, stderr);
1503          fprintf(stderr, "\n");
1504       }
1505       break;
1506    }
1507    case nir_op_imax3: {
1508       if (dst.size() == 1) {
1509          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1510       } else {
1511          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1512          nir_print_instr(&instr->instr, stderr);
1513          fprintf(stderr, "\n");
1514       }
1515       break;
1516    }
1517    case nir_op_imin3: {
1518       if (dst.size() == 1) {
1519          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1520       } else {
1521          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1522          nir_print_instr(&instr->instr, stderr);
1523          fprintf(stderr, "\n");
1524       }
1525       break;
1526    }
1527    case nir_op_imed3: {
1528       if (dst.size() == 1) {
1529          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1530       } else {
1531          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1532          nir_print_instr(&instr->instr, stderr);
1533          fprintf(stderr, "\n");
1534       }
1535       break;
1536    }
1537    case nir_op_cube_face_coord: {
1538       Temp in = get_alu_src(ctx, instr->src[0], 3);
1539       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1540                       emit_extract_vector(ctx, in, 1, v1),
1541                       emit_extract_vector(ctx, in, 2, v1) };
1542       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1543       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1544       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1545       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1546       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1547       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1548       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1549       break;
1550    }
1551    case nir_op_cube_face_index: {
1552       Temp in = get_alu_src(ctx, instr->src[0], 3);
1553       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1554                       emit_extract_vector(ctx, in, 1, v1),
1555                       emit_extract_vector(ctx, in, 2, v1) };
1556       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1557       break;
1558    }
1559    case nir_op_bcsel: {
1560       emit_bcsel(ctx, instr, dst);
1561       break;
1562    }
1563    case nir_op_frsq: {
1564       if (dst.size() == 1) {
1565          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1566       } else if (dst.size() == 2) {
1567          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1568       } else {
1569          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1570          nir_print_instr(&instr->instr, stderr);
1571          fprintf(stderr, "\n");
1572       }
1573       break;
1574    }
1575    case nir_op_fneg: {
1576       Temp src = get_alu_src(ctx, instr->src[0]);
1577       if (dst.size() == 1) {
1578          if (ctx->block->fp_mode.must_flush_denorms32)
1579             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1580          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1581       } else if (dst.size() == 2) {
1582          if (ctx->block->fp_mode.must_flush_denorms16_64)
1583             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1584          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1585          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1586          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1587          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1588       } else {
1589          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1590          nir_print_instr(&instr->instr, stderr);
1591          fprintf(stderr, "\n");
1592       }
1593       break;
1594    }
1595    case nir_op_fabs: {
1596       Temp src = get_alu_src(ctx, instr->src[0]);
1597       if (dst.size() == 1) {
1598          if (ctx->block->fp_mode.must_flush_denorms32)
1599             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1600          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1601       } else if (dst.size() == 2) {
1602          if (ctx->block->fp_mode.must_flush_denorms16_64)
1603             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1604          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1605          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1606          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1607          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1608       } else {
1609          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1610          nir_print_instr(&instr->instr, stderr);
1611          fprintf(stderr, "\n");
1612       }
1613       break;
1614    }
1615    case nir_op_fsat: {
1616       Temp src = get_alu_src(ctx, instr->src[0]);
1617       if (dst.size() == 1) {
1618          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1619          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1620          // TODO: confirm that this holds under any circumstances
1621       } else if (dst.size() == 2) {
1622          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1623          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1624          vop3->clamp = true;
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_flog2: {
1633       if (dst.size() == 1) {
1634          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1635       } else {
1636          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1637          nir_print_instr(&instr->instr, stderr);
1638          fprintf(stderr, "\n");
1639       }
1640       break;
1641    }
1642    case nir_op_frcp: {
1643       if (dst.size() == 1) {
1644          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1645       } else if (dst.size() == 2) {
1646          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1647       } else {
1648          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1649          nir_print_instr(&instr->instr, stderr);
1650          fprintf(stderr, "\n");
1651       }
1652       break;
1653    }
1654    case nir_op_fexp2: {
1655       if (dst.size() == 1) {
1656          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_fsqrt: {
1665       if (dst.size() == 1) {
1666          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1667       } else if (dst.size() == 2) {
1668          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_ffract: {
1677       if (dst.size() == 1) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1679       } else if (dst.size() == 2) {
1680          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1681       } else {
1682          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1683          nir_print_instr(&instr->instr, stderr);
1684          fprintf(stderr, "\n");
1685       }
1686       break;
1687    }
1688    case nir_op_ffloor: {
1689       if (dst.size() == 1) {
1690          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1691       } else if (dst.size() == 2) {
1692          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1693       } else {
1694          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1695          nir_print_instr(&instr->instr, stderr);
1696          fprintf(stderr, "\n");
1697       }
1698       break;
1699    }
1700    case nir_op_fceil: {
1701       if (dst.size() == 1) {
1702          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1703       } else if (dst.size() == 2) {
1704          if (ctx->options->chip_class >= GFX7) {
1705             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1706          } else {
1707             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1708             Temp src0 = get_alu_src(ctx, instr->src[0]);
1709
1710             /* trunc = trunc(src0)
1711              * if (src0 > 0.0 && src0 != trunc)
1712              *    trunc += 1.0
1713              */
1714             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1715             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1716             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1717             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1718             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1719             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1720             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1721          }
1722       } else {
1723          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1724          nir_print_instr(&instr->instr, stderr);
1725          fprintf(stderr, "\n");
1726       }
1727       break;
1728    }
1729    case nir_op_ftrunc: {
1730       if (dst.size() == 1) {
1731          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1732       } else if (dst.size() == 2) {
1733          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1734       } else {
1735          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1736          nir_print_instr(&instr->instr, stderr);
1737          fprintf(stderr, "\n");
1738       }
1739       break;
1740    }
1741    case nir_op_fround_even: {
1742       if (dst.size() == 1) {
1743          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1744       } else if (dst.size() == 2) {
1745          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1746       } else {
1747          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1748          nir_print_instr(&instr->instr, stderr);
1749          fprintf(stderr, "\n");
1750       }
1751       break;
1752    }
1753    case nir_op_fsin:
1754    case nir_op_fcos: {
1755       Temp src = get_alu_src(ctx, instr->src[0]);
1756       aco_ptr<Instruction> norm;
1757       if (dst.size() == 1) {
1758          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1759          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1760
1761          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1762          if (ctx->options->chip_class < GFX9)
1763             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1764
1765          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1766          bld.vop1(opcode, Definition(dst), tmp);
1767       } else {
1768          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1769          nir_print_instr(&instr->instr, stderr);
1770          fprintf(stderr, "\n");
1771       }
1772       break;
1773    }
1774    case nir_op_ldexp: {
1775       if (dst.size() == 1) {
1776          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1777                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1778                   get_alu_src(ctx, instr->src[1]));
1779       } else if (dst.size() == 2) {
1780          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1781                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1782                   get_alu_src(ctx, instr->src[1]));
1783       } else {
1784          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1785          nir_print_instr(&instr->instr, stderr);
1786          fprintf(stderr, "\n");
1787       }
1788       break;
1789    }
1790    case nir_op_frexp_sig: {
1791       if (dst.size() == 1) {
1792          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1793                   get_alu_src(ctx, instr->src[0]));
1794       } else if (dst.size() == 2) {
1795          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1796                   get_alu_src(ctx, instr->src[0]));
1797       } else {
1798          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1799          nir_print_instr(&instr->instr, stderr);
1800          fprintf(stderr, "\n");
1801       }
1802       break;
1803    }
1804    case nir_op_frexp_exp: {
1805       if (instr->src[0].src.ssa->bit_size == 32) {
1806          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1807                   get_alu_src(ctx, instr->src[0]));
1808       } else if (instr->src[0].src.ssa->bit_size == 64) {
1809          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1810                   get_alu_src(ctx, instr->src[0]));
1811       } else {
1812          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1813          nir_print_instr(&instr->instr, stderr);
1814          fprintf(stderr, "\n");
1815       }
1816       break;
1817    }
1818    case nir_op_fsign: {
1819       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1820       if (dst.size() == 1) {
1821          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1822          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1823          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1824          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1825       } else if (dst.size() == 2) {
1826          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1827          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1828          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1829
1830          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1831          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1832          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1833
1834          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1835       } else {
1836          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1837          nir_print_instr(&instr->instr, stderr);
1838          fprintf(stderr, "\n");
1839       }
1840       break;
1841    }
1842    case nir_op_f2f32: {
1843       if (instr->src[0].src.ssa->bit_size == 64) {
1844          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1845       } else {
1846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1847          nir_print_instr(&instr->instr, stderr);
1848          fprintf(stderr, "\n");
1849       }
1850       break;
1851    }
1852    case nir_op_f2f64: {
1853       if (instr->src[0].src.ssa->bit_size == 32) {
1854          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1855       } else {
1856          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1857          nir_print_instr(&instr->instr, stderr);
1858          fprintf(stderr, "\n");
1859       }
1860       break;
1861    }
1862    case nir_op_i2f32: {
1863       assert(dst.size() == 1);
1864       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1865       break;
1866    }
1867    case nir_op_i2f64: {
1868       if (instr->src[0].src.ssa->bit_size == 32) {
1869          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1870       } else if (instr->src[0].src.ssa->bit_size == 64) {
1871          Temp src = get_alu_src(ctx, instr->src[0]);
1872          RegClass rc = RegClass(src.type(), 1);
1873          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1874          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1875          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1876          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1877          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1878          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1879
1880       } else {
1881          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1882          nir_print_instr(&instr->instr, stderr);
1883          fprintf(stderr, "\n");
1884       }
1885       break;
1886    }
1887    case nir_op_u2f32: {
1888       assert(dst.size() == 1);
1889       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1890       break;
1891    }
1892    case nir_op_u2f64: {
1893       if (instr->src[0].src.ssa->bit_size == 32) {
1894          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1895       } else if (instr->src[0].src.ssa->bit_size == 64) {
1896          Temp src = get_alu_src(ctx, instr->src[0]);
1897          RegClass rc = RegClass(src.type(), 1);
1898          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1899          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1900          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1901          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1902          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1903          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_f2i32: {
1912       Temp src = get_alu_src(ctx, instr->src[0]);
1913       if (instr->src[0].src.ssa->bit_size == 32) {
1914          if (dst.type() == RegType::vgpr)
1915             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1916          else
1917             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1918                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1919
1920       } else if (instr->src[0].src.ssa->bit_size == 64) {
1921          if (dst.type() == RegType::vgpr)
1922             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1923          else
1924             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1925                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1926
1927       } else {
1928          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1929          nir_print_instr(&instr->instr, stderr);
1930          fprintf(stderr, "\n");
1931       }
1932       break;
1933    }
1934    case nir_op_f2u32: {
1935       Temp src = get_alu_src(ctx, instr->src[0]);
1936       if (instr->src[0].src.ssa->bit_size == 32) {
1937          if (dst.type() == RegType::vgpr)
1938             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1939          else
1940             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1941                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1942
1943       } else if (instr->src[0].src.ssa->bit_size == 64) {
1944          if (dst.type() == RegType::vgpr)
1945             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1946          else
1947             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1948                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1949
1950       } else {
1951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1952          nir_print_instr(&instr->instr, stderr);
1953          fprintf(stderr, "\n");
1954       }
1955       break;
1956    }
1957    case nir_op_f2i64: {
1958       Temp src = get_alu_src(ctx, instr->src[0]);
1959       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1960          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1961          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1962          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1963          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1964          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1965          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1966          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1967          Temp new_exponent = bld.tmp(v1);
1968          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1969          if (ctx->program->chip_class >= GFX8)
1970             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1971          else
1972             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
1973          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1974          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1975          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1976          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1977          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1978          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1979          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1980          Temp new_lower = bld.tmp(v1);
1981          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1982          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1983          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1984
1985       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1986          if (src.type() == RegType::vgpr)
1987             src = bld.as_uniform(src);
1988          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1989          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1990          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1991          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1992          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1993          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1994          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1995          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1996          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1997          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1998          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1999          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2000          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2001          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2002          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2003          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2004          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2005          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2006          Temp borrow = bld.tmp(s1);
2007          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2008          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2009          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2010
2011       } else if (instr->src[0].src.ssa->bit_size == 64) {
2012          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2013          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
2014          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2015          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2016          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
2017          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2018          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2019          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2020          if (dst.type() == RegType::sgpr) {
2021             lower = bld.as_uniform(lower);
2022             upper = bld.as_uniform(upper);
2023          }
2024          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2025
2026       } else {
2027          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2028          nir_print_instr(&instr->instr, stderr);
2029          fprintf(stderr, "\n");
2030       }
2031       break;
2032    }
2033    case nir_op_f2u64: {
2034       Temp src = get_alu_src(ctx, instr->src[0]);
2035       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2036          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2037          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2038          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2039          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2040          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2041          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2042          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2043          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2044          Temp new_exponent = bld.tmp(v1);
2045          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2046          if (ctx->program->chip_class >= GFX8)
2047             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2048          else
2049             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2050          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2051          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2052          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2053          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2054          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2055          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2056          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2057
2058       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2059          if (src.type() == RegType::vgpr)
2060             src = bld.as_uniform(src);
2061          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2062          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2063          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2064          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2065          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2066          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2067          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2068          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2069          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2070          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2071          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2072          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2073          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2074          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2075          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2076          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2077          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2078          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2079
2080       } else if (instr->src[0].src.ssa->bit_size == 64) {
2081          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2082          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
2083          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2084          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2085          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
2086          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2087          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2088          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2089          if (dst.type() == RegType::sgpr) {
2090             lower = bld.as_uniform(lower);
2091             upper = bld.as_uniform(upper);
2092          }
2093          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2094
2095       } else {
2096          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2097          nir_print_instr(&instr->instr, stderr);
2098          fprintf(stderr, "\n");
2099       }
2100       break;
2101    }
2102    case nir_op_b2f32: {
2103       Temp src = get_alu_src(ctx, instr->src[0]);
2104       assert(src.regClass() == bld.lm);
2105
2106       if (dst.regClass() == s1) {
2107          src = bool_to_scalar_condition(ctx, src);
2108          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2109       } else if (dst.regClass() == v1) {
2110          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2111       } else {
2112          unreachable("Wrong destination register class for nir_op_b2f32.");
2113       }
2114       break;
2115    }
2116    case nir_op_b2f64: {
2117       Temp src = get_alu_src(ctx, instr->src[0]);
2118       assert(src.regClass() == bld.lm);
2119
2120       if (dst.regClass() == s2) {
2121          src = bool_to_scalar_condition(ctx, src);
2122          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2123       } else if (dst.regClass() == v2) {
2124          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2125          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2126          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2127       } else {
2128          unreachable("Wrong destination register class for nir_op_b2f64.");
2129       }
2130       break;
2131    }
2132    case nir_op_i2i32: {
2133       Temp src = get_alu_src(ctx, instr->src[0]);
2134       if (instr->src[0].src.ssa->bit_size == 64) {
2135          /* we can actually just say dst = src, as it would map the lower register */
2136          emit_extract_vector(ctx, src, 0, dst);
2137       } else {
2138          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2139          nir_print_instr(&instr->instr, stderr);
2140          fprintf(stderr, "\n");
2141       }
2142       break;
2143    }
2144    case nir_op_u2u32: {
2145       Temp src = get_alu_src(ctx, instr->src[0]);
2146       if (instr->src[0].src.ssa->bit_size == 16) {
2147          if (dst.regClass() == s1) {
2148             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2149          } else {
2150             // TODO: do better with SDWA
2151             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2152          }
2153       } else if (instr->src[0].src.ssa->bit_size == 64) {
2154          /* we can actually just say dst = src, as it would map the lower register */
2155          emit_extract_vector(ctx, src, 0, dst);
2156       } else {
2157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2158          nir_print_instr(&instr->instr, stderr);
2159          fprintf(stderr, "\n");
2160       }
2161       break;
2162    }
2163    case nir_op_i2i64: {
2164       Temp src = get_alu_src(ctx, instr->src[0]);
2165       if (src.regClass() == s1) {
2166          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2167          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2168       } else if (src.regClass() == v1) {
2169          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2170          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2171       } else {
2172          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2173          nir_print_instr(&instr->instr, stderr);
2174          fprintf(stderr, "\n");
2175       }
2176       break;
2177    }
2178    case nir_op_u2u64: {
2179       Temp src = get_alu_src(ctx, instr->src[0]);
2180       if (instr->src[0].src.ssa->bit_size == 32) {
2181          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2182       } else {
2183          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2184          nir_print_instr(&instr->instr, stderr);
2185          fprintf(stderr, "\n");
2186       }
2187       break;
2188    }
2189    case nir_op_b2i32: {
2190       Temp src = get_alu_src(ctx, instr->src[0]);
2191       assert(src.regClass() == bld.lm);
2192
2193       if (dst.regClass() == s1) {
2194          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2195          bool_to_scalar_condition(ctx, src, dst);
2196       } else if (dst.regClass() == v1) {
2197          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2198       } else {
2199          unreachable("Invalid register class for b2i32");
2200       }
2201       break;
2202    }
2203    case nir_op_i2b1: {
2204       Temp src = get_alu_src(ctx, instr->src[0]);
2205       assert(dst.regClass() == bld.lm);
2206
2207       if (src.type() == RegType::vgpr) {
2208          assert(src.regClass() == v1 || src.regClass() == v2);
2209          assert(dst.regClass() == bld.lm);
2210          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2211                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2212       } else {
2213          assert(src.regClass() == s1 || src.regClass() == s2);
2214          Temp tmp;
2215          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2216             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2217          } else {
2218             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2219                            bld.scc(bld.def(s1)), Operand(0u), src);
2220          }
2221          bool_to_vector_condition(ctx, tmp, dst);
2222       }
2223       break;
2224    }
2225    case nir_op_pack_64_2x32_split: {
2226       Temp src0 = get_alu_src(ctx, instr->src[0]);
2227       Temp src1 = get_alu_src(ctx, instr->src[1]);
2228
2229       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2230       break;
2231    }
2232    case nir_op_unpack_64_2x32_split_x:
2233       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2234       break;
2235    case nir_op_unpack_64_2x32_split_y:
2236       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2237       break;
2238    case nir_op_pack_half_2x16: {
2239       Temp src = get_alu_src(ctx, instr->src[0], 2);
2240
2241       if (dst.regClass() == v1) {
2242          Temp src0 = bld.tmp(v1);
2243          Temp src1 = bld.tmp(v1);
2244          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2245          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2246             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2247          else
2248             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2249                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2250                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2251       } else {
2252          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2253          nir_print_instr(&instr->instr, stderr);
2254          fprintf(stderr, "\n");
2255       }
2256       break;
2257    }
2258    case nir_op_unpack_half_2x16_split_x: {
2259       if (dst.regClass() == v1) {
2260          Builder bld(ctx->program, ctx->block);
2261          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2262       } else {
2263          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2264          nir_print_instr(&instr->instr, stderr);
2265          fprintf(stderr, "\n");
2266       }
2267       break;
2268    }
2269    case nir_op_unpack_half_2x16_split_y: {
2270       if (dst.regClass() == v1) {
2271          Builder bld(ctx->program, ctx->block);
2272          /* TODO: use SDWA here */
2273          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2274                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2275       } else {
2276          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2277          nir_print_instr(&instr->instr, stderr);
2278          fprintf(stderr, "\n");
2279       }
2280       break;
2281    }
2282    case nir_op_fquantize2f16: {
2283       Temp src = get_alu_src(ctx, instr->src[0]);
2284       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2285       Temp f32, cmp_res;
2286
2287       if (ctx->program->chip_class >= GFX8) {
2288          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2289          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2290          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2291       } else {
2292          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2293           * so compare the result and flush to 0 if it's smaller.
2294           */
2295          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2296          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2297          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2298          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2299          cmp_res = vop3->definitions[0].getTemp();
2300       }
2301
2302       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2303          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2304          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2305       } else {
2306          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2307       }
2308       break;
2309    }
2310    case nir_op_bfm: {
2311       Temp bits = get_alu_src(ctx, instr->src[0]);
2312       Temp offset = get_alu_src(ctx, instr->src[1]);
2313
2314       if (dst.regClass() == s1) {
2315          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2316       } else if (dst.regClass() == v1) {
2317          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2318       } else {
2319          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2320          nir_print_instr(&instr->instr, stderr);
2321          fprintf(stderr, "\n");
2322       }
2323       break;
2324    }
2325    case nir_op_bitfield_select: {
2326       /* (mask & insert) | (~mask & base) */
2327       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2328       Temp insert = get_alu_src(ctx, instr->src[1]);
2329       Temp base = get_alu_src(ctx, instr->src[2]);
2330
2331       /* dst = (insert & bitmask) | (base & ~bitmask) */
2332       if (dst.regClass() == s1) {
2333          aco_ptr<Instruction> sop2;
2334          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2335          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2336          Operand lhs;
2337          if (const_insert && const_bitmask) {
2338             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2339          } else {
2340             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2341             lhs = Operand(insert);
2342          }
2343
2344          Operand rhs;
2345          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2346          if (const_base && const_bitmask) {
2347             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2348          } else {
2349             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2350             rhs = Operand(base);
2351          }
2352
2353          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2354
2355       } else if (dst.regClass() == v1) {
2356          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2357             base = as_vgpr(ctx, base);
2358          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2359             insert = as_vgpr(ctx, insert);
2360
2361          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2362
2363       } else {
2364          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2365          nir_print_instr(&instr->instr, stderr);
2366          fprintf(stderr, "\n");
2367       }
2368       break;
2369    }
2370    case nir_op_ubfe:
2371    case nir_op_ibfe: {
2372       Temp base = get_alu_src(ctx, instr->src[0]);
2373       Temp offset = get_alu_src(ctx, instr->src[1]);
2374       Temp bits = get_alu_src(ctx, instr->src[2]);
2375
2376       if (dst.type() == RegType::sgpr) {
2377          Operand extract;
2378          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2379          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2380          if (const_offset && const_bits) {
2381             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2382             extract = Operand(const_extract);
2383          } else {
2384             Operand width;
2385             if (const_bits) {
2386                width = Operand(const_bits->u32 << 16);
2387             } else {
2388                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2389             }
2390             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2391          }
2392
2393          aco_opcode opcode;
2394          if (dst.regClass() == s1) {
2395             if (instr->op == nir_op_ubfe)
2396                opcode = aco_opcode::s_bfe_u32;
2397             else
2398                opcode = aco_opcode::s_bfe_i32;
2399          } else if (dst.regClass() == s2) {
2400             if (instr->op == nir_op_ubfe)
2401                opcode = aco_opcode::s_bfe_u64;
2402             else
2403                opcode = aco_opcode::s_bfe_i64;
2404          } else {
2405             unreachable("Unsupported BFE bit size");
2406          }
2407
2408          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2409
2410       } else {
2411          aco_opcode opcode;
2412          if (dst.regClass() == v1) {
2413             if (instr->op == nir_op_ubfe)
2414                opcode = aco_opcode::v_bfe_u32;
2415             else
2416                opcode = aco_opcode::v_bfe_i32;
2417          } else {
2418             unreachable("Unsupported BFE bit size");
2419          }
2420
2421          emit_vop3a_instruction(ctx, instr, opcode, dst);
2422       }
2423       break;
2424    }
2425    case nir_op_bit_count: {
2426       Temp src = get_alu_src(ctx, instr->src[0]);
2427       if (src.regClass() == s1) {
2428          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2429       } else if (src.regClass() == v1) {
2430          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2431       } else if (src.regClass() == v2) {
2432          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2433                   emit_extract_vector(ctx, src, 1, v1),
2434                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2435                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2436       } else if (src.regClass() == s2) {
2437          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2438       } else {
2439          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2440          nir_print_instr(&instr->instr, stderr);
2441          fprintf(stderr, "\n");
2442       }
2443       break;
2444    }
2445    case nir_op_flt: {
2446       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2447       break;
2448    }
2449    case nir_op_fge: {
2450       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2451       break;
2452    }
2453    case nir_op_feq: {
2454       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2455       break;
2456    }
2457    case nir_op_fne: {
2458       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2459       break;
2460    }
2461    case nir_op_ilt: {
2462       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2463       break;
2464    }
2465    case nir_op_ige: {
2466       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2467       break;
2468    }
2469    case nir_op_ieq: {
2470       if (instr->src[0].src.ssa->bit_size == 1)
2471          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2472       else
2473          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2474                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2475       break;
2476    }
2477    case nir_op_ine: {
2478       if (instr->src[0].src.ssa->bit_size == 1)
2479          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2480       else
2481          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2482                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2483       break;
2484    }
2485    case nir_op_ult: {
2486       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2487       break;
2488    }
2489    case nir_op_uge: {
2490       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2491       break;
2492    }
2493    case nir_op_fddx:
2494    case nir_op_fddy:
2495    case nir_op_fddx_fine:
2496    case nir_op_fddy_fine:
2497    case nir_op_fddx_coarse:
2498    case nir_op_fddy_coarse: {
2499       Temp src = get_alu_src(ctx, instr->src[0]);
2500       uint16_t dpp_ctrl1, dpp_ctrl2;
2501       if (instr->op == nir_op_fddx_fine) {
2502          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2503          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2504       } else if (instr->op == nir_op_fddy_fine) {
2505          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2506          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2507       } else {
2508          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2509          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2510             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2511          else
2512             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2513       }
2514
2515       Temp tmp;
2516       if (ctx->program->chip_class >= GFX8) {
2517          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2518          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2519       } else {
2520          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2521          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2522          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2523       }
2524       emit_wqm(ctx, tmp, dst, true);
2525       break;
2526    }
2527    default:
2528       fprintf(stderr, "Unknown NIR ALU instr: ");
2529       nir_print_instr(&instr->instr, stderr);
2530       fprintf(stderr, "\n");
2531    }
2532 }
2533
2534 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2535 {
2536    Temp dst = get_ssa_temp(ctx, &instr->def);
2537
2538    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2539    // which get truncated the lsb if double and msb if int
2540    // for now, we only use s_mov_b64 with 64bit inline constants
2541    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2542    assert(dst.type() == RegType::sgpr);
2543
2544    Builder bld(ctx->program, ctx->block);
2545
2546    if (instr->def.bit_size == 1) {
2547       assert(dst.regClass() == bld.lm);
2548       int val = instr->value[0].b ? -1 : 0;
2549       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2550       bld.sop1(Builder::s_mov, Definition(dst), op);
2551    } else if (dst.size() == 1) {
2552       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2553    } else {
2554       assert(dst.size() != 1);
2555       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2556       if (instr->def.bit_size == 64)
2557          for (unsigned i = 0; i < dst.size(); i++)
2558             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2559       else {
2560          for (unsigned i = 0; i < dst.size(); i++)
2561             vec->operands[i] = Operand{instr->value[i].u32};
2562       }
2563       vec->definitions[0] = Definition(dst);
2564       ctx->block->instructions.emplace_back(std::move(vec));
2565    }
2566 }
2567
2568 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2569 {
2570    uint32_t new_mask = 0;
2571    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2572       if (mask & (1u << i))
2573          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2574    return new_mask;
2575 }
2576
2577 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2578 {
2579    /* This wouldn't work inside control flow or with indirect offsets but
2580     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2581
2582    unsigned write_mask = nir_intrinsic_write_mask(instr);
2583    unsigned component = nir_intrinsic_component(instr);
2584    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2585    unsigned idx = nir_intrinsic_base(instr) + component;
2586
2587    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2588    if (off_instr->type != nir_instr_type_load_const) {
2589       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2590       nir_print_instr(off_instr, stderr);
2591       fprintf(stderr, "\n");
2592    }
2593    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2594
2595    if (instr->src[0].ssa->bit_size == 64)
2596       write_mask = widen_mask(write_mask, 2);
2597
2598    for (unsigned i = 0; i < 8; ++i) {
2599       if (write_mask & (1 << i)) {
2600          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2601          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2602       }
2603       idx++;
2604    }
2605 }
2606
2607 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2608 {
2609    Builder bld(ctx->program, ctx->block);
2610    unsigned write_mask = nir_intrinsic_write_mask(instr);
2611    Operand values[4];
2612    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2613    for (unsigned i = 0; i < 4; ++i) {
2614       if (write_mask & (1 << i)) {
2615          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2616          values[i] = Operand(tmp);
2617       } else {
2618          values[i] = Operand(v1);
2619       }
2620    }
2621
2622    unsigned index = nir_intrinsic_base(instr) / 4;
2623    unsigned target, col_format;
2624    unsigned enabled_channels = 0xF;
2625    aco_opcode compr_op = (aco_opcode)0;
2626
2627    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2628    assert(offset && "Non-const offsets on exports not yet supported");
2629    index += offset->u32;
2630
2631    assert(index != FRAG_RESULT_COLOR);
2632
2633    /* Unlike vertex shader exports, it's fine to use multiple exports to
2634     * export separate channels of one target. So shaders which export both
2635     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2636     * TODO: combine the exports in those cases and create better code
2637     */
2638
2639    if (index == FRAG_RESULT_SAMPLE_MASK) {
2640
2641       if (ctx->program->info->ps.writes_z) {
2642          target = V_008DFC_SQ_EXP_MRTZ;
2643          enabled_channels = 0x4;
2644          col_format = (unsigned) -1;
2645
2646          values[2] = values[0];
2647          values[0] = Operand(v1);
2648       } else {
2649          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2650                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2651          return;
2652       }
2653
2654    } else if (index == FRAG_RESULT_DEPTH) {
2655
2656       target = V_008DFC_SQ_EXP_MRTZ;
2657       enabled_channels = 0x1;
2658       col_format = (unsigned) -1;
2659
2660    } else if (index == FRAG_RESULT_STENCIL) {
2661
2662       if (ctx->program->info->ps.writes_z) {
2663          target = V_008DFC_SQ_EXP_MRTZ;
2664          enabled_channels = 0x2;
2665          col_format = (unsigned) -1;
2666
2667          values[1] = values[0];
2668          values[0] = Operand(v1);
2669       } else {
2670          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2671          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2672                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2673          return;
2674       }
2675
2676    } else {
2677       index -= FRAG_RESULT_DATA0;
2678       target = V_008DFC_SQ_EXP_MRT + index;
2679       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2680    }
2681    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2682    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2683
2684    switch (col_format)
2685    {
2686    case V_028714_SPI_SHADER_ZERO:
2687       enabled_channels = 0; /* writemask */
2688       target = V_008DFC_SQ_EXP_NULL;
2689       break;
2690
2691    case V_028714_SPI_SHADER_32_R:
2692       enabled_channels = 1;
2693       break;
2694
2695    case V_028714_SPI_SHADER_32_GR:
2696       enabled_channels = 0x3;
2697       break;
2698
2699    case V_028714_SPI_SHADER_32_AR:
2700       if (ctx->options->chip_class >= GFX10) {
2701          /* Special case: on GFX10, the outputs are different for 32_AR */
2702          enabled_channels = 0x3;
2703          values[1] = values[3];
2704          values[3] = Operand(v1);
2705       } else {
2706          enabled_channels = 0x9;
2707       }
2708       break;
2709
2710    case V_028714_SPI_SHADER_FP16_ABGR:
2711       enabled_channels = 0x5;
2712       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2713       break;
2714
2715    case V_028714_SPI_SHADER_UNORM16_ABGR:
2716       enabled_channels = 0x5;
2717       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2718       break;
2719
2720    case V_028714_SPI_SHADER_SNORM16_ABGR:
2721       enabled_channels = 0x5;
2722       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2723       break;
2724
2725    case V_028714_SPI_SHADER_UINT16_ABGR: {
2726       enabled_channels = 0x5;
2727       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2728       if (is_int8 || is_int10) {
2729          /* clamp */
2730          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2731          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2732
2733          for (unsigned i = 0; i < 4; i++) {
2734             if ((write_mask >> i) & 1) {
2735                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2736                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2737                                     values[i]);
2738             }
2739          }
2740       }
2741       break;
2742    }
2743
2744    case V_028714_SPI_SHADER_SINT16_ABGR:
2745       enabled_channels = 0x5;
2746       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2747       if (is_int8 || is_int10) {
2748          /* clamp */
2749          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2750          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2751          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2752          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2753
2754          for (unsigned i = 0; i < 4; i++) {
2755             if ((write_mask >> i) & 1) {
2756                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2757                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2758                                     values[i]);
2759                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2760                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2761                                     values[i]);
2762             }
2763          }
2764       }
2765       break;
2766
2767    case V_028714_SPI_SHADER_32_ABGR:
2768       enabled_channels = 0xF;
2769       break;
2770
2771    default:
2772       break;
2773    }
2774
2775    if (target == V_008DFC_SQ_EXP_NULL)
2776       return;
2777
2778    if ((bool) compr_op) {
2779       for (int i = 0; i < 2; i++) {
2780          /* check if at least one of the values to be compressed is enabled */
2781          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2782          if (enabled) {
2783             enabled_channels |= enabled << (i*2);
2784             values[i] = bld.vop3(compr_op, bld.def(v1),
2785                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2786                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2787          } else {
2788             values[i] = Operand(v1);
2789          }
2790       }
2791       values[2] = Operand(v1);
2792       values[3] = Operand(v1);
2793    } else {
2794       for (int i = 0; i < 4; i++)
2795          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2796    }
2797
2798    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2799            enabled_channels, target, (bool) compr_op);
2800 }
2801
2802 Operand load_lds_size_m0(isel_context *ctx)
2803 {
2804    /* TODO: m0 does not need to be initialized on GFX9+ */
2805    Builder bld(ctx->program, ctx->block);
2806    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2807 }
2808
2809 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2810               Temp address, unsigned base_offset, unsigned align)
2811 {
2812    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2813
2814    Builder bld(ctx->program, ctx->block);
2815
2816    Operand m = load_lds_size_m0(ctx);
2817
2818    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2819    unsigned bytes_read = 0;
2820    unsigned result_size = 0;
2821    unsigned total_bytes = num_components * elem_size_bytes;
2822    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2823    bool large_ds_read = ctx->options->chip_class >= GFX7;
2824
2825    while (bytes_read < total_bytes) {
2826       unsigned todo = total_bytes - bytes_read;
2827       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2828       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2829
2830       aco_opcode op = aco_opcode::last_opcode;
2831       bool read2 = false;
2832       if (todo >= 16 && aligned16 && large_ds_read) {
2833          op = aco_opcode::ds_read_b128;
2834          todo = 16;
2835       } else if (todo >= 16 && aligned8) {
2836          op = aco_opcode::ds_read2_b64;
2837          read2 = true;
2838          todo = 16;
2839       } else if (todo >= 12 && aligned16 && large_ds_read) {
2840          op = aco_opcode::ds_read_b96;
2841          todo = 12;
2842       } else if (todo >= 8 && aligned8) {
2843          op = aco_opcode::ds_read_b64;
2844          todo = 8;
2845       } else if (todo >= 8) {
2846          op = aco_opcode::ds_read2_b32;
2847          read2 = true;
2848          todo = 8;
2849       } else if (todo >= 4) {
2850          op = aco_opcode::ds_read_b32;
2851          todo = 4;
2852       } else {
2853          assert(false);
2854       }
2855       assert(todo % elem_size_bytes == 0);
2856       unsigned num_elements = todo / elem_size_bytes;
2857       unsigned offset = base_offset + bytes_read;
2858       unsigned max_offset = read2 ? 1019 : 65535;
2859
2860       Temp address_offset = address;
2861       if (offset > max_offset) {
2862          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2863          offset = bytes_read;
2864       }
2865       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2866
2867       Temp res;
2868       if (num_components == 1 && dst.type() == RegType::vgpr)
2869          res = dst;
2870       else
2871          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2872
2873       if (read2)
2874          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2875       else
2876          res = bld.ds(op, Definition(res), address_offset, m, offset);
2877
2878       if (num_components == 1) {
2879          assert(todo == total_bytes);
2880          if (dst.type() == RegType::sgpr)
2881             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2882          return;
2883       }
2884
2885       if (dst.type() == RegType::sgpr) {
2886          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2887          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2888          res = new_res;
2889       }
2890
2891       if (num_elements == 1) {
2892          result[result_size++] = res;
2893       } else {
2894          assert(res != dst && res.size() % num_elements == 0);
2895          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2896          split->operands[0] = Operand(res);
2897          for (unsigned i = 0; i < num_elements; i++)
2898             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2899          ctx->block->instructions.emplace_back(std::move(split));
2900       }
2901
2902       bytes_read += todo;
2903    }
2904
2905    assert(result_size == num_components && result_size > 1);
2906    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2907    for (unsigned i = 0; i < result_size; i++)
2908       vec->operands[i] = Operand(result[i]);
2909    vec->definitions[0] = Definition(dst);
2910    ctx->block->instructions.emplace_back(std::move(vec));
2911    ctx->allocated_vec.emplace(dst.id(), result);
2912 }
2913
2914 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2915 {
2916    if (start == 0 && size == data.size())
2917       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2918
2919    unsigned size_hint = 1;
2920    auto it = ctx->allocated_vec.find(data.id());
2921    if (it != ctx->allocated_vec.end())
2922       size_hint = it->second[0].size();
2923    if (size % size_hint || start % size_hint)
2924       size_hint = 1;
2925
2926    start /= size_hint;
2927    size /= size_hint;
2928
2929    Temp elems[size];
2930    for (unsigned i = 0; i < size; i++)
2931       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2932
2933    if (size == 1)
2934       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2935
2936    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2937    for (unsigned i = 0; i < size; i++)
2938       vec->operands[i] = Operand(elems[i]);
2939    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2940    vec->definitions[0] = Definition(res);
2941    ctx->block->instructions.emplace_back(std::move(vec));
2942    return res;
2943 }
2944
2945 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2946 {
2947    Builder bld(ctx->program, ctx->block);
2948    unsigned bytes_written = 0;
2949    bool large_ds_write = ctx->options->chip_class >= GFX7;
2950
2951    while (bytes_written < total_size * 4) {
2952       unsigned todo = total_size * 4 - bytes_written;
2953       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2954       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2955
2956       aco_opcode op = aco_opcode::last_opcode;
2957       bool write2 = false;
2958       unsigned size = 0;
2959       if (todo >= 16 && aligned16 && large_ds_write) {
2960          op = aco_opcode::ds_write_b128;
2961          size = 4;
2962       } else if (todo >= 16 && aligned8) {
2963          op = aco_opcode::ds_write2_b64;
2964          write2 = true;
2965          size = 4;
2966       } else if (todo >= 12 && aligned16 && large_ds_write) {
2967          op = aco_opcode::ds_write_b96;
2968          size = 3;
2969       } else if (todo >= 8 && aligned8) {
2970          op = aco_opcode::ds_write_b64;
2971          size = 2;
2972       } else if (todo >= 8) {
2973          op = aco_opcode::ds_write2_b32;
2974          write2 = true;
2975          size = 2;
2976       } else if (todo >= 4) {
2977          op = aco_opcode::ds_write_b32;
2978          size = 1;
2979       } else {
2980          assert(false);
2981       }
2982
2983       unsigned offset = offset0 + offset1 + bytes_written;
2984       unsigned max_offset = write2 ? 1020 : 65535;
2985       Temp address_offset = address;
2986       if (offset > max_offset) {
2987          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2988          offset = offset1 + bytes_written;
2989       }
2990       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2991
2992       if (write2) {
2993          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2994          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2995          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2996       } else {
2997          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2998          bld.ds(op, address_offset, val, m, offset);
2999       }
3000
3001       bytes_written += size * 4;
3002    }
3003 }
3004
3005 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
3006                Temp address, unsigned base_offset, unsigned align)
3007 {
3008    assert(util_is_power_of_two_nonzero(align) && align >= 4);
3009
3010    Operand m = load_lds_size_m0(ctx);
3011
3012    /* we need at most two stores for 32bit variables */
3013    int start[2], count[2];
3014    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
3015    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
3016    assert(wrmask == 0);
3017
3018    /* one combined store is sufficient */
3019    if (count[0] == count[1]) {
3020       Builder bld(ctx->program, ctx->block);
3021
3022       Temp address_offset = address;
3023       if ((base_offset >> 2) + start[1] > 255) {
3024          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
3025          base_offset = 0;
3026       }
3027
3028       assert(count[0] == 1);
3029       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
3030       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
3031       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
3032       base_offset = base_offset / elem_size_bytes;
3033       bld.ds(op, address_offset, val0, val1, m,
3034              base_offset + start[0], base_offset + start[1]);
3035       return;
3036    }
3037
3038    for (unsigned i = 0; i < 2; i++) {
3039       if (count[i] == 0)
3040          continue;
3041
3042       unsigned elem_size_words = elem_size_bytes / 4;
3043       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
3044                       base_offset, start[i] * elem_size_bytes, align);
3045    }
3046    return;
3047 }
3048
3049 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
3050 {
3051    if (ctx->stage == vertex_vs) {
3052       visit_store_vs_output(ctx, instr);
3053    } else if (ctx->stage == fragment_fs) {
3054       visit_store_fs_output(ctx, instr);
3055    } else {
3056       unreachable("Shader stage not implemented");
3057    }
3058 }
3059
3060 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3061 {
3062    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3063    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3064
3065    Builder bld(ctx->program, ctx->block);
3066    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3067    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3068 }
3069
3070 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3071 {
3072    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3073    for (unsigned i = 0; i < num_components; i++)
3074       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3075    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3076       assert(num_components == 4);
3077       Builder bld(ctx->program, ctx->block);
3078       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3079    }
3080
3081    for (Operand& op : vec->operands)
3082       op = op.isUndefined() ? Operand(0u) : op;
3083
3084    vec->definitions[0] = Definition(dst);
3085    ctx->block->instructions.emplace_back(std::move(vec));
3086    emit_split_vector(ctx, dst, num_components);
3087    return;
3088 }
3089
3090 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3091 {
3092    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3093    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3094    unsigned idx = nir_intrinsic_base(instr);
3095    unsigned component = nir_intrinsic_component(instr);
3096    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3097
3098    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3099    if (offset) {
3100       assert(offset->u32 == 0);
3101    } else {
3102       /* the lower 15bit of the prim_mask contain the offset into LDS
3103        * while the upper bits contain the number of prims */
3104       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3105       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3106       Builder bld(ctx->program, ctx->block);
3107       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3108       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3109       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3110       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3111       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3112    }
3113
3114    if (instr->dest.ssa.num_components == 1) {
3115       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3116    } else {
3117       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3118       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3119       {
3120          Temp tmp = {ctx->program->allocateId(), v1};
3121          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3122          vec->operands[i] = Operand(tmp);
3123       }
3124       vec->definitions[0] = Definition(dst);
3125       ctx->block->instructions.emplace_back(std::move(vec));
3126    }
3127 }
3128
3129 unsigned get_num_channels_from_data_format(unsigned data_format)
3130 {
3131    switch (data_format) {
3132    case V_008F0C_BUF_DATA_FORMAT_8:
3133    case V_008F0C_BUF_DATA_FORMAT_16:
3134    case V_008F0C_BUF_DATA_FORMAT_32:
3135       return 1;
3136    case V_008F0C_BUF_DATA_FORMAT_8_8:
3137    case V_008F0C_BUF_DATA_FORMAT_16_16:
3138    case V_008F0C_BUF_DATA_FORMAT_32_32:
3139       return 2;
3140    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3141    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3142    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3143       return 3;
3144    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3145    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3146    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3147    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3148    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3149       return 4;
3150    default:
3151       break;
3152    }
3153
3154    return 4;
3155 }
3156
3157 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3158  * so we may need to fix it up. */
3159 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3160 {
3161    Builder bld(ctx->program, ctx->block);
3162
3163    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3164       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3165
3166    /* For the integer-like cases, do a natural sign extension.
3167     *
3168     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3169     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3170     * exponent.
3171     */
3172    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3173    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3174
3175    /* Convert back to the right type. */
3176    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3177       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3178       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3179       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3180    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3181       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3182    }
3183
3184    return alpha;
3185 }
3186
3187 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3188 {
3189    Builder bld(ctx->program, ctx->block);
3190    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3191    if (ctx->stage & sw_vs) {
3192
3193       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3194       if (off_instr->type != nir_instr_type_load_const) {
3195          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3196          nir_print_instr(off_instr, stderr);
3197          fprintf(stderr, "\n");
3198       }
3199       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3200
3201       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3202
3203       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3204       unsigned component = nir_intrinsic_component(instr);
3205       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3206       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3207       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3208       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3209
3210       unsigned dfmt = attrib_format & 0xf;
3211
3212       unsigned nfmt = (attrib_format >> 4) & 0x7;
3213       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3214       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3215       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3216       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3217       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3218       if (post_shuffle)
3219          num_channels = MAX2(num_channels, 3);
3220
3221       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3222
3223       Temp index;
3224       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3225          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3226          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3227          if (divisor) {
3228             ctx->needs_instance_id = true;
3229             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3230             if (divisor != 1) {
3231                Temp divided = bld.tmp(v1);
3232                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3233                index = bld.vadd32(bld.def(v1), start_instance, divided);
3234             } else {
3235                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3236             }
3237          } else {
3238             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3239          }
3240       } else {
3241          index = bld.vadd32(bld.def(v1),
3242                             get_arg(ctx, ctx->args->ac.base_vertex),
3243                             get_arg(ctx, ctx->args->ac.vertex_id));
3244       }
3245
3246       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3247          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3248          attrib_offset = attrib_offset % attrib_stride;
3249       }
3250
3251       Operand soffset(0u);
3252       if (attrib_offset >= 4096) {
3253          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3254          attrib_offset = 0;
3255       }
3256
3257       aco_opcode opcode;
3258       switch (num_channels) {
3259       case 1:
3260          opcode = aco_opcode::tbuffer_load_format_x;
3261          break;
3262       case 2:
3263          opcode = aco_opcode::tbuffer_load_format_xy;
3264          break;
3265       case 3:
3266          opcode = aco_opcode::tbuffer_load_format_xyz;
3267          break;
3268       case 4:
3269          opcode = aco_opcode::tbuffer_load_format_xyzw;
3270          break;
3271       default:
3272          unreachable("Unimplemented load_input vector size");
3273       }
3274
3275       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3276
3277       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3278       mubuf->operands[0] = Operand(index);
3279       mubuf->operands[1] = Operand(list);
3280       mubuf->operands[2] = soffset;
3281       mubuf->definitions[0] = Definition(tmp);
3282       mubuf->idxen = true;
3283       mubuf->can_reorder = true;
3284       mubuf->dfmt = dfmt;
3285       mubuf->nfmt = nfmt;
3286       assert(attrib_offset < 4096);
3287       mubuf->offset = attrib_offset;
3288       ctx->block->instructions.emplace_back(std::move(mubuf));
3289
3290       emit_split_vector(ctx, tmp, tmp.size());
3291
3292       if (tmp.id() != dst.id()) {
3293          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3294                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3295
3296          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3297          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3298          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3299
3300          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3301          for (unsigned i = 0; i < dst.size(); i++) {
3302             unsigned idx = i + component;
3303             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3304                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3305                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3306             } else if (idx < num_channels) {
3307                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3308             } else if (is_float && idx == 3) {
3309                vec->operands[i] = Operand(0x3f800000u);
3310             } else if (!is_float && idx == 3) {
3311                vec->operands[i] = Operand(1u);
3312             } else {
3313                vec->operands[i] = Operand(0u);
3314             }
3315          }
3316          vec->definitions[0] = Definition(dst);
3317          ctx->block->instructions.emplace_back(std::move(vec));
3318          emit_split_vector(ctx, dst, dst.size());
3319       }
3320
3321    } else if (ctx->stage == fragment_fs) {
3322       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3323       if (off_instr->type != nir_instr_type_load_const ||
3324           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3325          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3326          nir_print_instr(off_instr, stderr);
3327          fprintf(stderr, "\n");
3328       }
3329
3330       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3331       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3332       if (offset) {
3333          assert(offset->u32 == 0);
3334       } else {
3335          /* the lower 15bit of the prim_mask contain the offset into LDS
3336           * while the upper bits contain the number of prims */
3337          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3338          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3339          Builder bld(ctx->program, ctx->block);
3340          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3341          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3342          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3343          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3344          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3345       }
3346
3347       unsigned idx = nir_intrinsic_base(instr);
3348       unsigned component = nir_intrinsic_component(instr);
3349
3350       if (dst.size() == 1) {
3351          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3352       } else {
3353          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3354          for (unsigned i = 0; i < dst.size(); i++)
3355             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3356          vec->definitions[0] = Definition(dst);
3357          bld.insert(std::move(vec));
3358       }
3359
3360    } else {
3361       unreachable("Shader stage not implemented");
3362    }
3363 }
3364
3365 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3366 {
3367    if (ctx->program->info->need_indirect_descriptor_sets) {
3368       Builder bld(ctx->program, ctx->block);
3369       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3370       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3371    }
3372
3373    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3374 }
3375
3376
3377 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3378 {
3379    Builder bld(ctx->program, ctx->block);
3380    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3381    if (!ctx->divergent_vals[instr->dest.ssa.index])
3382       index = bld.as_uniform(index);
3383    unsigned desc_set = nir_intrinsic_desc_set(instr);
3384    unsigned binding = nir_intrinsic_binding(instr);
3385
3386    Temp desc_ptr;
3387    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3388    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3389    unsigned offset = layout->binding[binding].offset;
3390    unsigned stride;
3391    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3392        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3393       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3394       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3395       offset = pipeline_layout->push_constant_size + 16 * idx;
3396       stride = 16;
3397    } else {
3398       desc_ptr = load_desc_ptr(ctx, desc_set);
3399       stride = layout->binding[binding].size;
3400    }
3401
3402    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3403    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3404    if (stride != 1) {
3405       if (nir_const_index) {
3406          const_index = const_index * stride;
3407       } else if (index.type() == RegType::vgpr) {
3408          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3409          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3410       } else {
3411          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3412       }
3413    }
3414    if (offset) {
3415       if (nir_const_index) {
3416          const_index = const_index + offset;
3417       } else if (index.type() == RegType::vgpr) {
3418          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3419       } else {
3420          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3421       }
3422    }
3423
3424    if (nir_const_index && const_index == 0) {
3425       index = desc_ptr;
3426    } else if (index.type() == RegType::vgpr) {
3427       index = bld.vadd32(bld.def(v1),
3428                          nir_const_index ? Operand(const_index) : Operand(index),
3429                          Operand(desc_ptr));
3430    } else {
3431       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3432                        nir_const_index ? Operand(const_index) : Operand(index),
3433                        Operand(desc_ptr));
3434    }
3435
3436    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3437 }
3438
3439 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3440                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3441 {
3442    Builder bld(ctx->program, ctx->block);
3443
3444    unsigned num_bytes = dst.size() * 4;
3445    bool dlc = glc && ctx->options->chip_class >= GFX10;
3446
3447    aco_opcode op;
3448    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3449       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3450       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3451       unsigned const_offset = 0;
3452
3453       Temp lower = Temp();
3454       if (num_bytes > 16) {
3455          assert(num_components == 3 || num_components == 4);
3456          op = aco_opcode::buffer_load_dwordx4;
3457          lower = bld.tmp(v4);
3458          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3459          mubuf->definitions[0] = Definition(lower);
3460          mubuf->operands[0] = vaddr;
3461          mubuf->operands[1] = Operand(rsrc);
3462          mubuf->operands[2] = soffset;
3463          mubuf->offen = (offset.type() == RegType::vgpr);
3464          mubuf->glc = glc;
3465          mubuf->dlc = dlc;
3466          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3467          mubuf->can_reorder = readonly;
3468          bld.insert(std::move(mubuf));
3469          emit_split_vector(ctx, lower, 2);
3470          num_bytes -= 16;
3471          const_offset = 16;
3472       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3473          /* GFX6 doesn't support loading vec3, expand to vec4. */
3474          num_bytes = 16;
3475       }
3476
3477       switch (num_bytes) {
3478          case 4:
3479             op = aco_opcode::buffer_load_dword;
3480             break;
3481          case 8:
3482             op = aco_opcode::buffer_load_dwordx2;
3483             break;
3484          case 12:
3485             assert(ctx->options->chip_class > GFX6);
3486             op = aco_opcode::buffer_load_dwordx3;
3487             break;
3488          case 16:
3489             op = aco_opcode::buffer_load_dwordx4;
3490             break;
3491          default:
3492             unreachable("Load SSBO not implemented for this size.");
3493       }
3494       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3495       mubuf->operands[0] = vaddr;
3496       mubuf->operands[1] = Operand(rsrc);
3497       mubuf->operands[2] = soffset;
3498       mubuf->offen = (offset.type() == RegType::vgpr);
3499       mubuf->glc = glc;
3500       mubuf->dlc = dlc;
3501       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3502       mubuf->can_reorder = readonly;
3503       mubuf->offset = const_offset;
3504       aco_ptr<Instruction> instr = std::move(mubuf);
3505
3506       if (dst.size() > 4) {
3507          assert(lower != Temp());
3508          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3509          instr->definitions[0] = Definition(upper);
3510          bld.insert(std::move(instr));
3511          if (dst.size() == 8)
3512             emit_split_vector(ctx, upper, 2);
3513          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3514          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3515          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3516          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3517          if (dst.size() == 8)
3518             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3519       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3520          Temp vec = bld.tmp(v4);
3521          instr->definitions[0] = Definition(vec);
3522          bld.insert(std::move(instr));
3523          emit_split_vector(ctx, vec, 4);
3524
3525          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3526          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3527          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3528          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3529       }
3530
3531       if (dst.type() == RegType::sgpr) {
3532          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3533          instr->definitions[0] = Definition(vec);
3534          bld.insert(std::move(instr));
3535          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3536       } else {
3537          instr->definitions[0] = Definition(dst);
3538          bld.insert(std::move(instr));
3539          emit_split_vector(ctx, dst, num_components);
3540       }
3541    } else {
3542       switch (num_bytes) {
3543          case 4:
3544             op = aco_opcode::s_buffer_load_dword;
3545             break;
3546          case 8:
3547             op = aco_opcode::s_buffer_load_dwordx2;
3548             break;
3549          case 12:
3550          case 16:
3551             op = aco_opcode::s_buffer_load_dwordx4;
3552             break;
3553          case 24:
3554          case 32:
3555             op = aco_opcode::s_buffer_load_dwordx8;
3556             break;
3557          default:
3558             unreachable("Load SSBO not implemented for this size.");
3559       }
3560       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3561       load->operands[0] = Operand(rsrc);
3562       load->operands[1] = Operand(bld.as_uniform(offset));
3563       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3564       load->definitions[0] = Definition(dst);
3565       load->glc = glc;
3566       load->dlc = dlc;
3567       load->barrier = readonly ? barrier_none : barrier_buffer;
3568       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3569       assert(ctx->options->chip_class >= GFX8 || !glc);
3570
3571       /* trim vector */
3572       if (dst.size() == 3) {
3573          Temp vec = bld.tmp(s4);
3574          load->definitions[0] = Definition(vec);
3575          bld.insert(std::move(load));
3576          emit_split_vector(ctx, vec, 4);
3577
3578          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3579                     emit_extract_vector(ctx, vec, 0, s1),
3580                     emit_extract_vector(ctx, vec, 1, s1),
3581                     emit_extract_vector(ctx, vec, 2, s1));
3582       } else if (dst.size() == 6) {
3583          Temp vec = bld.tmp(s8);
3584          load->definitions[0] = Definition(vec);
3585          bld.insert(std::move(load));
3586          emit_split_vector(ctx, vec, 4);
3587
3588          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3589                     emit_extract_vector(ctx, vec, 0, s2),
3590                     emit_extract_vector(ctx, vec, 1, s2),
3591                     emit_extract_vector(ctx, vec, 2, s2));
3592       } else {
3593          bld.insert(std::move(load));
3594       }
3595       emit_split_vector(ctx, dst, num_components);
3596    }
3597 }
3598
3599 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3600 {
3601    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3602    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3603
3604    Builder bld(ctx->program, ctx->block);
3605
3606    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3607    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3608    unsigned binding = nir_intrinsic_binding(idx_instr);
3609    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3610
3611    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3612       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3613                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3614                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3615                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3616       if (ctx->options->chip_class >= GFX10) {
3617          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3618                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3619                       S_008F0C_RESOURCE_LEVEL(1);
3620       } else {
3621          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3622                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3623       }
3624       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3625                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3626                                      Operand(0xFFFFFFFFu),
3627                                      Operand(desc_type));
3628       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3629                         rsrc, upper_dwords);
3630    } else {
3631       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3632       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3633    }
3634
3635    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3636 }
3637
3638 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3639 {
3640    Builder bld(ctx->program, ctx->block);
3641    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3642
3643    unsigned offset = nir_intrinsic_base(instr);
3644    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3645    if (index_cv && instr->dest.ssa.bit_size == 32) {
3646
3647       unsigned count = instr->dest.ssa.num_components;
3648       unsigned start = (offset + index_cv->u32) / 4u;
3649       start -= ctx->args->ac.base_inline_push_consts;
3650       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3651          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3652          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3653          for (unsigned i = 0; i < count; ++i) {
3654             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3655             vec->operands[i] = Operand{elems[i]};
3656          }
3657          vec->definitions[0] = Definition(dst);
3658          ctx->block->instructions.emplace_back(std::move(vec));
3659          ctx->allocated_vec.emplace(dst.id(), elems);
3660          return;
3661       }
3662    }
3663
3664    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3665    if (offset != 0) // TODO check if index != 0 as well
3666       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3667    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3668    Temp vec = dst;
3669    bool trim = false;
3670    aco_opcode op;
3671
3672    switch (dst.size()) {
3673    case 1:
3674       op = aco_opcode::s_load_dword;
3675       break;
3676    case 2:
3677       op = aco_opcode::s_load_dwordx2;
3678       break;
3679    case 3:
3680       vec = bld.tmp(s4);
3681       trim = true;
3682    case 4:
3683       op = aco_opcode::s_load_dwordx4;
3684       break;
3685    case 6:
3686       vec = bld.tmp(s8);
3687       trim = true;
3688    case 8:
3689       op = aco_opcode::s_load_dwordx8;
3690       break;
3691    default:
3692       unreachable("unimplemented or forbidden load_push_constant.");
3693    }
3694
3695    bld.smem(op, Definition(vec), ptr, index);
3696
3697    if (trim) {
3698       emit_split_vector(ctx, vec, 4);
3699       RegClass rc = dst.size() == 3 ? s1 : s2;
3700       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3701                  emit_extract_vector(ctx, vec, 0, rc),
3702                  emit_extract_vector(ctx, vec, 1, rc),
3703                  emit_extract_vector(ctx, vec, 2, rc));
3704
3705    }
3706    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3707 }
3708
3709 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3710 {
3711    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3712
3713    Builder bld(ctx->program, ctx->block);
3714
3715    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3716                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3717                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3718                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3719    if (ctx->options->chip_class >= GFX10) {
3720       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3721                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3722                    S_008F0C_RESOURCE_LEVEL(1);
3723    } else {
3724       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3725                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3726    }
3727
3728    unsigned base = nir_intrinsic_base(instr);
3729    unsigned range = nir_intrinsic_range(instr);
3730
3731    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3732    if (base && offset.type() == RegType::sgpr)
3733       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3734    else if (base && offset.type() == RegType::vgpr)
3735       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3736
3737    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3738                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3739                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3740                           Operand(desc_type));
3741
3742    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3743 }
3744
3745 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3746 {
3747    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3748       ctx->cf_info.exec_potentially_empty = true;
3749
3750    ctx->program->needs_exact = true;
3751
3752    // TODO: optimize uniform conditions
3753    Builder bld(ctx->program, ctx->block);
3754    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3755    assert(src.regClass() == bld.lm);
3756    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3757    bld.pseudo(aco_opcode::p_discard_if, src);
3758    ctx->block->kind |= block_kind_uses_discard_if;
3759    return;
3760 }
3761
3762 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3763 {
3764    Builder bld(ctx->program, ctx->block);
3765
3766    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3767       ctx->cf_info.exec_potentially_empty = true;
3768
3769    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3770                     ctx->cf_info.parent_loop.has_divergent_continue;
3771
3772    if (ctx->block->loop_nest_depth &&
3773        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3774       /* we handle discards the same way as jump instructions */
3775       append_logical_end(ctx->block);
3776
3777       /* in loops, discard behaves like break */
3778       Block *linear_target = ctx->cf_info.parent_loop.exit;
3779       ctx->block->kind |= block_kind_discard;
3780
3781       if (!divergent) {
3782          /* uniform discard - loop ends here */
3783          assert(nir_instr_is_last(&instr->instr));
3784          ctx->block->kind |= block_kind_uniform;
3785          ctx->cf_info.has_branch = true;
3786          bld.branch(aco_opcode::p_branch);
3787          add_linear_edge(ctx->block->index, linear_target);
3788          return;
3789       }
3790
3791       /* we add a break right behind the discard() instructions */
3792       ctx->block->kind |= block_kind_break;
3793       unsigned idx = ctx->block->index;
3794
3795       /* remove critical edges from linear CFG */
3796       bld.branch(aco_opcode::p_branch);
3797       Block* break_block = ctx->program->create_and_insert_block();
3798       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3799       break_block->kind |= block_kind_uniform;
3800       add_linear_edge(idx, break_block);
3801       add_linear_edge(break_block->index, linear_target);
3802       bld.reset(break_block);
3803       bld.branch(aco_opcode::p_branch);
3804
3805       Block* continue_block = ctx->program->create_and_insert_block();
3806       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3807       add_linear_edge(idx, continue_block);
3808       append_logical_start(continue_block);
3809       ctx->block = continue_block;
3810
3811       return;
3812    }
3813
3814    /* it can currently happen that NIR doesn't remove the unreachable code */
3815    if (!nir_instr_is_last(&instr->instr)) {
3816       ctx->program->needs_exact = true;
3817       /* save exec somewhere temporarily so that it doesn't get
3818        * overwritten before the discard from outer exec masks */
3819       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3820       bld.pseudo(aco_opcode::p_discard_if, cond);
3821       ctx->block->kind |= block_kind_uses_discard_if;
3822       return;
3823    }
3824
3825    /* This condition is incorrect for uniformly branched discards in a loop
3826     * predicated by a divergent condition, but the above code catches that case
3827     * and the discard would end up turning into a discard_if.
3828     * For example:
3829     * if (divergent) {
3830     *    while (...) {
3831     *       if (uniform) {
3832     *          discard;
3833     *       }
3834     *    }
3835     * }
3836     */
3837    if (!ctx->cf_info.parent_if.is_divergent) {
3838       /* program just ends here */
3839       ctx->block->kind |= block_kind_uniform;
3840       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3841               0 /* enabled mask */, 9 /* dest */,
3842               false /* compressed */, true/* done */, true /* valid mask */);
3843       bld.sopp(aco_opcode::s_endpgm);
3844       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3845    } else {
3846       ctx->block->kind |= block_kind_discard;
3847       /* branch and linear edge is added by visit_if() */
3848    }
3849 }
3850
3851 enum aco_descriptor_type {
3852    ACO_DESC_IMAGE,
3853    ACO_DESC_FMASK,
3854    ACO_DESC_SAMPLER,
3855    ACO_DESC_BUFFER,
3856    ACO_DESC_PLANE_0,
3857    ACO_DESC_PLANE_1,
3858    ACO_DESC_PLANE_2,
3859 };
3860
3861 static bool
3862 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3863    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3864       return false;
3865    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3866    return dim == ac_image_cube ||
3867           dim == ac_image_1darray ||
3868           dim == ac_image_2darray ||
3869           dim == ac_image_2darraymsaa;
3870 }
3871
3872 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3873                       enum aco_descriptor_type desc_type,
3874                       const nir_tex_instr *tex_instr, bool image, bool write)
3875 {
3876 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3877    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3878    if (it != ctx->tex_desc.end())
3879       return it->second;
3880 */
3881    Temp index = Temp();
3882    bool index_set = false;
3883    unsigned constant_index = 0;
3884    unsigned descriptor_set;
3885    unsigned base_index;
3886    Builder bld(ctx->program, ctx->block);
3887
3888    if (!deref_instr) {
3889       assert(tex_instr && !image);
3890       descriptor_set = 0;
3891       base_index = tex_instr->sampler_index;
3892    } else {
3893       while(deref_instr->deref_type != nir_deref_type_var) {
3894          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3895          if (!array_size)
3896             array_size = 1;
3897
3898          assert(deref_instr->deref_type == nir_deref_type_array);
3899          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3900          if (const_value) {
3901             constant_index += array_size * const_value->u32;
3902          } else {
3903             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3904             if (indirect.type() == RegType::vgpr)
3905                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3906
3907             if (array_size != 1)
3908                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3909
3910             if (!index_set) {
3911                index = indirect;
3912                index_set = true;
3913             } else {
3914                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3915             }
3916          }
3917
3918          deref_instr = nir_src_as_deref(deref_instr->parent);
3919       }
3920       descriptor_set = deref_instr->var->data.descriptor_set;
3921       base_index = deref_instr->var->data.binding;
3922    }
3923
3924    Temp list = load_desc_ptr(ctx, descriptor_set);
3925    list = convert_pointer_to_64_bit(ctx, list);
3926
3927    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3928    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3929    unsigned offset = binding->offset;
3930    unsigned stride = binding->size;
3931    aco_opcode opcode;
3932    RegClass type;
3933
3934    assert(base_index < layout->binding_count);
3935
3936    switch (desc_type) {
3937    case ACO_DESC_IMAGE:
3938       type = s8;
3939       opcode = aco_opcode::s_load_dwordx8;
3940       break;
3941    case ACO_DESC_FMASK:
3942       type = s8;
3943       opcode = aco_opcode::s_load_dwordx8;
3944       offset += 32;
3945       break;
3946    case ACO_DESC_SAMPLER:
3947       type = s4;
3948       opcode = aco_opcode::s_load_dwordx4;
3949       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3950          offset += radv_combined_image_descriptor_sampler_offset(binding);
3951       break;
3952    case ACO_DESC_BUFFER:
3953       type = s4;
3954       opcode = aco_opcode::s_load_dwordx4;
3955       break;
3956    case ACO_DESC_PLANE_0:
3957    case ACO_DESC_PLANE_1:
3958       type = s8;
3959       opcode = aco_opcode::s_load_dwordx8;
3960       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3961       break;
3962    case ACO_DESC_PLANE_2:
3963       type = s4;
3964       opcode = aco_opcode::s_load_dwordx4;
3965       offset += 64;
3966       break;
3967    default:
3968       unreachable("invalid desc_type\n");
3969    }
3970
3971    offset += constant_index * stride;
3972
3973    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3974       (!index_set || binding->immutable_samplers_equal)) {
3975       if (binding->immutable_samplers_equal)
3976          constant_index = 0;
3977
3978       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3979       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3980                         Operand(samplers[constant_index * 4 + 0]),
3981                         Operand(samplers[constant_index * 4 + 1]),
3982                         Operand(samplers[constant_index * 4 + 2]),
3983                         Operand(samplers[constant_index * 4 + 3]));
3984    }
3985
3986    Operand off;
3987    if (!index_set) {
3988       off = Operand(offset);
3989    } else {
3990       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3991                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3992    }
3993
3994    Temp res = bld.smem(opcode, bld.def(type), list, off);
3995
3996    if (desc_type == ACO_DESC_PLANE_2) {
3997       Temp components[8];
3998       for (unsigned i = 0; i < 8; i++)
3999          components[i] = bld.tmp(s1);
4000       bld.pseudo(aco_opcode::p_split_vector,
4001                  Definition(components[0]),
4002                  Definition(components[1]),
4003                  Definition(components[2]),
4004                  Definition(components[3]),
4005                  res);
4006
4007       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4008       bld.pseudo(aco_opcode::p_split_vector,
4009                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4010                  Definition(components[4]),
4011                  Definition(components[5]),
4012                  Definition(components[6]),
4013                  Definition(components[7]),
4014                  desc2);
4015
4016       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4017                        components[0], components[1], components[2], components[3],
4018                        components[4], components[5], components[6], components[7]);
4019    }
4020
4021    return res;
4022 }
4023
4024 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4025 {
4026    switch (dim) {
4027    case GLSL_SAMPLER_DIM_BUF:
4028       return 1;
4029    case GLSL_SAMPLER_DIM_1D:
4030       return array ? 2 : 1;
4031    case GLSL_SAMPLER_DIM_2D:
4032       return array ? 3 : 2;
4033    case GLSL_SAMPLER_DIM_MS:
4034       return array ? 4 : 3;
4035    case GLSL_SAMPLER_DIM_3D:
4036    case GLSL_SAMPLER_DIM_CUBE:
4037       return 3;
4038    case GLSL_SAMPLER_DIM_RECT:
4039    case GLSL_SAMPLER_DIM_SUBPASS:
4040       return 2;
4041    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4042       return 3;
4043    default:
4044       break;
4045    }
4046    return 0;
4047 }
4048
4049
4050 /* Adjust the sample index according to FMASK.
4051  *
4052  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4053  * which is the identity mapping. Each nibble says which physical sample
4054  * should be fetched to get that sample.
4055  *
4056  * For example, 0x11111100 means there are only 2 samples stored and
4057  * the second sample covers 3/4 of the pixel. When reading samples 0
4058  * and 1, return physical sample 0 (determined by the first two 0s
4059  * in FMASK), otherwise return physical sample 1.
4060  *
4061  * The sample index should be adjusted as follows:
4062  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4063  */
4064 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4065 {
4066    Builder bld(ctx->program, ctx->block);
4067    Temp fmask = bld.tmp(v1);
4068    unsigned dim = ctx->options->chip_class >= GFX10
4069                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4070                   : 0;
4071
4072    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4073    load->operands[0] = Operand(coords);
4074    load->operands[1] = Operand(fmask_desc_ptr);
4075    load->definitions[0] = Definition(fmask);
4076    load->glc = false;
4077    load->dlc = false;
4078    load->dmask = 0x1;
4079    load->unrm = true;
4080    load->da = da;
4081    load->dim = dim;
4082    load->can_reorder = true; /* fmask images shouldn't be modified */
4083    ctx->block->instructions.emplace_back(std::move(load));
4084
4085    Operand sample_index4;
4086    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4087       sample_index4 = Operand(sample_index.constantValue() << 2);
4088    } else if (sample_index.regClass() == s1) {
4089       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4090    } else {
4091       assert(sample_index.regClass() == v1);
4092       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4093    }
4094
4095    Temp final_sample;
4096    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4097       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4098    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4099       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4100    else
4101       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4102
4103    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4104     * resource descriptor is 0 (invalid),
4105     */
4106    Temp compare = bld.tmp(bld.lm);
4107    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4108                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4109
4110    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4111
4112    /* Replace the MSAA sample index. */
4113    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4114 }
4115
4116 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4117 {
4118
4119    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4120    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4121    bool is_array = glsl_sampler_type_is_array(type);
4122    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4123    assert(!add_frag_pos && "Input attachments should be lowered.");
4124    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4125    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4126    int count = image_type_to_components_count(dim, is_array);
4127    std::vector<Operand> coords(count);
4128
4129    if (is_ms) {
4130       Operand sample_index;
4131       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4132       if (sample_cv)
4133          sample_index = Operand(sample_cv->u32);
4134       else
4135          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4136
4137       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4138          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4139          for (unsigned i = 0; i < vec->operands.size(); i++)
4140             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4141          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4142          vec->definitions[0] = Definition(fmask_load_address);
4143          ctx->block->instructions.emplace_back(std::move(vec));
4144
4145          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4146          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4147       }
4148       count--;
4149       coords[count] = sample_index;
4150    }
4151
4152    if (count == 1 && !gfx9_1d)
4153       return emit_extract_vector(ctx, src0, 0, v1);
4154
4155    if (gfx9_1d) {
4156       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4157       coords.resize(coords.size() + 1);
4158       coords[1] = Operand((uint32_t) 0);
4159       if (is_array)
4160          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4161    } else {
4162       for (int i = 0; i < count; i++)
4163          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4164    }
4165
4166    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4167        instr->intrinsic == nir_intrinsic_image_deref_store) {
4168       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4169       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4170
4171       if (!level_zero)
4172          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4173    }
4174
4175    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4176    for (unsigned i = 0; i < coords.size(); i++)
4177       vec->operands[i] = coords[i];
4178    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4179    vec->definitions[0] = Definition(res);
4180    ctx->block->instructions.emplace_back(std::move(vec));
4181    return res;
4182 }
4183
4184
4185 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4186 {
4187    Builder bld(ctx->program, ctx->block);
4188    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4189    const struct glsl_type *type = glsl_without_array(var->type);
4190    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4191    bool is_array = glsl_sampler_type_is_array(type);
4192    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4193
4194    if (dim == GLSL_SAMPLER_DIM_BUF) {
4195       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4196       unsigned num_channels = util_last_bit(mask);
4197       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4198       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4199
4200       aco_opcode opcode;
4201       switch (num_channels) {
4202       case 1:
4203          opcode = aco_opcode::buffer_load_format_x;
4204          break;
4205       case 2:
4206          opcode = aco_opcode::buffer_load_format_xy;
4207          break;
4208       case 3:
4209          opcode = aco_opcode::buffer_load_format_xyz;
4210          break;
4211       case 4:
4212          opcode = aco_opcode::buffer_load_format_xyzw;
4213          break;
4214       default:
4215          unreachable(">4 channel buffer image load");
4216       }
4217       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4218       load->operands[0] = Operand(vindex);
4219       load->operands[1] = Operand(rsrc);
4220       load->operands[2] = Operand((uint32_t) 0);
4221       Temp tmp;
4222       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4223          tmp = dst;
4224       else
4225          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4226       load->definitions[0] = Definition(tmp);
4227       load->idxen = true;
4228       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4229       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4230       load->barrier = barrier_image;
4231       ctx->block->instructions.emplace_back(std::move(load));
4232
4233       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4234       return;
4235    }
4236
4237    Temp coords = get_image_coords(ctx, instr, type);
4238    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4239
4240    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4241    unsigned num_components = util_bitcount(dmask);
4242    Temp tmp;
4243    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4244       tmp = dst;
4245    else
4246       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4247
4248    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4249    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4250
4251    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4252    load->operands[0] = Operand(coords);
4253    load->operands[1] = Operand(resource);
4254    load->definitions[0] = Definition(tmp);
4255    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4256    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4257    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4258    load->dmask = dmask;
4259    load->unrm = true;
4260    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4261    load->barrier = barrier_image;
4262    ctx->block->instructions.emplace_back(std::move(load));
4263
4264    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4265    return;
4266 }
4267
4268 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4269 {
4270    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4271    const struct glsl_type *type = glsl_without_array(var->type);
4272    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4273    bool is_array = glsl_sampler_type_is_array(type);
4274    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4275
4276    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4277
4278    if (dim == GLSL_SAMPLER_DIM_BUF) {
4279       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4280       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4281       aco_opcode opcode;
4282       switch (data.size()) {
4283       case 1:
4284          opcode = aco_opcode::buffer_store_format_x;
4285          break;
4286       case 2:
4287          opcode = aco_opcode::buffer_store_format_xy;
4288          break;
4289       case 3:
4290          opcode = aco_opcode::buffer_store_format_xyz;
4291          break;
4292       case 4:
4293          opcode = aco_opcode::buffer_store_format_xyzw;
4294          break;
4295       default:
4296          unreachable(">4 channel buffer image store");
4297       }
4298       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4299       store->operands[0] = Operand(vindex);
4300       store->operands[1] = Operand(rsrc);
4301       store->operands[2] = Operand((uint32_t) 0);
4302       store->operands[3] = Operand(data);
4303       store->idxen = true;
4304       store->glc = glc;
4305       store->dlc = false;
4306       store->disable_wqm = true;
4307       store->barrier = barrier_image;
4308       ctx->program->needs_exact = true;
4309       ctx->block->instructions.emplace_back(std::move(store));
4310       return;
4311    }
4312
4313    assert(data.type() == RegType::vgpr);
4314    Temp coords = get_image_coords(ctx, instr, type);
4315    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4316
4317    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4318    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4319
4320    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4321    store->operands[0] = Operand(coords);
4322    store->operands[1] = Operand(resource);
4323    store->operands[2] = Operand(s4);
4324    store->operands[3] = Operand(data);
4325    store->glc = glc;
4326    store->dlc = false;
4327    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4328    store->dmask = (1 << data.size()) - 1;
4329    store->unrm = true;
4330    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4331    store->disable_wqm = true;
4332    store->barrier = barrier_image;
4333    ctx->program->needs_exact = true;
4334    ctx->block->instructions.emplace_back(std::move(store));
4335    return;
4336 }
4337
4338 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4339 {
4340    /* return the previous value if dest is ever used */
4341    bool return_previous = false;
4342    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4343       return_previous = true;
4344       break;
4345    }
4346    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4347       return_previous = true;
4348       break;
4349    }
4350
4351    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4352    const struct glsl_type *type = glsl_without_array(var->type);
4353    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4354    bool is_array = glsl_sampler_type_is_array(type);
4355    Builder bld(ctx->program, ctx->block);
4356
4357    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4358    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4359
4360    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4361       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4362
4363    aco_opcode buf_op, image_op;
4364    switch (instr->intrinsic) {
4365       case nir_intrinsic_image_deref_atomic_add:
4366          buf_op = aco_opcode::buffer_atomic_add;
4367          image_op = aco_opcode::image_atomic_add;
4368          break;
4369       case nir_intrinsic_image_deref_atomic_umin:
4370          buf_op = aco_opcode::buffer_atomic_umin;
4371          image_op = aco_opcode::image_atomic_umin;
4372          break;
4373       case nir_intrinsic_image_deref_atomic_imin:
4374          buf_op = aco_opcode::buffer_atomic_smin;
4375          image_op = aco_opcode::image_atomic_smin;
4376          break;
4377       case nir_intrinsic_image_deref_atomic_umax:
4378          buf_op = aco_opcode::buffer_atomic_umax;
4379          image_op = aco_opcode::image_atomic_umax;
4380          break;
4381       case nir_intrinsic_image_deref_atomic_imax:
4382          buf_op = aco_opcode::buffer_atomic_smax;
4383          image_op = aco_opcode::image_atomic_smax;
4384          break;
4385       case nir_intrinsic_image_deref_atomic_and:
4386          buf_op = aco_opcode::buffer_atomic_and;
4387          image_op = aco_opcode::image_atomic_and;
4388          break;
4389       case nir_intrinsic_image_deref_atomic_or:
4390          buf_op = aco_opcode::buffer_atomic_or;
4391          image_op = aco_opcode::image_atomic_or;
4392          break;
4393       case nir_intrinsic_image_deref_atomic_xor:
4394          buf_op = aco_opcode::buffer_atomic_xor;
4395          image_op = aco_opcode::image_atomic_xor;
4396          break;
4397       case nir_intrinsic_image_deref_atomic_exchange:
4398          buf_op = aco_opcode::buffer_atomic_swap;
4399          image_op = aco_opcode::image_atomic_swap;
4400          break;
4401       case nir_intrinsic_image_deref_atomic_comp_swap:
4402          buf_op = aco_opcode::buffer_atomic_cmpswap;
4403          image_op = aco_opcode::image_atomic_cmpswap;
4404          break;
4405       default:
4406          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4407    }
4408
4409    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4410
4411    if (dim == GLSL_SAMPLER_DIM_BUF) {
4412       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4413       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4414       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4415       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4416       mubuf->operands[0] = Operand(vindex);
4417       mubuf->operands[1] = Operand(resource);
4418       mubuf->operands[2] = Operand((uint32_t)0);
4419       mubuf->operands[3] = Operand(data);
4420       if (return_previous)
4421          mubuf->definitions[0] = Definition(dst);
4422       mubuf->offset = 0;
4423       mubuf->idxen = true;
4424       mubuf->glc = return_previous;
4425       mubuf->dlc = false; /* Not needed for atomics */
4426       mubuf->disable_wqm = true;
4427       mubuf->barrier = barrier_image;
4428       ctx->program->needs_exact = true;
4429       ctx->block->instructions.emplace_back(std::move(mubuf));
4430       return;
4431    }
4432
4433    Temp coords = get_image_coords(ctx, instr, type);
4434    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4435    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4436    mimg->operands[0] = Operand(coords);
4437    mimg->operands[1] = Operand(resource);
4438    mimg->operands[2] = Operand(s4); /* no sampler */
4439    mimg->operands[3] = Operand(data);
4440    if (return_previous)
4441       mimg->definitions[0] = Definition(dst);
4442    mimg->glc = return_previous;
4443    mimg->dlc = false; /* Not needed for atomics */
4444    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4445    mimg->dmask = (1 << data.size()) - 1;
4446    mimg->unrm = true;
4447    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4448    mimg->disable_wqm = true;
4449    mimg->barrier = barrier_image;
4450    ctx->program->needs_exact = true;
4451    ctx->block->instructions.emplace_back(std::move(mimg));
4452    return;
4453 }
4454
4455 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4456 {
4457    if (in_elements && ctx->options->chip_class == GFX8) {
4458       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4459       Builder bld(ctx->program, ctx->block);
4460
4461       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4462
4463       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4464       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4465
4466       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4467       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4468
4469       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4470       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4471
4472       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4473       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4474                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4475       if (dst.type() == RegType::vgpr)
4476          bld.copy(Definition(dst), shr_dst);
4477
4478       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4479    } else {
4480       emit_extract_vector(ctx, desc, 2, dst);
4481    }
4482 }
4483
4484 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4485 {
4486    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4487    const struct glsl_type *type = glsl_without_array(var->type);
4488    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4489    bool is_array = glsl_sampler_type_is_array(type);
4490    Builder bld(ctx->program, ctx->block);
4491
4492    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4493       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4494       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4495    }
4496
4497    /* LOD */
4498    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4499
4500    /* Resource */
4501    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4502
4503    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4504
4505    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4506    mimg->operands[0] = Operand(lod);
4507    mimg->operands[1] = Operand(resource);
4508    uint8_t& dmask = mimg->dmask;
4509    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4510    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4511    mimg->da = glsl_sampler_type_is_array(type);
4512    mimg->can_reorder = true;
4513    Definition& def = mimg->definitions[0];
4514    ctx->block->instructions.emplace_back(std::move(mimg));
4515
4516    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4517        glsl_sampler_type_is_array(type)) {
4518
4519       assert(instr->dest.ssa.num_components == 3);
4520       Temp tmp = {ctx->program->allocateId(), v3};
4521       def = Definition(tmp);
4522       emit_split_vector(ctx, tmp, 3);
4523
4524       /* divide 3rd value by 6 by multiplying with magic number */
4525       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4526       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4527
4528       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4529                  emit_extract_vector(ctx, tmp, 0, v1),
4530                  emit_extract_vector(ctx, tmp, 1, v1),
4531                  by_6);
4532
4533    } else if (ctx->options->chip_class == GFX9 &&
4534               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4535               glsl_sampler_type_is_array(type)) {
4536       assert(instr->dest.ssa.num_components == 2);
4537       def = Definition(dst);
4538       dmask = 0x5;
4539    } else {
4540       def = Definition(dst);
4541    }
4542
4543    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4544 }
4545
4546 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4547 {
4548    Builder bld(ctx->program, ctx->block);
4549    unsigned num_components = instr->num_components;
4550
4551    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4552    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4553    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4554
4555    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4556    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4557 }
4558
4559 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4560 {
4561    Builder bld(ctx->program, ctx->block);
4562    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4563    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4564    unsigned writemask = nir_intrinsic_write_mask(instr);
4565    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4566
4567    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4568    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4569
4570    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4571                ctx->options->chip_class >= GFX8;
4572    if (smem)
4573       offset = bld.as_uniform(offset);
4574    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4575
4576    while (writemask) {
4577       int start, count;
4578       u_bit_scan_consecutive_range(&writemask, &start, &count);
4579       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4580          /* GFX6 doesn't support storing vec3, split it. */
4581          writemask |= 1u << (start + 2);
4582          count = 2;
4583       }
4584       int num_bytes = count * elem_size_bytes;
4585
4586       if (num_bytes > 16) {
4587          assert(elem_size_bytes == 8);
4588          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4589          count = 2;
4590          num_bytes = 16;
4591       }
4592
4593       // TODO: check alignment of sub-dword stores
4594       // TODO: split 3 bytes. there is no store instruction for that
4595
4596       Temp write_data;
4597       if (count != instr->num_components) {
4598          emit_split_vector(ctx, data, instr->num_components);
4599          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4600          for (int i = 0; i < count; i++) {
4601             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4602             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4603          }
4604          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4605          vec->definitions[0] = Definition(write_data);
4606          ctx->block->instructions.emplace_back(std::move(vec));
4607       } else if (!smem && data.type() != RegType::vgpr) {
4608          assert(num_bytes % 4 == 0);
4609          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4610       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4611          assert(num_bytes % 4 == 0);
4612          write_data = bld.as_uniform(data);
4613       } else {
4614          write_data = data;
4615       }
4616
4617       aco_opcode vmem_op, smem_op;
4618       switch (num_bytes) {
4619          case 4:
4620             vmem_op = aco_opcode::buffer_store_dword;
4621             smem_op = aco_opcode::s_buffer_store_dword;
4622             break;
4623          case 8:
4624             vmem_op = aco_opcode::buffer_store_dwordx2;
4625             smem_op = aco_opcode::s_buffer_store_dwordx2;
4626             break;
4627          case 12:
4628             vmem_op = aco_opcode::buffer_store_dwordx3;
4629             smem_op = aco_opcode::last_opcode;
4630             assert(!smem && ctx->options->chip_class > GFX6);
4631             break;
4632          case 16:
4633             vmem_op = aco_opcode::buffer_store_dwordx4;
4634             smem_op = aco_opcode::s_buffer_store_dwordx4;
4635             break;
4636          default:
4637             unreachable("Store SSBO not implemented for this size.");
4638       }
4639       if (ctx->stage == fragment_fs)
4640          smem_op = aco_opcode::p_fs_buffer_store_smem;
4641
4642       if (smem) {
4643          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4644          store->operands[0] = Operand(rsrc);
4645          if (start) {
4646             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4647                                 offset, Operand(start * elem_size_bytes));
4648             store->operands[1] = Operand(off);
4649          } else {
4650             store->operands[1] = Operand(offset);
4651          }
4652          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4653             store->operands[1].setFixed(m0);
4654          store->operands[2] = Operand(write_data);
4655          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4656          store->dlc = false;
4657          store->disable_wqm = true;
4658          store->barrier = barrier_buffer;
4659          ctx->block->instructions.emplace_back(std::move(store));
4660          ctx->program->wb_smem_l1_on_end = true;
4661          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4662             ctx->block->kind |= block_kind_needs_lowering;
4663             ctx->program->needs_exact = true;
4664          }
4665       } else {
4666          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4667          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4668          store->operands[1] = Operand(rsrc);
4669          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4670          store->operands[3] = Operand(write_data);
4671          store->offset = start * elem_size_bytes;
4672          store->offen = (offset.type() == RegType::vgpr);
4673          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4674          store->dlc = false;
4675          store->disable_wqm = true;
4676          store->barrier = barrier_buffer;
4677          ctx->program->needs_exact = true;
4678          ctx->block->instructions.emplace_back(std::move(store));
4679       }
4680    }
4681 }
4682
4683 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4684 {
4685    /* return the previous value if dest is ever used */
4686    bool return_previous = false;
4687    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4688       return_previous = true;
4689       break;
4690    }
4691    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4692       return_previous = true;
4693       break;
4694    }
4695
4696    Builder bld(ctx->program, ctx->block);
4697    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4698
4699    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4700       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4701                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4702
4703    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4704    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4705    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4706
4707    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4708
4709    aco_opcode op32, op64;
4710    switch (instr->intrinsic) {
4711       case nir_intrinsic_ssbo_atomic_add:
4712          op32 = aco_opcode::buffer_atomic_add;
4713          op64 = aco_opcode::buffer_atomic_add_x2;
4714          break;
4715       case nir_intrinsic_ssbo_atomic_imin:
4716          op32 = aco_opcode::buffer_atomic_smin;
4717          op64 = aco_opcode::buffer_atomic_smin_x2;
4718          break;
4719       case nir_intrinsic_ssbo_atomic_umin:
4720          op32 = aco_opcode::buffer_atomic_umin;
4721          op64 = aco_opcode::buffer_atomic_umin_x2;
4722          break;
4723       case nir_intrinsic_ssbo_atomic_imax:
4724          op32 = aco_opcode::buffer_atomic_smax;
4725          op64 = aco_opcode::buffer_atomic_smax_x2;
4726          break;
4727       case nir_intrinsic_ssbo_atomic_umax:
4728          op32 = aco_opcode::buffer_atomic_umax;
4729          op64 = aco_opcode::buffer_atomic_umax_x2;
4730          break;
4731       case nir_intrinsic_ssbo_atomic_and:
4732          op32 = aco_opcode::buffer_atomic_and;
4733          op64 = aco_opcode::buffer_atomic_and_x2;
4734          break;
4735       case nir_intrinsic_ssbo_atomic_or:
4736          op32 = aco_opcode::buffer_atomic_or;
4737          op64 = aco_opcode::buffer_atomic_or_x2;
4738          break;
4739       case nir_intrinsic_ssbo_atomic_xor:
4740          op32 = aco_opcode::buffer_atomic_xor;
4741          op64 = aco_opcode::buffer_atomic_xor_x2;
4742          break;
4743       case nir_intrinsic_ssbo_atomic_exchange:
4744          op32 = aco_opcode::buffer_atomic_swap;
4745          op64 = aco_opcode::buffer_atomic_swap_x2;
4746          break;
4747       case nir_intrinsic_ssbo_atomic_comp_swap:
4748          op32 = aco_opcode::buffer_atomic_cmpswap;
4749          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4750          break;
4751       default:
4752          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4753    }
4754    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4755    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4756    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4757    mubuf->operands[1] = Operand(rsrc);
4758    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4759    mubuf->operands[3] = Operand(data);
4760    if (return_previous)
4761       mubuf->definitions[0] = Definition(dst);
4762    mubuf->offset = 0;
4763    mubuf->offen = (offset.type() == RegType::vgpr);
4764    mubuf->glc = return_previous;
4765    mubuf->dlc = false; /* Not needed for atomics */
4766    mubuf->disable_wqm = true;
4767    mubuf->barrier = barrier_buffer;
4768    ctx->program->needs_exact = true;
4769    ctx->block->instructions.emplace_back(std::move(mubuf));
4770 }
4771
4772 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4773
4774    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4775    Builder bld(ctx->program, ctx->block);
4776    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4777    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4778 }
4779
4780 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4781 {
4782    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4783                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4784
4785    if (addr.type() == RegType::vgpr)
4786       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4787    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4788 }
4789
4790 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4791 {
4792    Builder bld(ctx->program, ctx->block);
4793    unsigned num_components = instr->num_components;
4794    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4795
4796    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4797    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4798
4799    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4800    bool dlc = glc && ctx->options->chip_class >= GFX10;
4801    aco_opcode op;
4802    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4803       bool global = ctx->options->chip_class >= GFX9;
4804
4805       if (ctx->options->chip_class >= GFX7) {
4806          aco_opcode op;
4807          switch (num_bytes) {
4808          case 4:
4809             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4810             break;
4811          case 8:
4812             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4813             break;
4814          case 12:
4815             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4816             break;
4817          case 16:
4818             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4819             break;
4820          default:
4821             unreachable("load_global not implemented for this size.");
4822          }
4823
4824          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4825          flat->operands[0] = Operand(addr);
4826          flat->operands[1] = Operand(s1);
4827          flat->glc = glc;
4828          flat->dlc = dlc;
4829          flat->barrier = barrier_buffer;
4830
4831          if (dst.type() == RegType::sgpr) {
4832             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4833             flat->definitions[0] = Definition(vec);
4834             ctx->block->instructions.emplace_back(std::move(flat));
4835             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4836          } else {
4837             flat->definitions[0] = Definition(dst);
4838             ctx->block->instructions.emplace_back(std::move(flat));
4839          }
4840          emit_split_vector(ctx, dst, num_components);
4841       } else {
4842          assert(ctx->options->chip_class == GFX6);
4843
4844          /* GFX6 doesn't support loading vec3, expand to vec4. */
4845          num_bytes = num_bytes == 12 ? 16 : num_bytes;
4846
4847          aco_opcode op;
4848          switch (num_bytes) {
4849          case 4:
4850             op = aco_opcode::buffer_load_dword;
4851             break;
4852          case 8:
4853             op = aco_opcode::buffer_load_dwordx2;
4854             break;
4855          case 16:
4856             op = aco_opcode::buffer_load_dwordx4;
4857             break;
4858          default:
4859             unreachable("load_global not implemented for this size.");
4860          }
4861
4862          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
4863
4864          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4865          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
4866          mubuf->operands[1] = Operand(rsrc);
4867          mubuf->operands[2] = Operand(0u);
4868          mubuf->glc = glc;
4869          mubuf->dlc = false;
4870          mubuf->offset = 0;
4871          mubuf->addr64 = addr.type() == RegType::vgpr;
4872          mubuf->disable_wqm = false;
4873          mubuf->barrier = barrier_buffer;
4874          aco_ptr<Instruction> instr = std::move(mubuf);
4875
4876          /* expand vector */
4877          if (dst.size() == 3) {
4878             Temp vec = bld.tmp(v4);
4879             instr->definitions[0] = Definition(vec);
4880             bld.insert(std::move(instr));
4881             emit_split_vector(ctx, vec, 4);
4882
4883             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4884             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4885             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4886             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4887          }
4888
4889          if (dst.type() == RegType::sgpr) {
4890             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4891             instr->definitions[0] = Definition(vec);
4892             bld.insert(std::move(instr));
4893             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4894             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4895          } else {
4896             instr->definitions[0] = Definition(dst);
4897             bld.insert(std::move(instr));
4898             emit_split_vector(ctx, dst, num_components);
4899          }
4900       }
4901    } else {
4902       switch (num_bytes) {
4903          case 4:
4904             op = aco_opcode::s_load_dword;
4905             break;
4906          case 8:
4907             op = aco_opcode::s_load_dwordx2;
4908             break;
4909          case 12:
4910          case 16:
4911             op = aco_opcode::s_load_dwordx4;
4912             break;
4913          default:
4914             unreachable("load_global not implemented for this size.");
4915       }
4916       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4917       load->operands[0] = Operand(addr);
4918       load->operands[1] = Operand(0u);
4919       load->definitions[0] = Definition(dst);
4920       load->glc = glc;
4921       load->dlc = dlc;
4922       load->barrier = barrier_buffer;
4923       assert(ctx->options->chip_class >= GFX8 || !glc);
4924
4925       if (dst.size() == 3) {
4926          /* trim vector */
4927          Temp vec = bld.tmp(s4);
4928          load->definitions[0] = Definition(vec);
4929          ctx->block->instructions.emplace_back(std::move(load));
4930          emit_split_vector(ctx, vec, 4);
4931
4932          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4933                     emit_extract_vector(ctx, vec, 0, s1),
4934                     emit_extract_vector(ctx, vec, 1, s1),
4935                     emit_extract_vector(ctx, vec, 2, s1));
4936       } else {
4937          ctx->block->instructions.emplace_back(std::move(load));
4938       }
4939    }
4940 }
4941
4942 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4943 {
4944    Builder bld(ctx->program, ctx->block);
4945    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4946
4947    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4948    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
4949
4950    if (ctx->options->chip_class >= GFX7)
4951       addr = as_vgpr(ctx, addr);
4952
4953    unsigned writemask = nir_intrinsic_write_mask(instr);
4954    while (writemask) {
4955       int start, count;
4956       u_bit_scan_consecutive_range(&writemask, &start, &count);
4957       if (count == 3 && ctx->options->chip_class == GFX6) {
4958          /* GFX6 doesn't support storing vec3, split it. */
4959          writemask |= 1u << (start + 2);
4960          count = 2;
4961       }
4962       unsigned num_bytes = count * elem_size_bytes;
4963
4964       Temp write_data = data;
4965       if (count != instr->num_components) {
4966          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4967          for (int i = 0; i < count; i++)
4968             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4969          write_data = bld.tmp(RegType::vgpr, count);
4970          vec->definitions[0] = Definition(write_data);
4971          ctx->block->instructions.emplace_back(std::move(vec));
4972       }
4973
4974       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4975       unsigned offset = start * elem_size_bytes;
4976
4977       if (ctx->options->chip_class >= GFX7) {
4978          if (offset > 0 && ctx->options->chip_class < GFX9) {
4979             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4980             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4981             Temp carry = bld.tmp(bld.lm);
4982             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4983
4984             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4985                      Operand(offset), addr0);
4986             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4987                      Operand(0u), addr1,
4988                      carry).def(1).setHint(vcc);
4989
4990             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4991
4992             offset = 0;
4993          }
4994
4995          bool global = ctx->options->chip_class >= GFX9;
4996          aco_opcode op;
4997          switch (num_bytes) {
4998          case 4:
4999             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5000             break;
5001          case 8:
5002             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5003             break;
5004          case 12:
5005             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5006             break;
5007          case 16:
5008             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5009             break;
5010          default:
5011             unreachable("store_global not implemented for this size.");
5012          }
5013
5014          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5015          flat->operands[0] = Operand(addr);
5016          flat->operands[1] = Operand(s1);
5017          flat->operands[2] = Operand(data);
5018          flat->glc = glc;
5019          flat->dlc = false;
5020          flat->offset = offset;
5021          flat->disable_wqm = true;
5022          flat->barrier = barrier_buffer;
5023          ctx->program->needs_exact = true;
5024          ctx->block->instructions.emplace_back(std::move(flat));
5025       } else {
5026          assert(ctx->options->chip_class == GFX6);
5027
5028          aco_opcode op;
5029          switch (num_bytes) {
5030          case 4:
5031             op = aco_opcode::buffer_store_dword;
5032             break;
5033          case 8:
5034             op = aco_opcode::buffer_store_dwordx2;
5035             break;
5036          case 16:
5037             op = aco_opcode::buffer_store_dwordx4;
5038             break;
5039          default:
5040             unreachable("store_global not implemented for this size.");
5041          }
5042
5043          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5044
5045          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5046          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5047          mubuf->operands[1] = Operand(rsrc);
5048          mubuf->operands[2] = Operand(0u);
5049          mubuf->operands[3] = Operand(write_data);
5050          mubuf->glc = glc;
5051          mubuf->dlc = false;
5052          mubuf->offset = offset;
5053          mubuf->addr64 = addr.type() == RegType::vgpr;
5054          mubuf->disable_wqm = true;
5055          mubuf->barrier = barrier_buffer;
5056          ctx->program->needs_exact = true;
5057          ctx->block->instructions.emplace_back(std::move(mubuf));
5058       }
5059    }
5060 }
5061
5062 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5063 {
5064    /* return the previous value if dest is ever used */
5065    bool return_previous = false;
5066    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5067       return_previous = true;
5068       break;
5069    }
5070    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5071       return_previous = true;
5072       break;
5073    }
5074
5075    Builder bld(ctx->program, ctx->block);
5076    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5077    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5078
5079    if (ctx->options->chip_class >= GFX7)
5080       addr = as_vgpr(ctx, addr);
5081
5082    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5083       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5084                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5085
5086    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5087
5088    aco_opcode op32, op64;
5089
5090    if (ctx->options->chip_class >= GFX7) {
5091       bool global = ctx->options->chip_class >= GFX9;
5092       switch (instr->intrinsic) {
5093          case nir_intrinsic_global_atomic_add:
5094             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5095             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5096             break;
5097          case nir_intrinsic_global_atomic_imin:
5098             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5099             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5100             break;
5101          case nir_intrinsic_global_atomic_umin:
5102             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5103             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5104             break;
5105          case nir_intrinsic_global_atomic_imax:
5106             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5107             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5108             break;
5109          case nir_intrinsic_global_atomic_umax:
5110             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5111             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5112             break;
5113          case nir_intrinsic_global_atomic_and:
5114             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5115             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5116             break;
5117          case nir_intrinsic_global_atomic_or:
5118             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5119             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5120             break;
5121          case nir_intrinsic_global_atomic_xor:
5122             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5123             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5124             break;
5125          case nir_intrinsic_global_atomic_exchange:
5126             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5127             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5128             break;
5129          case nir_intrinsic_global_atomic_comp_swap:
5130             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5131             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5132             break;
5133          default:
5134             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5135       }
5136
5137       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5138       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5139       flat->operands[0] = Operand(addr);
5140       flat->operands[1] = Operand(s1);
5141       flat->operands[2] = Operand(data);
5142       if (return_previous)
5143          flat->definitions[0] = Definition(dst);
5144       flat->glc = return_previous;
5145       flat->dlc = false; /* Not needed for atomics */
5146       flat->offset = 0;
5147       flat->disable_wqm = true;
5148       flat->barrier = barrier_buffer;
5149       ctx->program->needs_exact = true;
5150       ctx->block->instructions.emplace_back(std::move(flat));
5151    } else {
5152       assert(ctx->options->chip_class == GFX6);
5153
5154       switch (instr->intrinsic) {
5155          case nir_intrinsic_global_atomic_add:
5156             op32 = aco_opcode::buffer_atomic_add;
5157             op64 = aco_opcode::buffer_atomic_add_x2;
5158             break;
5159          case nir_intrinsic_global_atomic_imin:
5160             op32 = aco_opcode::buffer_atomic_smin;
5161             op64 = aco_opcode::buffer_atomic_smin_x2;
5162             break;
5163          case nir_intrinsic_global_atomic_umin:
5164             op32 = aco_opcode::buffer_atomic_umin;
5165             op64 = aco_opcode::buffer_atomic_umin_x2;
5166             break;
5167          case nir_intrinsic_global_atomic_imax:
5168             op32 = aco_opcode::buffer_atomic_smax;
5169             op64 = aco_opcode::buffer_atomic_smax_x2;
5170             break;
5171          case nir_intrinsic_global_atomic_umax:
5172             op32 = aco_opcode::buffer_atomic_umax;
5173             op64 = aco_opcode::buffer_atomic_umax_x2;
5174             break;
5175          case nir_intrinsic_global_atomic_and:
5176             op32 = aco_opcode::buffer_atomic_and;
5177             op64 = aco_opcode::buffer_atomic_and_x2;
5178             break;
5179          case nir_intrinsic_global_atomic_or:
5180             op32 = aco_opcode::buffer_atomic_or;
5181             op64 = aco_opcode::buffer_atomic_or_x2;
5182             break;
5183          case nir_intrinsic_global_atomic_xor:
5184             op32 = aco_opcode::buffer_atomic_xor;
5185             op64 = aco_opcode::buffer_atomic_xor_x2;
5186             break;
5187          case nir_intrinsic_global_atomic_exchange:
5188             op32 = aco_opcode::buffer_atomic_swap;
5189             op64 = aco_opcode::buffer_atomic_swap_x2;
5190             break;
5191          case nir_intrinsic_global_atomic_comp_swap:
5192             op32 = aco_opcode::buffer_atomic_cmpswap;
5193             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5194             break;
5195          default:
5196             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5197       }
5198
5199       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5200
5201       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5202
5203       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5204       mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5205       mubuf->operands[1] = Operand(rsrc);
5206       mubuf->operands[2] = Operand(0u);
5207       mubuf->operands[3] = Operand(data);
5208       if (return_previous)
5209          mubuf->definitions[0] = Definition(dst);
5210       mubuf->glc = return_previous;
5211       mubuf->dlc = false;
5212       mubuf->offset = 0;
5213       mubuf->addr64 = addr.type() == RegType::vgpr;
5214       mubuf->disable_wqm = true;
5215       mubuf->barrier = barrier_buffer;
5216       ctx->program->needs_exact = true;
5217       ctx->block->instructions.emplace_back(std::move(mubuf));
5218    }
5219 }
5220
5221 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5222    Builder bld(ctx->program, ctx->block);
5223    switch(instr->intrinsic) {
5224       case nir_intrinsic_group_memory_barrier:
5225       case nir_intrinsic_memory_barrier:
5226          bld.barrier(aco_opcode::p_memory_barrier_all);
5227          break;
5228       case nir_intrinsic_memory_barrier_buffer:
5229          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5230          break;
5231       case nir_intrinsic_memory_barrier_image:
5232          bld.barrier(aco_opcode::p_memory_barrier_image);
5233          break;
5234       case nir_intrinsic_memory_barrier_shared:
5235          bld.barrier(aco_opcode::p_memory_barrier_shared);
5236          break;
5237       default:
5238          unreachable("Unimplemented memory barrier intrinsic");
5239          break;
5240    }
5241 }
5242
5243 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5244 {
5245    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5246    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5247    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5248    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5249    Builder bld(ctx->program, ctx->block);
5250
5251    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5252    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5253    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5254 }
5255
5256 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5257 {
5258    unsigned writemask = nir_intrinsic_write_mask(instr);
5259    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5260    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5261    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5262    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5263
5264    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5265    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5266 }
5267
5268 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5269 {
5270    unsigned offset = nir_intrinsic_base(instr);
5271    Operand m = load_lds_size_m0(ctx);
5272    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5273    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5274
5275    unsigned num_operands = 3;
5276    aco_opcode op32, op64, op32_rtn, op64_rtn;
5277    switch(instr->intrinsic) {
5278       case nir_intrinsic_shared_atomic_add:
5279          op32 = aco_opcode::ds_add_u32;
5280          op64 = aco_opcode::ds_add_u64;
5281          op32_rtn = aco_opcode::ds_add_rtn_u32;
5282          op64_rtn = aco_opcode::ds_add_rtn_u64;
5283          break;
5284       case nir_intrinsic_shared_atomic_imin:
5285          op32 = aco_opcode::ds_min_i32;
5286          op64 = aco_opcode::ds_min_i64;
5287          op32_rtn = aco_opcode::ds_min_rtn_i32;
5288          op64_rtn = aco_opcode::ds_min_rtn_i64;
5289          break;
5290       case nir_intrinsic_shared_atomic_umin:
5291          op32 = aco_opcode::ds_min_u32;
5292          op64 = aco_opcode::ds_min_u64;
5293          op32_rtn = aco_opcode::ds_min_rtn_u32;
5294          op64_rtn = aco_opcode::ds_min_rtn_u64;
5295          break;
5296       case nir_intrinsic_shared_atomic_imax:
5297          op32 = aco_opcode::ds_max_i32;
5298          op64 = aco_opcode::ds_max_i64;
5299          op32_rtn = aco_opcode::ds_max_rtn_i32;
5300          op64_rtn = aco_opcode::ds_max_rtn_i64;
5301          break;
5302       case nir_intrinsic_shared_atomic_umax:
5303          op32 = aco_opcode::ds_max_u32;
5304          op64 = aco_opcode::ds_max_u64;
5305          op32_rtn = aco_opcode::ds_max_rtn_u32;
5306          op64_rtn = aco_opcode::ds_max_rtn_u64;
5307          break;
5308       case nir_intrinsic_shared_atomic_and:
5309          op32 = aco_opcode::ds_and_b32;
5310          op64 = aco_opcode::ds_and_b64;
5311          op32_rtn = aco_opcode::ds_and_rtn_b32;
5312          op64_rtn = aco_opcode::ds_and_rtn_b64;
5313          break;
5314       case nir_intrinsic_shared_atomic_or:
5315          op32 = aco_opcode::ds_or_b32;
5316          op64 = aco_opcode::ds_or_b64;
5317          op32_rtn = aco_opcode::ds_or_rtn_b32;
5318          op64_rtn = aco_opcode::ds_or_rtn_b64;
5319          break;
5320       case nir_intrinsic_shared_atomic_xor:
5321          op32 = aco_opcode::ds_xor_b32;
5322          op64 = aco_opcode::ds_xor_b64;
5323          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5324          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5325          break;
5326       case nir_intrinsic_shared_atomic_exchange:
5327          op32 = aco_opcode::ds_write_b32;
5328          op64 = aco_opcode::ds_write_b64;
5329          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5330          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5331          break;
5332       case nir_intrinsic_shared_atomic_comp_swap:
5333          op32 = aco_opcode::ds_cmpst_b32;
5334          op64 = aco_opcode::ds_cmpst_b64;
5335          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5336          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5337          num_operands = 4;
5338          break;
5339       default:
5340          unreachable("Unhandled shared atomic intrinsic");
5341    }
5342
5343    /* return the previous value if dest is ever used */
5344    bool return_previous = false;
5345    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5346       return_previous = true;
5347       break;
5348    }
5349    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5350       return_previous = true;
5351       break;
5352    }
5353
5354    aco_opcode op;
5355    if (data.size() == 1) {
5356       assert(instr->dest.ssa.bit_size == 32);
5357       op = return_previous ? op32_rtn : op32;
5358    } else {
5359       assert(instr->dest.ssa.bit_size == 64);
5360       op = return_previous ? op64_rtn : op64;
5361    }
5362
5363    if (offset > 65535) {
5364       Builder bld(ctx->program, ctx->block);
5365       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5366       offset = 0;
5367    }
5368
5369    aco_ptr<DS_instruction> ds;
5370    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5371    ds->operands[0] = Operand(address);
5372    ds->operands[1] = Operand(data);
5373    if (num_operands == 4)
5374       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5375    ds->operands[num_operands - 1] = m;
5376    ds->offset0 = offset;
5377    if (return_previous)
5378       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5379    ctx->block->instructions.emplace_back(std::move(ds));
5380 }
5381
5382 Temp get_scratch_resource(isel_context *ctx)
5383 {
5384    Builder bld(ctx->program, ctx->block);
5385    Temp scratch_addr = ctx->program->private_segment_buffer;
5386    if (ctx->stage != compute_cs)
5387       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5388
5389    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5390                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5391
5392    if (ctx->program->chip_class >= GFX10) {
5393       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5394                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5395                    S_008F0C_RESOURCE_LEVEL(1);
5396    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5397       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5398                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5399    }
5400
5401    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5402    if (ctx->program->chip_class <= GFX8)
5403       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5404
5405    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5406 }
5407
5408 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5409    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5410    Builder bld(ctx->program, ctx->block);
5411    Temp rsrc = get_scratch_resource(ctx);
5412    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5413    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5414
5415    aco_opcode op;
5416    switch (dst.size()) {
5417       case 1:
5418          op = aco_opcode::buffer_load_dword;
5419          break;
5420       case 2:
5421          op = aco_opcode::buffer_load_dwordx2;
5422          break;
5423       case 3:
5424          op = aco_opcode::buffer_load_dwordx3;
5425          break;
5426       case 4:
5427          op = aco_opcode::buffer_load_dwordx4;
5428          break;
5429       case 6:
5430       case 8: {
5431          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5432          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5433                                 bld.def(v4), offset, rsrc,
5434                                 ctx->program->scratch_offset, 0, true);
5435          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5436                                                   aco_opcode::buffer_load_dwordx4,
5437                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5438                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5439          emit_split_vector(ctx, lower, 2);
5440          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5441          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5442          if (dst.size() == 8) {
5443             emit_split_vector(ctx, upper, 2);
5444             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5445             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5446          } else {
5447             elems[2] = upper;
5448          }
5449
5450          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5451                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5452          for (unsigned i = 0; i < dst.size() / 2; i++)
5453             vec->operands[i] = Operand(elems[i]);
5454          vec->definitions[0] = Definition(dst);
5455          bld.insert(std::move(vec));
5456          ctx->allocated_vec.emplace(dst.id(), elems);
5457          return;
5458       }
5459       default:
5460          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5461    }
5462
5463    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5464    emit_split_vector(ctx, dst, instr->num_components);
5465 }
5466
5467 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5468    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5469    Builder bld(ctx->program, ctx->block);
5470    Temp rsrc = get_scratch_resource(ctx);
5471    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5472    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5473
5474    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5475    unsigned writemask = nir_intrinsic_write_mask(instr);
5476
5477    while (writemask) {
5478       int start, count;
5479       u_bit_scan_consecutive_range(&writemask, &start, &count);
5480       int num_bytes = count * elem_size_bytes;
5481
5482       if (num_bytes > 16) {
5483          assert(elem_size_bytes == 8);
5484          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5485          count = 2;
5486          num_bytes = 16;
5487       }
5488
5489       // TODO: check alignment of sub-dword stores
5490       // TODO: split 3 bytes. there is no store instruction for that
5491
5492       Temp write_data;
5493       if (count != instr->num_components) {
5494          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5495          for (int i = 0; i < count; i++) {
5496             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5497             vec->operands[i] = Operand(elem);
5498          }
5499          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5500          vec->definitions[0] = Definition(write_data);
5501          ctx->block->instructions.emplace_back(std::move(vec));
5502       } else {
5503          write_data = data;
5504       }
5505
5506       aco_opcode op;
5507       switch (num_bytes) {
5508          case 4:
5509             op = aco_opcode::buffer_store_dword;
5510             break;
5511          case 8:
5512             op = aco_opcode::buffer_store_dwordx2;
5513             break;
5514          case 12:
5515             op = aco_opcode::buffer_store_dwordx3;
5516             break;
5517          case 16:
5518             op = aco_opcode::buffer_store_dwordx4;
5519             break;
5520          default:
5521             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5522       }
5523
5524       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5525    }
5526 }
5527
5528 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5529    uint8_t log2_ps_iter_samples;
5530    if (ctx->program->info->ps.force_persample) {
5531       log2_ps_iter_samples =
5532          util_logbase2(ctx->options->key.fs.num_samples);
5533    } else {
5534       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5535    }
5536
5537    /* The bit pattern matches that used by fixed function fragment
5538     * processing. */
5539    static const unsigned ps_iter_masks[] = {
5540       0xffff, /* not used */
5541       0x5555,
5542       0x1111,
5543       0x0101,
5544       0x0001,
5545    };
5546    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5547
5548    Builder bld(ctx->program, ctx->block);
5549
5550    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5551                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5552    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5553    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5554    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5555    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5556 }
5557
5558 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5559 {
5560    Builder bld(ctx->program, ctx->block);
5561
5562    if (cluster_size == 1) {
5563       return src;
5564    } if (op == nir_op_iand && cluster_size == 4) {
5565       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5566       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5567       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5568                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5569    } else if (op == nir_op_ior && cluster_size == 4) {
5570       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5571       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5572                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5573    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5574       //subgroupAnd(val) -> (exec & ~val) == 0
5575       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5576       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5577       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5578    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5579       //subgroupOr(val) -> (val & exec) != 0
5580       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5581       return bool_to_vector_condition(ctx, tmp);
5582    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5583       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5584       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5585       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5586       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5587       return bool_to_vector_condition(ctx, tmp);
5588    } else {
5589       //subgroupClustered{And,Or,Xor}(val, n) ->
5590       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5591       //cluster_offset = ~(n - 1) & lane_id
5592       //cluster_mask = ((1 << n) - 1)
5593       //subgroupClusteredAnd():
5594       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5595       //subgroupClusteredOr():
5596       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5597       //subgroupClusteredXor():
5598       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5599       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5600       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5601
5602       Temp tmp;
5603       if (op == nir_op_iand)
5604          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5605       else
5606          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5607
5608       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5609
5610       if (ctx->program->chip_class <= GFX7)
5611          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5612       else if (ctx->program->wave_size == 64)
5613          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5614       else
5615          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5616       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5617       if (cluster_mask != 0xffffffff)
5618          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5619
5620       Definition cmp_def = Definition();
5621       if (op == nir_op_iand) {
5622          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5623       } else if (op == nir_op_ior) {
5624          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5625       } else if (op == nir_op_ixor) {
5626          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5627                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5628          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5629       }
5630       cmp_def.setHint(vcc);
5631       return cmp_def.getTemp();
5632    }
5633 }
5634
5635 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5636 {
5637    Builder bld(ctx->program, ctx->block);
5638
5639    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5640    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5641    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5642    Temp tmp;
5643    if (op == nir_op_iand)
5644       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5645    else
5646       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5647
5648    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5649    Temp lo = lohi.def(0).getTemp();
5650    Temp hi = lohi.def(1).getTemp();
5651    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5652
5653    Definition cmp_def = Definition();
5654    if (op == nir_op_iand)
5655       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5656    else if (op == nir_op_ior)
5657       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5658    else if (op == nir_op_ixor)
5659       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5660                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5661    cmp_def.setHint(vcc);
5662    return cmp_def.getTemp();
5663 }
5664
5665 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5666 {
5667    Builder bld(ctx->program, ctx->block);
5668
5669    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5670    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5671    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5672    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5673    if (op == nir_op_iand)
5674       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5675    else if (op == nir_op_ior)
5676       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5677    else if (op == nir_op_ixor)
5678       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5679
5680    assert(false);
5681    return Temp();
5682 }
5683
5684 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5685 {
5686    Builder bld(ctx->program, ctx->block);
5687    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5688    if (src.regClass().type() == RegType::vgpr) {
5689       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5690    } else if (src.regClass() == s1) {
5691       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5692    } else if (src.regClass() == s2) {
5693       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5694    } else {
5695       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5696       nir_print_instr(&instr->instr, stderr);
5697       fprintf(stderr, "\n");
5698    }
5699 }
5700
5701 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5702 {
5703    Builder bld(ctx->program, ctx->block);
5704    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5705    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5706    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5707
5708    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5709    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5710    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5711    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5712
5713    /* Build DD X/Y */
5714    if (ctx->program->chip_class >= GFX8) {
5715       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5716       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5717       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5718       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5719       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5720       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5721    } else {
5722       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5723       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5724       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5725       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5726       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5727       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5728       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5729       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5730       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5731       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5732    }
5733
5734    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5735    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5736    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5737    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5738    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5739    Temp wqm1 = bld.tmp(v1);
5740    emit_wqm(ctx, tmp1, wqm1, true);
5741    Temp wqm2 = bld.tmp(v1);
5742    emit_wqm(ctx, tmp2, wqm2, true);
5743    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5744    return;
5745 }
5746
5747 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5748 {
5749    Builder bld(ctx->program, ctx->block);
5750    switch(instr->intrinsic) {
5751    case nir_intrinsic_load_barycentric_sample:
5752    case nir_intrinsic_load_barycentric_pixel:
5753    case nir_intrinsic_load_barycentric_centroid: {
5754       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5755       Temp bary = Temp(0, s2);
5756       switch (mode) {
5757       case INTERP_MODE_SMOOTH:
5758       case INTERP_MODE_NONE:
5759          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5760             bary = get_arg(ctx, ctx->args->ac.persp_center);
5761          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5762             bary = ctx->persp_centroid;
5763          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5764             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5765          break;
5766       case INTERP_MODE_NOPERSPECTIVE:
5767          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5768             bary = get_arg(ctx, ctx->args->ac.linear_center);
5769          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5770             bary = ctx->linear_centroid;
5771          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5772             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5773          break;
5774       default:
5775          break;
5776       }
5777       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5778       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5779       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5780       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5781                  Operand(p1), Operand(p2));
5782       emit_split_vector(ctx, dst, 2);
5783       break;
5784    }
5785    case nir_intrinsic_load_barycentric_at_sample: {
5786       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5787       switch (ctx->options->key.fs.num_samples) {
5788          case 2: sample_pos_offset += 1 << 3; break;
5789          case 4: sample_pos_offset += 3 << 3; break;
5790          case 8: sample_pos_offset += 7 << 3; break;
5791          default: break;
5792       }
5793       Temp sample_pos;
5794       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5795       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5796       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5797       if (addr.type() == RegType::sgpr) {
5798          Operand offset;
5799          if (const_addr) {
5800             sample_pos_offset += const_addr->u32 << 3;
5801             offset = Operand(sample_pos_offset);
5802          } else if (ctx->options->chip_class >= GFX9) {
5803             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5804          } else {
5805             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5806             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5807          }
5808          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5809
5810       } else if (ctx->options->chip_class >= GFX9) {
5811          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5812          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5813       } else if (ctx->options->chip_class >= GFX7) {
5814          /* addr += private_segment_buffer + sample_pos_offset */
5815          Temp tmp0 = bld.tmp(s1);
5816          Temp tmp1 = bld.tmp(s1);
5817          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5818          Definition scc_tmp = bld.def(s1, scc);
5819          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5820          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5821          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5822          Temp pck0 = bld.tmp(v1);
5823          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5824          tmp1 = as_vgpr(ctx, tmp1);
5825          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5826          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5827
5828          /* sample_pos = flat_load_dwordx2 addr */
5829          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5830       } else {
5831          assert(ctx->options->chip_class == GFX6);
5832
5833          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5834                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5835          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
5836
5837          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5838          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
5839
5840          sample_pos = bld.tmp(v2);
5841
5842          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
5843          load->definitions[0] = Definition(sample_pos);
5844          load->operands[0] = Operand(addr);
5845          load->operands[1] = Operand(rsrc);
5846          load->operands[2] = Operand(0u);
5847          load->offset = sample_pos_offset;
5848          load->offen = 0;
5849          load->addr64 = true;
5850          load->glc = false;
5851          load->dlc = false;
5852          load->disable_wqm = false;
5853          load->barrier = barrier_none;
5854          load->can_reorder = true;
5855          ctx->block->instructions.emplace_back(std::move(load));
5856       }
5857
5858       /* sample_pos -= 0.5 */
5859       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5860       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5861       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5862       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5863       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5864
5865       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5866       break;
5867    }
5868    case nir_intrinsic_load_barycentric_at_offset: {
5869       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5870       RegClass rc = RegClass(offset.type(), 1);
5871       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5872       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5873       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5874       break;
5875    }
5876    case nir_intrinsic_load_front_face: {
5877       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5878                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5879       break;
5880    }
5881    case nir_intrinsic_load_view_index:
5882    case nir_intrinsic_load_layer_id: {
5883       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5884          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5885          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5886          break;
5887       }
5888
5889       unsigned idx = nir_intrinsic_base(instr);
5890       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5891                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5892       break;
5893    }
5894    case nir_intrinsic_load_frag_coord: {
5895       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5896       break;
5897    }
5898    case nir_intrinsic_load_sample_pos: {
5899       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5900       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5901       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5902                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5903                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5904       break;
5905    }
5906    case nir_intrinsic_load_interpolated_input:
5907       visit_load_interpolated_input(ctx, instr);
5908       break;
5909    case nir_intrinsic_store_output:
5910       visit_store_output(ctx, instr);
5911       break;
5912    case nir_intrinsic_load_input:
5913       visit_load_input(ctx, instr);
5914       break;
5915    case nir_intrinsic_load_ubo:
5916       visit_load_ubo(ctx, instr);
5917       break;
5918    case nir_intrinsic_load_push_constant:
5919       visit_load_push_constant(ctx, instr);
5920       break;
5921    case nir_intrinsic_load_constant:
5922       visit_load_constant(ctx, instr);
5923       break;
5924    case nir_intrinsic_vulkan_resource_index:
5925       visit_load_resource(ctx, instr);
5926       break;
5927    case nir_intrinsic_discard:
5928       visit_discard(ctx, instr);
5929       break;
5930    case nir_intrinsic_discard_if:
5931       visit_discard_if(ctx, instr);
5932       break;
5933    case nir_intrinsic_load_shared:
5934       visit_load_shared(ctx, instr);
5935       break;
5936    case nir_intrinsic_store_shared:
5937       visit_store_shared(ctx, instr);
5938       break;
5939    case nir_intrinsic_shared_atomic_add:
5940    case nir_intrinsic_shared_atomic_imin:
5941    case nir_intrinsic_shared_atomic_umin:
5942    case nir_intrinsic_shared_atomic_imax:
5943    case nir_intrinsic_shared_atomic_umax:
5944    case nir_intrinsic_shared_atomic_and:
5945    case nir_intrinsic_shared_atomic_or:
5946    case nir_intrinsic_shared_atomic_xor:
5947    case nir_intrinsic_shared_atomic_exchange:
5948    case nir_intrinsic_shared_atomic_comp_swap:
5949       visit_shared_atomic(ctx, instr);
5950       break;
5951    case nir_intrinsic_image_deref_load:
5952       visit_image_load(ctx, instr);
5953       break;
5954    case nir_intrinsic_image_deref_store:
5955       visit_image_store(ctx, instr);
5956       break;
5957    case nir_intrinsic_image_deref_atomic_add:
5958    case nir_intrinsic_image_deref_atomic_umin:
5959    case nir_intrinsic_image_deref_atomic_imin:
5960    case nir_intrinsic_image_deref_atomic_umax:
5961    case nir_intrinsic_image_deref_atomic_imax:
5962    case nir_intrinsic_image_deref_atomic_and:
5963    case nir_intrinsic_image_deref_atomic_or:
5964    case nir_intrinsic_image_deref_atomic_xor:
5965    case nir_intrinsic_image_deref_atomic_exchange:
5966    case nir_intrinsic_image_deref_atomic_comp_swap:
5967       visit_image_atomic(ctx, instr);
5968       break;
5969    case nir_intrinsic_image_deref_size:
5970       visit_image_size(ctx, instr);
5971       break;
5972    case nir_intrinsic_load_ssbo:
5973       visit_load_ssbo(ctx, instr);
5974       break;
5975    case nir_intrinsic_store_ssbo:
5976       visit_store_ssbo(ctx, instr);
5977       break;
5978    case nir_intrinsic_load_global:
5979       visit_load_global(ctx, instr);
5980       break;
5981    case nir_intrinsic_store_global:
5982       visit_store_global(ctx, instr);
5983       break;
5984    case nir_intrinsic_global_atomic_add:
5985    case nir_intrinsic_global_atomic_imin:
5986    case nir_intrinsic_global_atomic_umin:
5987    case nir_intrinsic_global_atomic_imax:
5988    case nir_intrinsic_global_atomic_umax:
5989    case nir_intrinsic_global_atomic_and:
5990    case nir_intrinsic_global_atomic_or:
5991    case nir_intrinsic_global_atomic_xor:
5992    case nir_intrinsic_global_atomic_exchange:
5993    case nir_intrinsic_global_atomic_comp_swap:
5994       visit_global_atomic(ctx, instr);
5995       break;
5996    case nir_intrinsic_ssbo_atomic_add:
5997    case nir_intrinsic_ssbo_atomic_imin:
5998    case nir_intrinsic_ssbo_atomic_umin:
5999    case nir_intrinsic_ssbo_atomic_imax:
6000    case nir_intrinsic_ssbo_atomic_umax:
6001    case nir_intrinsic_ssbo_atomic_and:
6002    case nir_intrinsic_ssbo_atomic_or:
6003    case nir_intrinsic_ssbo_atomic_xor:
6004    case nir_intrinsic_ssbo_atomic_exchange:
6005    case nir_intrinsic_ssbo_atomic_comp_swap:
6006       visit_atomic_ssbo(ctx, instr);
6007       break;
6008    case nir_intrinsic_load_scratch:
6009       visit_load_scratch(ctx, instr);
6010       break;
6011    case nir_intrinsic_store_scratch:
6012       visit_store_scratch(ctx, instr);
6013       break;
6014    case nir_intrinsic_get_buffer_size:
6015       visit_get_buffer_size(ctx, instr);
6016       break;
6017    case nir_intrinsic_control_barrier: {
6018       unsigned* bsize = ctx->program->info->cs.block_size;
6019       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6020       if (workgroup_size > ctx->program->wave_size)
6021          bld.sopp(aco_opcode::s_barrier);
6022       break;
6023    }
6024    case nir_intrinsic_group_memory_barrier:
6025    case nir_intrinsic_memory_barrier:
6026    case nir_intrinsic_memory_barrier_buffer:
6027    case nir_intrinsic_memory_barrier_image:
6028    case nir_intrinsic_memory_barrier_shared:
6029       emit_memory_barrier(ctx, instr);
6030       break;
6031    case nir_intrinsic_memory_barrier_tcs_patch:
6032       break;
6033    case nir_intrinsic_load_num_work_groups: {
6034       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6035       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6036       emit_split_vector(ctx, dst, 3);
6037       break;
6038    }
6039    case nir_intrinsic_load_local_invocation_id: {
6040       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6041       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6042       emit_split_vector(ctx, dst, 3);
6043       break;
6044    }
6045    case nir_intrinsic_load_work_group_id: {
6046       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6047       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6048       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6049                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6050                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6051                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6052       emit_split_vector(ctx, dst, 3);
6053       break;
6054    }
6055    case nir_intrinsic_load_local_invocation_index: {
6056       Temp id = emit_mbcnt(ctx, bld.def(v1));
6057
6058       /* The tg_size bits [6:11] contain the subgroup id,
6059        * we need this multiplied by the wave size, and then OR the thread id to it.
6060        */
6061       if (ctx->program->wave_size == 64) {
6062          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6063          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6064                                 get_arg(ctx, ctx->args->ac.tg_size));
6065          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6066       } else {
6067          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6068          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6069                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6070          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6071       }
6072       break;
6073    }
6074    case nir_intrinsic_load_subgroup_id: {
6075       if (ctx->stage == compute_cs) {
6076          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6077                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6078       } else {
6079          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6080       }
6081       break;
6082    }
6083    case nir_intrinsic_load_subgroup_invocation: {
6084       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6085       break;
6086    }
6087    case nir_intrinsic_load_num_subgroups: {
6088       if (ctx->stage == compute_cs)
6089          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6090                   get_arg(ctx, ctx->args->ac.tg_size));
6091       else
6092          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6093       break;
6094    }
6095    case nir_intrinsic_ballot: {
6096       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6097       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6098       Definition tmp = bld.def(dst.regClass());
6099       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6100       if (instr->src[0].ssa->bit_size == 1) {
6101          assert(src.regClass() == bld.lm);
6102          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6103       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6104          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6105       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6106          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6107       } else {
6108          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6109          nir_print_instr(&instr->instr, stderr);
6110          fprintf(stderr, "\n");
6111       }
6112       if (dst.size() != bld.lm.size()) {
6113          /* Wave32 with ballot size set to 64 */
6114          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6115       }
6116       emit_wqm(ctx, tmp.getTemp(), dst);
6117       break;
6118    }
6119    case nir_intrinsic_shuffle:
6120    case nir_intrinsic_read_invocation: {
6121       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6122       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6123          emit_uniform_subgroup(ctx, instr, src);
6124       } else {
6125          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6126          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6127             tid = bld.as_uniform(tid);
6128          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6129          if (src.regClass() == v1) {
6130             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6131          } else if (src.regClass() == v2) {
6132             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6133             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6134             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6135             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6136             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6137             emit_split_vector(ctx, dst, 2);
6138          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6139             assert(src.regClass() == bld.lm);
6140             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6141             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6142          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6143             assert(src.regClass() == bld.lm);
6144             Temp tmp;
6145             if (ctx->program->chip_class <= GFX7)
6146                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6147             else if (ctx->program->wave_size == 64)
6148                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6149             else
6150                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6151             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6152             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6153             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6154          } else {
6155             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6156             nir_print_instr(&instr->instr, stderr);
6157             fprintf(stderr, "\n");
6158          }
6159       }
6160       break;
6161    }
6162    case nir_intrinsic_load_sample_id: {
6163       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6164                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6165       break;
6166    }
6167    case nir_intrinsic_load_sample_mask_in: {
6168       visit_load_sample_mask_in(ctx, instr);
6169       break;
6170    }
6171    case nir_intrinsic_read_first_invocation: {
6172       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6173       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6174       if (src.regClass() == v1) {
6175          emit_wqm(ctx,
6176                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6177                   dst);
6178       } else if (src.regClass() == v2) {
6179          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6180          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6181          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6182          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6183          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6184          emit_split_vector(ctx, dst, 2);
6185       } else if (instr->dest.ssa.bit_size == 1) {
6186          assert(src.regClass() == bld.lm);
6187          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6188                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6189          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6190       } else if (src.regClass() == s1) {
6191          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6192       } else if (src.regClass() == s2) {
6193          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6194       } else {
6195          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6196          nir_print_instr(&instr->instr, stderr);
6197          fprintf(stderr, "\n");
6198       }
6199       break;
6200    }
6201    case nir_intrinsic_vote_all: {
6202       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6203       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6204       assert(src.regClass() == bld.lm);
6205       assert(dst.regClass() == bld.lm);
6206
6207       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6208       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6209       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6210       break;
6211    }
6212    case nir_intrinsic_vote_any: {
6213       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6214       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6215       assert(src.regClass() == bld.lm);
6216       assert(dst.regClass() == bld.lm);
6217
6218       Temp tmp = bool_to_scalar_condition(ctx, src);
6219       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6220       break;
6221    }
6222    case nir_intrinsic_reduce:
6223    case nir_intrinsic_inclusive_scan:
6224    case nir_intrinsic_exclusive_scan: {
6225       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6226       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6227       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6228       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6229          nir_intrinsic_cluster_size(instr) : 0;
6230       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6231
6232       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6233          emit_uniform_subgroup(ctx, instr, src);
6234       } else if (instr->dest.ssa.bit_size == 1) {
6235          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6236             op = nir_op_iand;
6237          else if (op == nir_op_iadd)
6238             op = nir_op_ixor;
6239          else if (op == nir_op_umax || op == nir_op_imax)
6240             op = nir_op_ior;
6241          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6242
6243          switch (instr->intrinsic) {
6244          case nir_intrinsic_reduce:
6245             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6246             break;
6247          case nir_intrinsic_exclusive_scan:
6248             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6249             break;
6250          case nir_intrinsic_inclusive_scan:
6251             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6252             break;
6253          default:
6254             assert(false);
6255          }
6256       } else if (cluster_size == 1) {
6257          bld.copy(Definition(dst), src);
6258       } else {
6259          src = as_vgpr(ctx, src);
6260
6261          ReduceOp reduce_op;
6262          switch (op) {
6263          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6264             CASE(iadd)
6265             CASE(imul)
6266             CASE(fadd)
6267             CASE(fmul)
6268             CASE(imin)
6269             CASE(umin)
6270             CASE(fmin)
6271             CASE(imax)
6272             CASE(umax)
6273             CASE(fmax)
6274             CASE(iand)
6275             CASE(ior)
6276             CASE(ixor)
6277             default:
6278                unreachable("unknown reduction op");
6279          #undef CASE
6280          }
6281
6282          aco_opcode aco_op;
6283          switch (instr->intrinsic) {
6284             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6285             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6286             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6287             default:
6288                unreachable("unknown reduce intrinsic");
6289          }
6290
6291          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6292          reduce->operands[0] = Operand(src);
6293          // filled in by aco_reduce_assign.cpp, used internally as part of the
6294          // reduce sequence
6295          assert(dst.size() == 1 || dst.size() == 2);
6296          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6297          reduce->operands[2] = Operand(v1.as_linear());
6298
6299          Temp tmp_dst = bld.tmp(dst.regClass());
6300          reduce->definitions[0] = Definition(tmp_dst);
6301          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6302          reduce->definitions[2] = Definition();
6303          reduce->definitions[3] = Definition(scc, s1);
6304          reduce->definitions[4] = Definition();
6305          reduce->reduce_op = reduce_op;
6306          reduce->cluster_size = cluster_size;
6307          ctx->block->instructions.emplace_back(std::move(reduce));
6308
6309          emit_wqm(ctx, tmp_dst, dst);
6310       }
6311       break;
6312    }
6313    case nir_intrinsic_quad_broadcast: {
6314       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6315       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6316          emit_uniform_subgroup(ctx, instr, src);
6317       } else {
6318          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6319          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6320          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6321
6322          if (instr->dest.ssa.bit_size == 1) {
6323             assert(src.regClass() == bld.lm);
6324             assert(dst.regClass() == bld.lm);
6325             uint32_t half_mask = 0x11111111u << lane;
6326             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6327             Temp tmp = bld.tmp(bld.lm);
6328             bld.sop1(Builder::s_wqm, Definition(tmp),
6329                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6330                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6331             emit_wqm(ctx, tmp, dst);
6332          } else if (instr->dest.ssa.bit_size == 32) {
6333             if (ctx->program->chip_class >= GFX8)
6334                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6335             else
6336                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6337          } else if (instr->dest.ssa.bit_size == 64) {
6338             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6339             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6340             if (ctx->program->chip_class >= GFX8) {
6341                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6342                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6343             } else {
6344                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6345                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6346             }
6347             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6348             emit_split_vector(ctx, dst, 2);
6349          } else {
6350             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6351             nir_print_instr(&instr->instr, stderr);
6352             fprintf(stderr, "\n");
6353          }
6354       }
6355       break;
6356    }
6357    case nir_intrinsic_quad_swap_horizontal:
6358    case nir_intrinsic_quad_swap_vertical:
6359    case nir_intrinsic_quad_swap_diagonal:
6360    case nir_intrinsic_quad_swizzle_amd: {
6361       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6362       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6363          emit_uniform_subgroup(ctx, instr, src);
6364          break;
6365       }
6366       uint16_t dpp_ctrl = 0;
6367       switch (instr->intrinsic) {
6368       case nir_intrinsic_quad_swap_horizontal:
6369          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6370          break;
6371       case nir_intrinsic_quad_swap_vertical:
6372          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6373          break;
6374       case nir_intrinsic_quad_swap_diagonal:
6375          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6376          break;
6377       case nir_intrinsic_quad_swizzle_amd:
6378          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6379          break;
6380       default:
6381          break;
6382       }
6383       if (ctx->program->chip_class < GFX8)
6384          dpp_ctrl |= (1 << 15);
6385
6386       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6387       if (instr->dest.ssa.bit_size == 1) {
6388          assert(src.regClass() == bld.lm);
6389          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6390          if (ctx->program->chip_class >= GFX8)
6391             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6392          else
6393             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6394          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6395          emit_wqm(ctx, tmp, dst);
6396       } else if (instr->dest.ssa.bit_size == 32) {
6397          Temp tmp;
6398          if (ctx->program->chip_class >= GFX8)
6399             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6400          else
6401             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6402          emit_wqm(ctx, tmp, dst);
6403       } else if (instr->dest.ssa.bit_size == 64) {
6404          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6405          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6406          if (ctx->program->chip_class >= GFX8) {
6407             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6408             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6409          } else {
6410             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6411             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6412          }
6413          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6414          emit_split_vector(ctx, dst, 2);
6415       } else {
6416          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6417          nir_print_instr(&instr->instr, stderr);
6418          fprintf(stderr, "\n");
6419       }
6420       break;
6421    }
6422    case nir_intrinsic_masked_swizzle_amd: {
6423       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6424       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6425          emit_uniform_subgroup(ctx, instr, src);
6426          break;
6427       }
6428       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6429       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6430       if (dst.regClass() == v1) {
6431          emit_wqm(ctx,
6432                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6433                   dst);
6434       } else if (dst.regClass() == v2) {
6435          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6436          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6437          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6438          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6439          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6440          emit_split_vector(ctx, dst, 2);
6441       } else {
6442          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6443          nir_print_instr(&instr->instr, stderr);
6444          fprintf(stderr, "\n");
6445       }
6446       break;
6447    }
6448    case nir_intrinsic_write_invocation_amd: {
6449       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6450       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6451       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6452       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6453       if (dst.regClass() == v1) {
6454          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6455          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6456       } else if (dst.regClass() == v2) {
6457          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6458          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6459          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6460          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6461          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6462          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6463          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6464          emit_split_vector(ctx, dst, 2);
6465       } else {
6466          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6467          nir_print_instr(&instr->instr, stderr);
6468          fprintf(stderr, "\n");
6469       }
6470       break;
6471    }
6472    case nir_intrinsic_mbcnt_amd: {
6473       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6474       RegClass rc = RegClass(src.type(), 1);
6475       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6476       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6477       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6478       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6479       emit_wqm(ctx, wqm_tmp, dst);
6480       break;
6481    }
6482    case nir_intrinsic_load_helper_invocation: {
6483       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6484       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6485       ctx->block->kind |= block_kind_needs_lowering;
6486       ctx->program->needs_exact = true;
6487       break;
6488    }
6489    case nir_intrinsic_is_helper_invocation: {
6490       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6491       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6492       ctx->block->kind |= block_kind_needs_lowering;
6493       ctx->program->needs_exact = true;
6494       break;
6495    }
6496    case nir_intrinsic_demote:
6497       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6498
6499       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6500          ctx->cf_info.exec_potentially_empty = true;
6501       ctx->block->kind |= block_kind_uses_demote;
6502       ctx->program->needs_exact = true;
6503       break;
6504    case nir_intrinsic_demote_if: {
6505       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6506       assert(src.regClass() == bld.lm);
6507       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6508       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6509
6510       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6511          ctx->cf_info.exec_potentially_empty = true;
6512       ctx->block->kind |= block_kind_uses_demote;
6513       ctx->program->needs_exact = true;
6514       break;
6515    }
6516    case nir_intrinsic_first_invocation: {
6517       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6518                get_ssa_temp(ctx, &instr->dest.ssa));
6519       break;
6520    }
6521    case nir_intrinsic_shader_clock:
6522       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6523       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6524       break;
6525    case nir_intrinsic_load_vertex_id_zero_base: {
6526       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6527       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6528       break;
6529    }
6530    case nir_intrinsic_load_first_vertex: {
6531       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6532       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6533       break;
6534    }
6535    case nir_intrinsic_load_base_instance: {
6536       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6537       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6538       break;
6539    }
6540    case nir_intrinsic_load_instance_id: {
6541       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6542       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6543       break;
6544    }
6545    case nir_intrinsic_load_draw_id: {
6546       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6547       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6548       break;
6549    }
6550    default:
6551       fprintf(stderr, "Unimplemented intrinsic instr: ");
6552       nir_print_instr(&instr->instr, stderr);
6553       fprintf(stderr, "\n");
6554       abort();
6555
6556       break;
6557    }
6558 }
6559
6560
6561 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6562                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6563                     enum glsl_base_type *stype)
6564 {
6565    nir_deref_instr *texture_deref_instr = NULL;
6566    nir_deref_instr *sampler_deref_instr = NULL;
6567    int plane = -1;
6568
6569    for (unsigned i = 0; i < instr->num_srcs; i++) {
6570       switch (instr->src[i].src_type) {
6571       case nir_tex_src_texture_deref:
6572          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6573          break;
6574       case nir_tex_src_sampler_deref:
6575          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6576          break;
6577       case nir_tex_src_plane:
6578          plane = nir_src_as_int(instr->src[i].src);
6579          break;
6580       default:
6581          break;
6582       }
6583    }
6584
6585    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6586
6587    if (!sampler_deref_instr)
6588       sampler_deref_instr = texture_deref_instr;
6589
6590    if (plane >= 0) {
6591       assert(instr->op != nir_texop_txf_ms &&
6592              instr->op != nir_texop_samples_identical);
6593       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6594       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6595    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6596       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6597    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6598       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6599    } else {
6600       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6601    }
6602    if (samp_ptr) {
6603       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6604
6605       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6606          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6607          Builder bld(ctx->program, ctx->block);
6608
6609          /* to avoid unnecessary moves, we split and recombine sampler and image */
6610          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6611                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6612          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6613          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6614                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6615                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6616          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6617                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6618
6619          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6620          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6621                                img[0], img[1], img[2], img[3],
6622                                img[4], img[5], img[6], img[7]);
6623          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6624                                 samp[0], samp[1], samp[2], samp[3]);
6625       }
6626    }
6627    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6628                      instr->op == nir_texop_samples_identical))
6629       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6630 }
6631
6632 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6633                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6634 {
6635    Builder bld(ctx->program, ctx->block);
6636
6637    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6638    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6639    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6640
6641    Operand neg_one(0xbf800000u);
6642    Operand one(0x3f800000u);
6643    Operand two(0x40000000u);
6644    Operand four(0x40800000u);
6645
6646    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6647    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6648    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6649
6650    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6651    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6652    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6653    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6654
6655    // select sc
6656    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6657    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6658                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6659                        one, is_ma_y);
6660    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6661
6662    // select tc
6663    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6664    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6665    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6666
6667    // select ma
6668    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6669                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6670                   deriv_z, is_ma_z);
6671    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6672    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6673 }
6674
6675 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6676 {
6677    Builder bld(ctx->program, ctx->block);
6678    Temp coord_args[4], ma, tc, sc, id;
6679    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6680       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6681
6682    if (is_array) {
6683       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6684
6685       // see comment in ac_prepare_cube_coords()
6686       if (ctx->options->chip_class <= GFX8)
6687          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6688    }
6689
6690    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6691
6692    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6693    vop3a->operands[0] = Operand(ma);
6694    vop3a->abs[0] = true;
6695    Temp invma = bld.tmp(v1);
6696    vop3a->definitions[0] = Definition(invma);
6697    ctx->block->instructions.emplace_back(std::move(vop3a));
6698
6699    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6700    if (!is_deriv)
6701       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6702
6703    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6704    if (!is_deriv)
6705       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6706
6707    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6708
6709    if (is_deriv) {
6710       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6711       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6712
6713       for (unsigned i = 0; i < 2; i++) {
6714          // see comment in ac_prepare_cube_coords()
6715          Temp deriv_ma;
6716          Temp deriv_sc, deriv_tc;
6717          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6718                            &deriv_ma, &deriv_sc, &deriv_tc);
6719
6720          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6721
6722          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6723                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6724                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6725          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6726                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6727                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6728          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6729       }
6730
6731       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6732       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6733    }
6734
6735    if (is_array)
6736       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6737    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6738
6739 }
6740
6741 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6742 {
6743    Temp coord_vec[3];
6744    for (unsigned i = 0; i < coords.size(); i++)
6745       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6746
6747    Builder bld(ctx->program, ctx->block);
6748    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6749
6750    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6751    for (unsigned i = 0; i < coords.size(); i++)
6752       vec->operands[i] = Operand(coord_vec[i]);
6753    Temp res = bld.tmp(RegType::vgpr, coords.size());
6754    vec->definitions[0] = Definition(res);
6755    ctx->block->instructions.emplace_back(std::move(vec));
6756    return res;
6757 }
6758
6759 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6760 {
6761    if (vec->parent_instr->type != nir_instr_type_alu)
6762       return;
6763    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6764    if (vec_instr->op != nir_op_vec(vec->num_components))
6765       return;
6766
6767    for (unsigned i = 0; i < vec->num_components; i++) {
6768       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6769               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6770    }
6771 }
6772
6773 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6774 {
6775    Builder bld(ctx->program, ctx->block);
6776    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6777         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6778    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6779         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6780    nir_const_value *sample_index_cv = NULL;
6781    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6782    enum glsl_base_type stype;
6783    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6784
6785    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6786                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6787    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6788                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6789
6790    for (unsigned i = 0; i < instr->num_srcs; i++) {
6791       switch (instr->src[i].src_type) {
6792       case nir_tex_src_coord:
6793          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6794          break;
6795       case nir_tex_src_bias:
6796          if (instr->op == nir_texop_txb) {
6797             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6798             has_bias = true;
6799          }
6800          break;
6801       case nir_tex_src_lod: {
6802          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6803
6804          if (val && val->f32 <= 0.0) {
6805             level_zero = true;
6806          } else {
6807             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6808             has_lod = true;
6809          }
6810          break;
6811       }
6812       case nir_tex_src_comparator:
6813          if (instr->is_shadow) {
6814             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6815             has_compare = true;
6816          }
6817          break;
6818       case nir_tex_src_offset:
6819          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6820          get_const_vec(instr->src[i].src.ssa, const_offset);
6821          has_offset = true;
6822          break;
6823       case nir_tex_src_ddx:
6824          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6825          has_ddx = true;
6826          break;
6827       case nir_tex_src_ddy:
6828          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6829          has_ddy = true;
6830          break;
6831       case nir_tex_src_ms_index:
6832          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6833          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6834          has_sample_index = true;
6835          break;
6836       case nir_tex_src_texture_offset:
6837       case nir_tex_src_sampler_offset:
6838       default:
6839          break;
6840       }
6841    }
6842 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6843    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6844       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6845
6846    if (instr->op == nir_texop_texture_samples) {
6847       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6848
6849       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6850       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6851       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6852       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6853
6854       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6855                samples, Operand(1u), bld.scc(is_msaa));
6856       return;
6857    }
6858
6859    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6860       aco_ptr<Instruction> tmp_instr;
6861       Temp acc, pack = Temp();
6862
6863       uint32_t pack_const = 0;
6864       for (unsigned i = 0; i < offset.size(); i++) {
6865          if (!const_offset[i])
6866             continue;
6867          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6868       }
6869
6870       if (offset.type() == RegType::sgpr) {
6871          for (unsigned i = 0; i < offset.size(); i++) {
6872             if (const_offset[i])
6873                continue;
6874
6875             acc = emit_extract_vector(ctx, offset, i, s1);
6876             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6877
6878             if (i) {
6879                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6880             }
6881
6882             if (pack == Temp()) {
6883                pack = acc;
6884             } else {
6885                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6886             }
6887          }
6888
6889          if (pack_const && pack != Temp())
6890             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6891       } else {
6892          for (unsigned i = 0; i < offset.size(); i++) {
6893             if (const_offset[i])
6894                continue;
6895
6896             acc = emit_extract_vector(ctx, offset, i, v1);
6897             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6898
6899             if (i) {
6900                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6901             }
6902
6903             if (pack == Temp()) {
6904                pack = acc;
6905             } else {
6906                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6907             }
6908          }
6909
6910          if (pack_const && pack != Temp())
6911             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6912       }
6913       if (pack_const && pack == Temp())
6914          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6915       else if (pack == Temp())
6916          has_offset = false;
6917       else
6918          offset = pack;
6919    }
6920
6921    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6922       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6923
6924    /* pack derivatives */
6925    if (has_ddx || has_ddy) {
6926       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6927          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6928                              ddx, Operand(0u), ddy, Operand(0u));
6929       } else {
6930          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6931       }
6932       has_derivs = true;
6933    }
6934
6935    if (instr->coord_components > 1 &&
6936        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6937        instr->is_array &&
6938        instr->op != nir_texop_txf)
6939       coords = apply_round_slice(ctx, coords, 1);
6940
6941    if (instr->coord_components > 2 &&
6942       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6943        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6944        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6945        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6946        instr->is_array &&
6947        instr->op != nir_texop_txf &&
6948        instr->op != nir_texop_txf_ms &&
6949        instr->op != nir_texop_fragment_fetch &&
6950        instr->op != nir_texop_fragment_mask_fetch)
6951       coords = apply_round_slice(ctx, coords, 2);
6952
6953    if (ctx->options->chip_class == GFX9 &&
6954        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6955        instr->op != nir_texop_lod && instr->coord_components) {
6956       assert(coords.size() > 0 && coords.size() < 3);
6957
6958       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6959       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6960       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6961       if (coords.size() > 1)
6962          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6963       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6964       vec->definitions[0] = Definition(coords);
6965       ctx->block->instructions.emplace_back(std::move(vec));
6966    }
6967
6968    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6969
6970    if (instr->op == nir_texop_samples_identical)
6971       resource = fmask_ptr;
6972
6973    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6974              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6975             instr->op != nir_texop_txs &&
6976             instr->op != nir_texop_fragment_fetch &&
6977             instr->op != nir_texop_fragment_mask_fetch) {
6978       assert(has_sample_index);
6979       Operand op(sample_index);
6980       if (sample_index_cv)
6981          op = Operand(sample_index_cv->u32);
6982       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6983    }
6984
6985    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6986       Temp split_coords[coords.size()];
6987       emit_split_vector(ctx, coords, coords.size());
6988       for (unsigned i = 0; i < coords.size(); i++)
6989          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6990
6991       unsigned i = 0;
6992       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6993          Temp off = emit_extract_vector(ctx, offset, i, v1);
6994          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6995       }
6996
6997       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6998       for (unsigned i = 0; i < coords.size(); i++)
6999          vec->operands[i] = Operand(split_coords[i]);
7000       coords = bld.tmp(coords.regClass());
7001       vec->definitions[0] = Definition(coords);
7002       ctx->block->instructions.emplace_back(std::move(vec));
7003
7004       has_offset = false;
7005    }
7006
7007    /* Build tex instruction */
7008    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7009    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7010                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7011                   : 0;
7012    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7013    Temp tmp_dst = dst;
7014
7015    /* gather4 selects the component by dmask and always returns vec4 */
7016    if (instr->op == nir_texop_tg4) {
7017       assert(instr->dest.ssa.num_components == 4);
7018       if (instr->is_shadow)
7019          dmask = 1;
7020       else
7021          dmask = 1 << instr->component;
7022       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7023          tmp_dst = bld.tmp(v4);
7024    } else if (instr->op == nir_texop_samples_identical) {
7025       tmp_dst = bld.tmp(v1);
7026    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7027       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7028    }
7029
7030    aco_ptr<MIMG_instruction> tex;
7031    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7032       if (!has_lod)
7033          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7034
7035       bool div_by_6 = instr->op == nir_texop_txs &&
7036                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7037                       instr->is_array &&
7038                       (dmask & (1 << 2));
7039       if (tmp_dst.id() == dst.id() && div_by_6)
7040          tmp_dst = bld.tmp(tmp_dst.regClass());
7041
7042       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7043       tex->operands[0] = Operand(as_vgpr(ctx,lod));
7044       tex->operands[1] = Operand(resource);
7045       if (ctx->options->chip_class == GFX9 &&
7046           instr->op == nir_texop_txs &&
7047           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7048           instr->is_array) {
7049          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7050       } else if (instr->op == nir_texop_query_levels) {
7051          tex->dmask = 1 << 3;
7052       } else {
7053          tex->dmask = dmask;
7054       }
7055       tex->da = da;
7056       tex->definitions[0] = Definition(tmp_dst);
7057       tex->dim = dim;
7058       tex->can_reorder = true;
7059       ctx->block->instructions.emplace_back(std::move(tex));
7060
7061       if (div_by_6) {
7062          /* divide 3rd value by 6 by multiplying with magic number */
7063          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7064          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7065          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7066          assert(instr->dest.ssa.num_components == 3);
7067          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7068          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7069                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7070                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7071                               by_6);
7072
7073       }
7074
7075       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7076       return;
7077    }
7078
7079    Temp tg4_compare_cube_wa64 = Temp();
7080
7081    if (tg4_integer_workarounds) {
7082       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7083       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7084       tex->operands[1] = Operand(resource);
7085       tex->dim = dim;
7086       tex->dmask = 0x3;
7087       tex->da = da;
7088       Temp size = bld.tmp(v2);
7089       tex->definitions[0] = Definition(size);
7090       tex->can_reorder = true;
7091       ctx->block->instructions.emplace_back(std::move(tex));
7092       emit_split_vector(ctx, size, size.size());
7093
7094       Temp half_texel[2];
7095       for (unsigned i = 0; i < 2; i++) {
7096          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7097          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7098          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7099          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7100       }
7101
7102       Temp orig_coords[2] = {
7103          emit_extract_vector(ctx, coords, 0, v1),
7104          emit_extract_vector(ctx, coords, 1, v1)};
7105       Temp new_coords[2] = {
7106          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
7107          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
7108       };
7109
7110       if (tg4_integer_cube_workaround) {
7111          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7112          Temp desc[resource.size()];
7113          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7114                                                                            Format::PSEUDO, 1, resource.size())};
7115          split->operands[0] = Operand(resource);
7116          for (unsigned i = 0; i < resource.size(); i++) {
7117             desc[i] = bld.tmp(s1);
7118             split->definitions[i] = Definition(desc[i]);
7119          }
7120          ctx->block->instructions.emplace_back(std::move(split));
7121
7122          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7123          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7124                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7125
7126          Temp nfmt;
7127          if (stype == GLSL_TYPE_UINT) {
7128             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7129                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7130                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7131                             bld.scc(compare_cube_wa));
7132          } else {
7133             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7134                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7135                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7136                             bld.scc(compare_cube_wa));
7137          }
7138          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7139          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7140
7141          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7142
7143          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7144                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7145          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7146
7147          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7148                                                                          Format::PSEUDO, resource.size(), 1)};
7149          for (unsigned i = 0; i < resource.size(); i++)
7150             vec->operands[i] = Operand(desc[i]);
7151          resource = bld.tmp(resource.regClass());
7152          vec->definitions[0] = Definition(resource);
7153          ctx->block->instructions.emplace_back(std::move(vec));
7154
7155          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7156                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
7157          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7158                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
7159       }
7160
7161       if (coords.size() == 3) {
7162          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
7163                              new_coords[0], new_coords[1],
7164                              emit_extract_vector(ctx, coords, 2, v1));
7165       } else {
7166          assert(coords.size() == 2);
7167          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
7168                              new_coords[0], new_coords[1]);
7169       }
7170    }
7171
7172    std::vector<Operand> args;
7173    if (has_offset)
7174       args.emplace_back(Operand(offset));
7175    if (has_bias)
7176       args.emplace_back(Operand(bias));
7177    if (has_compare)
7178       args.emplace_back(Operand(compare));
7179    if (has_derivs)
7180       args.emplace_back(Operand(derivs));
7181    args.emplace_back(Operand(coords));
7182    if (has_sample_index)
7183       args.emplace_back(Operand(sample_index));
7184    if (has_lod)
7185       args.emplace_back(lod);
7186
7187    Temp arg;
7188    if (args.size() > 1) {
7189       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7190       unsigned size = 0;
7191       for (unsigned i = 0; i < args.size(); i++) {
7192          size += args[i].size();
7193          vec->operands[i] = args[i];
7194       }
7195       RegClass rc = RegClass(RegType::vgpr, size);
7196       Temp tmp = bld.tmp(rc);
7197       vec->definitions[0] = Definition(tmp);
7198       ctx->block->instructions.emplace_back(std::move(vec));
7199       arg = tmp;
7200    } else {
7201       assert(args[0].isTemp());
7202       arg = as_vgpr(ctx, args[0].getTemp());
7203    }
7204
7205    /* we don't need the bias, sample index, compare value or offset to be
7206     * computed in WQM but if the p_create_vector copies the coordinates, then it
7207     * needs to be in WQM */
7208    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
7209        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7210        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7211       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7212
7213    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7214       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7215
7216       assert(coords.size() == 1);
7217       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7218       aco_opcode op;
7219       switch (last_bit) {
7220       case 1:
7221          op = aco_opcode::buffer_load_format_x; break;
7222       case 2:
7223          op = aco_opcode::buffer_load_format_xy; break;
7224       case 3:
7225          op = aco_opcode::buffer_load_format_xyz; break;
7226       case 4:
7227          op = aco_opcode::buffer_load_format_xyzw; break;
7228       default:
7229          unreachable("Tex instruction loads more than 4 components.");
7230       }
7231
7232       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7233       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7234          tmp_dst = dst;
7235       else
7236          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7237
7238       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7239       mubuf->operands[0] = Operand(coords);
7240       mubuf->operands[1] = Operand(resource);
7241       mubuf->operands[2] = Operand((uint32_t) 0);
7242       mubuf->definitions[0] = Definition(tmp_dst);
7243       mubuf->idxen = true;
7244       mubuf->can_reorder = true;
7245       ctx->block->instructions.emplace_back(std::move(mubuf));
7246
7247       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7248       return;
7249    }
7250
7251
7252    if (instr->op == nir_texop_txf ||
7253        instr->op == nir_texop_txf_ms ||
7254        instr->op == nir_texop_samples_identical ||
7255        instr->op == nir_texop_fragment_fetch ||
7256        instr->op == nir_texop_fragment_mask_fetch) {
7257       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7258       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
7259       tex->operands[0] = Operand(arg);
7260       tex->operands[1] = Operand(resource);
7261       tex->dim = dim;
7262       tex->dmask = dmask;
7263       tex->unrm = true;
7264       tex->da = da;
7265       tex->definitions[0] = Definition(tmp_dst);
7266       tex->can_reorder = true;
7267       ctx->block->instructions.emplace_back(std::move(tex));
7268
7269       if (instr->op == nir_texop_samples_identical) {
7270          assert(dmask == 1 && dst.regClass() == v1);
7271          assert(dst.id() != tmp_dst.id());
7272
7273          Temp tmp = bld.tmp(bld.lm);
7274          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7275          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7276
7277       } else {
7278          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7279       }
7280       return;
7281    }
7282
7283    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7284    aco_opcode opcode = aco_opcode::image_sample;
7285    if (has_offset) { /* image_sample_*_o */
7286       if (has_compare) {
7287          opcode = aco_opcode::image_sample_c_o;
7288          if (has_derivs)
7289             opcode = aco_opcode::image_sample_c_d_o;
7290          if (has_bias)
7291             opcode = aco_opcode::image_sample_c_b_o;
7292          if (level_zero)
7293             opcode = aco_opcode::image_sample_c_lz_o;
7294          if (has_lod)
7295             opcode = aco_opcode::image_sample_c_l_o;
7296       } else {
7297          opcode = aco_opcode::image_sample_o;
7298          if (has_derivs)
7299             opcode = aco_opcode::image_sample_d_o;
7300          if (has_bias)
7301             opcode = aco_opcode::image_sample_b_o;
7302          if (level_zero)
7303             opcode = aco_opcode::image_sample_lz_o;
7304          if (has_lod)
7305             opcode = aco_opcode::image_sample_l_o;
7306       }
7307    } else { /* no offset */
7308       if (has_compare) {
7309          opcode = aco_opcode::image_sample_c;
7310          if (has_derivs)
7311             opcode = aco_opcode::image_sample_c_d;
7312          if (has_bias)
7313             opcode = aco_opcode::image_sample_c_b;
7314          if (level_zero)
7315             opcode = aco_opcode::image_sample_c_lz;
7316          if (has_lod)
7317             opcode = aco_opcode::image_sample_c_l;
7318       } else {
7319          opcode = aco_opcode::image_sample;
7320          if (has_derivs)
7321             opcode = aco_opcode::image_sample_d;
7322          if (has_bias)
7323             opcode = aco_opcode::image_sample_b;
7324          if (level_zero)
7325             opcode = aco_opcode::image_sample_lz;
7326          if (has_lod)
7327             opcode = aco_opcode::image_sample_l;
7328       }
7329    }
7330
7331    if (instr->op == nir_texop_tg4) {
7332       if (has_offset) {
7333          opcode = aco_opcode::image_gather4_lz_o;
7334          if (has_compare)
7335             opcode = aco_opcode::image_gather4_c_lz_o;
7336       } else {
7337          opcode = aco_opcode::image_gather4_lz;
7338          if (has_compare)
7339             opcode = aco_opcode::image_gather4_c_lz;
7340       }
7341    } else if (instr->op == nir_texop_lod) {
7342       opcode = aco_opcode::image_get_lod;
7343    }
7344
7345    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7346    tex->operands[0] = Operand(arg);
7347    tex->operands[1] = Operand(resource);
7348    tex->operands[2] = Operand(sampler);
7349    tex->dim = dim;
7350    tex->dmask = dmask;
7351    tex->da = da;
7352    tex->definitions[0] = Definition(tmp_dst);
7353    tex->can_reorder = true;
7354    ctx->block->instructions.emplace_back(std::move(tex));
7355
7356    if (tg4_integer_cube_workaround) {
7357       assert(tmp_dst.id() != dst.id());
7358       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7359
7360       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7361       Temp val[4];
7362       for (unsigned i = 0; i < dst.size(); i++) {
7363          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7364          Temp cvt_val;
7365          if (stype == GLSL_TYPE_UINT)
7366             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7367          else
7368             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7369          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7370       }
7371       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7372       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7373                            val[0], val[1], val[2], val[3]);
7374    }
7375    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7376    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7377
7378 }
7379
7380
7381 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7382 {
7383    Temp tmp = get_ssa_temp(ctx, ssa);
7384    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7385       return Operand(tmp.regClass());
7386    else
7387       return Operand(tmp);
7388 }
7389
7390 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7391 {
7392    aco_ptr<Pseudo_instruction> phi;
7393    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7394    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7395
7396    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7397    logical |= ctx->block->kind & block_kind_merge;
7398    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7399
7400    /* we want a sorted list of sources, since the predecessor list is also sorted */
7401    std::map<unsigned, nir_ssa_def*> phi_src;
7402    nir_foreach_phi_src(src, instr)
7403       phi_src[src->pred->index] = src->src.ssa;
7404
7405    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7406    unsigned num_operands = 0;
7407    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7408    unsigned num_defined = 0;
7409    unsigned cur_pred_idx = 0;
7410    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7411       if (cur_pred_idx < preds.size()) {
7412          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7413          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7414          unsigned skipped = 0;
7415          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7416             skipped++;
7417          if (cur_pred_idx + skipped < preds.size()) {
7418             for (unsigned i = 0; i < skipped; i++)
7419                operands[num_operands++] = Operand(dst.regClass());
7420             cur_pred_idx += skipped;
7421          } else {
7422             continue;
7423          }
7424       }
7425       cur_pred_idx++;
7426       Operand op = get_phi_operand(ctx, src.second);
7427       operands[num_operands++] = op;
7428       num_defined += !op.isUndefined();
7429    }
7430    /* handle block_kind_continue_or_break at loop exit blocks */
7431    while (cur_pred_idx++ < preds.size())
7432       operands[num_operands++] = Operand(dst.regClass());
7433
7434    if (num_defined == 0) {
7435       Builder bld(ctx->program, ctx->block);
7436       if (dst.regClass() == s1) {
7437          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7438       } else if (dst.regClass() == v1) {
7439          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7440       } else {
7441          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7442          for (unsigned i = 0; i < dst.size(); i++)
7443             vec->operands[i] = Operand(0u);
7444          vec->definitions[0] = Definition(dst);
7445          ctx->block->instructions.emplace_back(std::move(vec));
7446       }
7447       return;
7448    }
7449
7450    /* we can use a linear phi in some cases if one src is undef */
7451    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7452       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7453
7454       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7455       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7456       assert(invert->kind & block_kind_invert);
7457
7458       unsigned then_block = invert->linear_preds[0];
7459
7460       Block* insert_block = NULL;
7461       for (unsigned i = 0; i < num_operands; i++) {
7462          Operand op = operands[i];
7463          if (op.isUndefined())
7464             continue;
7465          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7466          phi->operands[0] = op;
7467          break;
7468       }
7469       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7470       phi->operands[1] = Operand(dst.regClass());
7471       phi->definitions[0] = Definition(dst);
7472       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7473       return;
7474    }
7475
7476    /* try to scalarize vector phis */
7477    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7478       // TODO: scalarize linear phis on divergent ifs
7479       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7480       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7481       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7482          Operand src = operands[i];
7483          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7484             can_scalarize = false;
7485       }
7486       if (can_scalarize) {
7487          unsigned num_components = instr->dest.ssa.num_components;
7488          assert(dst.size() % num_components == 0);
7489          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7490
7491          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7492          for (unsigned k = 0; k < num_components; k++) {
7493             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7494             for (unsigned i = 0; i < num_operands; i++) {
7495                Operand src = operands[i];
7496                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7497             }
7498             Temp phi_dst = {ctx->program->allocateId(), rc};
7499             phi->definitions[0] = Definition(phi_dst);
7500             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7501             new_vec[k] = phi_dst;
7502             vec->operands[k] = Operand(phi_dst);
7503          }
7504          vec->definitions[0] = Definition(dst);
7505          ctx->block->instructions.emplace_back(std::move(vec));
7506          ctx->allocated_vec.emplace(dst.id(), new_vec);
7507          return;
7508       }
7509    }
7510
7511    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7512    for (unsigned i = 0; i < num_operands; i++)
7513       phi->operands[i] = operands[i];
7514    phi->definitions[0] = Definition(dst);
7515    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7516 }
7517
7518
7519 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7520 {
7521    Temp dst = get_ssa_temp(ctx, &instr->def);
7522
7523    assert(dst.type() == RegType::sgpr);
7524
7525    if (dst.size() == 1) {
7526       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7527    } else {
7528       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7529       for (unsigned i = 0; i < dst.size(); i++)
7530          vec->operands[i] = Operand(0u);
7531       vec->definitions[0] = Definition(dst);
7532       ctx->block->instructions.emplace_back(std::move(vec));
7533    }
7534 }
7535
7536 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7537 {
7538    Builder bld(ctx->program, ctx->block);
7539    Block *logical_target;
7540    append_logical_end(ctx->block);
7541    unsigned idx = ctx->block->index;
7542
7543    switch (instr->type) {
7544    case nir_jump_break:
7545       logical_target = ctx->cf_info.parent_loop.exit;
7546       add_logical_edge(idx, logical_target);
7547       ctx->block->kind |= block_kind_break;
7548
7549       if (!ctx->cf_info.parent_if.is_divergent &&
7550           !ctx->cf_info.parent_loop.has_divergent_continue) {
7551          /* uniform break - directly jump out of the loop */
7552          ctx->block->kind |= block_kind_uniform;
7553          ctx->cf_info.has_branch = true;
7554          bld.branch(aco_opcode::p_branch);
7555          add_linear_edge(idx, logical_target);
7556          return;
7557       }
7558       ctx->cf_info.parent_loop.has_divergent_branch = true;
7559       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7560       break;
7561    case nir_jump_continue:
7562       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7563       add_logical_edge(idx, logical_target);
7564       ctx->block->kind |= block_kind_continue;
7565
7566       if (ctx->cf_info.parent_if.is_divergent) {
7567          /* for potential uniform breaks after this continue,
7568             we must ensure that they are handled correctly */
7569          ctx->cf_info.parent_loop.has_divergent_continue = true;
7570          ctx->cf_info.parent_loop.has_divergent_branch = true;
7571          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7572       } else {
7573          /* uniform continue - directly jump to the loop header */
7574          ctx->block->kind |= block_kind_uniform;
7575          ctx->cf_info.has_branch = true;
7576          bld.branch(aco_opcode::p_branch);
7577          add_linear_edge(idx, logical_target);
7578          return;
7579       }
7580       break;
7581    default:
7582       fprintf(stderr, "Unknown NIR jump instr: ");
7583       nir_print_instr(&instr->instr, stderr);
7584       fprintf(stderr, "\n");
7585       abort();
7586    }
7587
7588    /* remove critical edges from linear CFG */
7589    bld.branch(aco_opcode::p_branch);
7590    Block* break_block = ctx->program->create_and_insert_block();
7591    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7592    break_block->kind |= block_kind_uniform;
7593    add_linear_edge(idx, break_block);
7594    /* the loop_header pointer might be invalidated by this point */
7595    if (instr->type == nir_jump_continue)
7596       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7597    add_linear_edge(break_block->index, logical_target);
7598    bld.reset(break_block);
7599    bld.branch(aco_opcode::p_branch);
7600
7601    Block* continue_block = ctx->program->create_and_insert_block();
7602    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7603    add_linear_edge(idx, continue_block);
7604    append_logical_start(continue_block);
7605    ctx->block = continue_block;
7606    return;
7607 }
7608
7609 void visit_block(isel_context *ctx, nir_block *block)
7610 {
7611    nir_foreach_instr(instr, block) {
7612       switch (instr->type) {
7613       case nir_instr_type_alu:
7614          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7615          break;
7616       case nir_instr_type_load_const:
7617          visit_load_const(ctx, nir_instr_as_load_const(instr));
7618          break;
7619       case nir_instr_type_intrinsic:
7620          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7621          break;
7622       case nir_instr_type_tex:
7623          visit_tex(ctx, nir_instr_as_tex(instr));
7624          break;
7625       case nir_instr_type_phi:
7626          visit_phi(ctx, nir_instr_as_phi(instr));
7627          break;
7628       case nir_instr_type_ssa_undef:
7629          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7630          break;
7631       case nir_instr_type_deref:
7632          break;
7633       case nir_instr_type_jump:
7634          visit_jump(ctx, nir_instr_as_jump(instr));
7635          break;
7636       default:
7637          fprintf(stderr, "Unknown NIR instr type: ");
7638          nir_print_instr(instr, stderr);
7639          fprintf(stderr, "\n");
7640          //abort();
7641       }
7642    }
7643
7644    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7645       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7646 }
7647
7648
7649
7650 static void visit_loop(isel_context *ctx, nir_loop *loop)
7651 {
7652    append_logical_end(ctx->block);
7653    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7654    Builder bld(ctx->program, ctx->block);
7655    bld.branch(aco_opcode::p_branch);
7656    unsigned loop_preheader_idx = ctx->block->index;
7657
7658    Block loop_exit = Block();
7659    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7660    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7661
7662    Block* loop_header = ctx->program->create_and_insert_block();
7663    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7664    loop_header->kind |= block_kind_loop_header;
7665    add_edge(loop_preheader_idx, loop_header);
7666    ctx->block = loop_header;
7667
7668    /* emit loop body */
7669    unsigned loop_header_idx = loop_header->index;
7670    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7671    append_logical_start(ctx->block);
7672    visit_cf_list(ctx, &loop->body);
7673
7674    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7675    if (!ctx->cf_info.has_branch) {
7676       append_logical_end(ctx->block);
7677       if (ctx->cf_info.exec_potentially_empty) {
7678          /* Discards can result in code running with an empty exec mask.
7679           * This would result in divergent breaks not ever being taken. As a
7680           * workaround, break the loop when the loop mask is empty instead of
7681           * always continuing. */
7682          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7683          unsigned block_idx = ctx->block->index;
7684
7685          /* create helper blocks to avoid critical edges */
7686          Block *break_block = ctx->program->create_and_insert_block();
7687          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7688          break_block->kind = block_kind_uniform;
7689          bld.reset(break_block);
7690          bld.branch(aco_opcode::p_branch);
7691          add_linear_edge(block_idx, break_block);
7692          add_linear_edge(break_block->index, &loop_exit);
7693
7694          Block *continue_block = ctx->program->create_and_insert_block();
7695          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7696          continue_block->kind = block_kind_uniform;
7697          bld.reset(continue_block);
7698          bld.branch(aco_opcode::p_branch);
7699          add_linear_edge(block_idx, continue_block);
7700          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7701
7702          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7703          ctx->block = &ctx->program->blocks[block_idx];
7704       } else {
7705          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7706          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7707             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7708          else
7709             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7710       }
7711
7712       bld.reset(ctx->block);
7713       bld.branch(aco_opcode::p_branch);
7714    }
7715
7716    /* fixup phis in loop header from unreachable blocks */
7717    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7718       bool linear = ctx->cf_info.has_branch;
7719       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7720       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7721          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7722              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7723             /* the last operand should be the one that needs to be removed */
7724             instr->operands.pop_back();
7725          } else if (!is_phi(instr)) {
7726             break;
7727          }
7728       }
7729    }
7730
7731    ctx->cf_info.has_branch = false;
7732
7733    // TODO: if the loop has not a single exit, we must add one °°
7734    /* emit loop successor block */
7735    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7736    append_logical_start(ctx->block);
7737
7738    #if 0
7739    // TODO: check if it is beneficial to not branch on continues
7740    /* trim linear phis in loop header */
7741    for (auto&& instr : loop_entry->instructions) {
7742       if (instr->opcode == aco_opcode::p_linear_phi) {
7743          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7744          new_phi->definitions[0] = instr->definitions[0];
7745          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7746             new_phi->operands[i] = instr->operands[i];
7747          /* check that the remaining operands are all the same */
7748          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7749             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7750          instr.swap(new_phi);
7751       } else if (instr->opcode == aco_opcode::p_phi) {
7752          continue;
7753       } else {
7754          break;
7755       }
7756    }
7757    #endif
7758 }
7759
7760 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7761 {
7762    ic->cond = cond;
7763
7764    append_logical_end(ctx->block);
7765    ctx->block->kind |= block_kind_branch;
7766
7767    /* branch to linear then block */
7768    assert(cond.regClass() == ctx->program->lane_mask);
7769    aco_ptr<Pseudo_branch_instruction> branch;
7770    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7771    branch->operands[0] = Operand(cond);
7772    ctx->block->instructions.push_back(std::move(branch));
7773
7774    ic->BB_if_idx = ctx->block->index;
7775    ic->BB_invert = Block();
7776    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7777    /* Invert blocks are intentionally not marked as top level because they
7778     * are not part of the logical cfg. */
7779    ic->BB_invert.kind |= block_kind_invert;
7780    ic->BB_endif = Block();
7781    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7782    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7783
7784    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7785    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7786    ctx->cf_info.parent_if.is_divergent = true;
7787    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7788
7789    /** emit logical then block */
7790    Block* BB_then_logical = ctx->program->create_and_insert_block();
7791    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7792    add_edge(ic->BB_if_idx, BB_then_logical);
7793    ctx->block = BB_then_logical;
7794    append_logical_start(BB_then_logical);
7795 }
7796
7797 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7798 {
7799    Block *BB_then_logical = ctx->block;
7800    append_logical_end(BB_then_logical);
7801     /* branch from logical then block to invert block */
7802    aco_ptr<Pseudo_branch_instruction> branch;
7803    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7804    BB_then_logical->instructions.emplace_back(std::move(branch));
7805    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7806    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7807       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7808    BB_then_logical->kind |= block_kind_uniform;
7809    assert(!ctx->cf_info.has_branch);
7810    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7811    ctx->cf_info.parent_loop.has_divergent_branch = false;
7812
7813    /** emit linear then block */
7814    Block* BB_then_linear = ctx->program->create_and_insert_block();
7815    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7816    BB_then_linear->kind |= block_kind_uniform;
7817    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7818    /* branch from linear then block to invert block */
7819    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7820    BB_then_linear->instructions.emplace_back(std::move(branch));
7821    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7822
7823    /** emit invert merge block */
7824    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7825    ic->invert_idx = ctx->block->index;
7826
7827    /* branch to linear else block (skip else) */
7828    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7829    branch->operands[0] = Operand(ic->cond);
7830    ctx->block->instructions.push_back(std::move(branch));
7831
7832    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7833    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7834
7835    /** emit logical else block */
7836    Block* BB_else_logical = ctx->program->create_and_insert_block();
7837    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7838    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7839    add_linear_edge(ic->invert_idx, BB_else_logical);
7840    ctx->block = BB_else_logical;
7841    append_logical_start(BB_else_logical);
7842 }
7843
7844 static void end_divergent_if(isel_context *ctx, if_context *ic)
7845 {
7846    Block *BB_else_logical = ctx->block;
7847    append_logical_end(BB_else_logical);
7848
7849    /* branch from logical else block to endif block */
7850    aco_ptr<Pseudo_branch_instruction> branch;
7851    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7852    BB_else_logical->instructions.emplace_back(std::move(branch));
7853    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7854    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7855       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7856    BB_else_logical->kind |= block_kind_uniform;
7857
7858    assert(!ctx->cf_info.has_branch);
7859    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7860
7861
7862    /** emit linear else block */
7863    Block* BB_else_linear = ctx->program->create_and_insert_block();
7864    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7865    BB_else_linear->kind |= block_kind_uniform;
7866    add_linear_edge(ic->invert_idx, BB_else_linear);
7867
7868    /* branch from linear else block to endif block */
7869    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7870    BB_else_linear->instructions.emplace_back(std::move(branch));
7871    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7872
7873
7874    /** emit endif merge block */
7875    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7876    append_logical_start(ctx->block);
7877
7878
7879    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7880    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7881    /* uniform control flow never has an empty exec-mask */
7882    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7883       ctx->cf_info.exec_potentially_empty = false;
7884 }
7885
7886 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7887 {
7888    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7889    Builder bld(ctx->program, ctx->block);
7890    aco_ptr<Pseudo_branch_instruction> branch;
7891
7892    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7893       /**
7894        * Uniform conditionals are represented in the following way*) :
7895        *
7896        * The linear and logical CFG:
7897        *                        BB_IF
7898        *                        /    \
7899        *       BB_THEN (logical)      BB_ELSE (logical)
7900        *                        \    /
7901        *                        BB_ENDIF
7902        *
7903        * *) Exceptions may be due to break and continue statements within loops
7904        *    If a break/continue happens within uniform control flow, it branches
7905        *    to the loop exit/entry block. Otherwise, it branches to the next
7906        *    merge block.
7907        **/
7908       append_logical_end(ctx->block);
7909       ctx->block->kind |= block_kind_uniform;
7910
7911       /* emit branch */
7912       assert(cond.regClass() == bld.lm);
7913       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7914       cond = bool_to_scalar_condition(ctx, cond);
7915
7916       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7917       branch->operands[0] = Operand(cond);
7918       branch->operands[0].setFixed(scc);
7919       ctx->block->instructions.emplace_back(std::move(branch));
7920
7921       unsigned BB_if_idx = ctx->block->index;
7922       Block BB_endif = Block();
7923       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7924       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7925
7926       /** emit then block */
7927       Block* BB_then = ctx->program->create_and_insert_block();
7928       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7929       add_edge(BB_if_idx, BB_then);
7930       append_logical_start(BB_then);
7931       ctx->block = BB_then;
7932       visit_cf_list(ctx, &if_stmt->then_list);
7933       BB_then = ctx->block;
7934       bool then_branch = ctx->cf_info.has_branch;
7935       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7936
7937       if (!then_branch) {
7938          append_logical_end(BB_then);
7939          /* branch from then block to endif block */
7940          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7941          BB_then->instructions.emplace_back(std::move(branch));
7942          add_linear_edge(BB_then->index, &BB_endif);
7943          if (!then_branch_divergent)
7944             add_logical_edge(BB_then->index, &BB_endif);
7945          BB_then->kind |= block_kind_uniform;
7946       }
7947
7948       ctx->cf_info.has_branch = false;
7949       ctx->cf_info.parent_loop.has_divergent_branch = false;
7950
7951       /** emit else block */
7952       Block* BB_else = ctx->program->create_and_insert_block();
7953       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7954       add_edge(BB_if_idx, BB_else);
7955       append_logical_start(BB_else);
7956       ctx->block = BB_else;
7957       visit_cf_list(ctx, &if_stmt->else_list);
7958       BB_else = ctx->block;
7959
7960       if (!ctx->cf_info.has_branch) {
7961          append_logical_end(BB_else);
7962          /* branch from then block to endif block */
7963          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7964          BB_else->instructions.emplace_back(std::move(branch));
7965          add_linear_edge(BB_else->index, &BB_endif);
7966          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7967             add_logical_edge(BB_else->index, &BB_endif);
7968          BB_else->kind |= block_kind_uniform;
7969       }
7970
7971       ctx->cf_info.has_branch &= then_branch;
7972       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7973
7974       /** emit endif merge block */
7975       if (!ctx->cf_info.has_branch) {
7976          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7977          append_logical_start(ctx->block);
7978       }
7979    } else { /* non-uniform condition */
7980       /**
7981        * To maintain a logical and linear CFG without critical edges,
7982        * non-uniform conditionals are represented in the following way*) :
7983        *
7984        * The linear CFG:
7985        *                        BB_IF
7986        *                        /    \
7987        *       BB_THEN (logical)      BB_THEN (linear)
7988        *                        \    /
7989        *                        BB_INVERT (linear)
7990        *                        /    \
7991        *       BB_ELSE (logical)      BB_ELSE (linear)
7992        *                        \    /
7993        *                        BB_ENDIF
7994        *
7995        * The logical CFG:
7996        *                        BB_IF
7997        *                        /    \
7998        *       BB_THEN (logical)      BB_ELSE (logical)
7999        *                        \    /
8000        *                        BB_ENDIF
8001        *
8002        * *) Exceptions may be due to break and continue statements within loops
8003        **/
8004
8005       if_context ic;
8006
8007       begin_divergent_if_then(ctx, &ic, cond);
8008       visit_cf_list(ctx, &if_stmt->then_list);
8009
8010       begin_divergent_if_else(ctx, &ic);
8011       visit_cf_list(ctx, &if_stmt->else_list);
8012
8013       end_divergent_if(ctx, &ic);
8014    }
8015 }
8016
8017 static void visit_cf_list(isel_context *ctx,
8018                           struct exec_list *list)
8019 {
8020    foreach_list_typed(nir_cf_node, node, node, list) {
8021       switch (node->type) {
8022       case nir_cf_node_block:
8023          visit_block(ctx, nir_cf_node_as_block(node));
8024          break;
8025       case nir_cf_node_if:
8026          visit_if(ctx, nir_cf_node_as_if(node));
8027          break;
8028       case nir_cf_node_loop:
8029          visit_loop(ctx, nir_cf_node_as_loop(node));
8030          break;
8031       default:
8032          unreachable("unimplemented cf list type");
8033       }
8034    }
8035 }
8036
8037 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8038 {
8039    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8040    uint64_t mask = ctx->vs_output.mask[slot];
8041    if (!is_pos && !mask)
8042       return;
8043    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8044       return;
8045    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8046    exp->enabled_mask = mask;
8047    for (unsigned i = 0; i < 4; ++i) {
8048       if (mask & (1 << i))
8049          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
8050       else
8051          exp->operands[i] = Operand(v1);
8052    }
8053    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8054     * Setting valid_mask=1 prevents it and has no other effect.
8055     */
8056    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8057    exp->done = false;
8058    exp->compressed = false;
8059    if (is_pos)
8060       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8061    else
8062       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8063    ctx->block->instructions.emplace_back(std::move(exp));
8064 }
8065
8066 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8067 {
8068    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8069    exp->enabled_mask = 0;
8070    for (unsigned i = 0; i < 4; ++i)
8071       exp->operands[i] = Operand(v1);
8072    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
8073       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
8074       exp->enabled_mask |= 0x1;
8075    }
8076    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
8077       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
8078       exp->enabled_mask |= 0x4;
8079    }
8080    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
8081       if (ctx->options->chip_class < GFX9) {
8082          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
8083          exp->enabled_mask |= 0x8;
8084       } else {
8085          Builder bld(ctx->program, ctx->block);
8086
8087          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8088                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
8089          if (exp->operands[2].isTemp())
8090             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8091
8092          exp->operands[2] = Operand(out);
8093          exp->enabled_mask |= 0x4;
8094       }
8095    }
8096    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8097    exp->done = false;
8098    exp->compressed = false;
8099    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8100    ctx->block->instructions.emplace_back(std::move(exp));
8101 }
8102
8103 static void create_vs_exports(isel_context *ctx)
8104 {
8105    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8106
8107    if (outinfo->export_prim_id) {
8108       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8109       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8110    }
8111
8112    if (ctx->options->key.has_multiview_view_index) {
8113       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
8114       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8115    }
8116
8117    /* the order these position exports are created is important */
8118    int next_pos = 0;
8119    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8120    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8121       export_vs_psiz_layer_viewport(ctx, &next_pos);
8122    }
8123    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8124       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8125    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8126       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8127
8128    if (ctx->options->key.vs_common_out.export_clip_dists) {
8129       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8130          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8131       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8132          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8133    }
8134
8135    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8136       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8137           i != VARYING_SLOT_PRIMITIVE_ID)
8138          continue;
8139
8140       export_vs_varying(ctx, i, false, NULL);
8141    }
8142 }
8143
8144 static void emit_stream_output(isel_context *ctx,
8145                                Temp const *so_buffers,
8146                                Temp const *so_write_offset,
8147                                const struct radv_stream_output *output)
8148 {
8149    unsigned num_comps = util_bitcount(output->component_mask);
8150    unsigned writemask = (1 << num_comps) - 1;
8151    unsigned loc = output->location;
8152    unsigned buf = output->buffer;
8153
8154    assert(num_comps && num_comps <= 4);
8155    if (!num_comps || num_comps > 4)
8156       return;
8157
8158    unsigned start = ffs(output->component_mask) - 1;
8159
8160    Temp out[4];
8161    bool all_undef = true;
8162    assert(ctx->stage == vertex_vs);
8163    for (unsigned i = 0; i < num_comps; i++) {
8164       out[i] = ctx->vs_output.outputs[loc][start + i];
8165       all_undef = all_undef && !out[i].id();
8166    }
8167    if (all_undef)
8168       return;
8169
8170    while (writemask) {
8171       int start, count;
8172       u_bit_scan_consecutive_range(&writemask, &start, &count);
8173       if (count == 3 && ctx->options->chip_class == GFX6) {
8174          /* GFX6 doesn't support storing vec3, split it. */
8175          writemask |= 1u << (start + 2);
8176          count = 2;
8177       }
8178
8179       unsigned offset = output->offset + start * 4;
8180
8181       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8182       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8183       for (int i = 0; i < count; ++i)
8184          vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8185       vec->definitions[0] = Definition(write_data);
8186       ctx->block->instructions.emplace_back(std::move(vec));
8187
8188       aco_opcode opcode;
8189       switch (count) {
8190       case 1:
8191          opcode = aco_opcode::buffer_store_dword;
8192          break;
8193       case 2:
8194          opcode = aco_opcode::buffer_store_dwordx2;
8195          break;
8196       case 3:
8197          opcode = aco_opcode::buffer_store_dwordx3;
8198          break;
8199       case 4:
8200          opcode = aco_opcode::buffer_store_dwordx4;
8201          break;
8202       default:
8203          unreachable("Unsupported dword count.");
8204       }
8205
8206       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8207       store->operands[0] = Operand(so_write_offset[buf]);
8208       store->operands[1] = Operand(so_buffers[buf]);
8209       store->operands[2] = Operand((uint32_t) 0);
8210       store->operands[3] = Operand(write_data);
8211       if (offset > 4095) {
8212          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8213          Builder bld(ctx->program, ctx->block);
8214          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8215       } else {
8216          store->offset = offset;
8217       }
8218       store->offen = true;
8219       store->glc = true;
8220       store->dlc = false;
8221       store->slc = true;
8222       store->can_reorder = true;
8223       ctx->block->instructions.emplace_back(std::move(store));
8224    }
8225 }
8226
8227 static void emit_streamout(isel_context *ctx, unsigned stream)
8228 {
8229    Builder bld(ctx->program, ctx->block);
8230
8231    Temp so_buffers[4];
8232    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8233    for (unsigned i = 0; i < 4; i++) {
8234       unsigned stride = ctx->program->info->so.strides[i];
8235       if (!stride)
8236          continue;
8237
8238       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
8239    }
8240
8241    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8242                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8243
8244    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8245
8246    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8247
8248    if_context ic;
8249    begin_divergent_if_then(ctx, &ic, can_emit);
8250
8251    bld.reset(ctx->block);
8252
8253    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8254
8255    Temp so_write_offset[4];
8256
8257    for (unsigned i = 0; i < 4; i++) {
8258       unsigned stride = ctx->program->info->so.strides[i];
8259       if (!stride)
8260          continue;
8261
8262       if (stride == 1) {
8263          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8264                                 get_arg(ctx, ctx->args->streamout_write_idx),
8265                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8266          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8267
8268          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8269       } else {
8270          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8271          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8272                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8273          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8274       }
8275    }
8276
8277    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8278       struct radv_stream_output *output =
8279          &ctx->program->info->so.outputs[i];
8280       if (stream != output->stream)
8281          continue;
8282
8283       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8284    }
8285
8286    begin_divergent_if_else(ctx, &ic);
8287    end_divergent_if(ctx, &ic);
8288 }
8289
8290 } /* end namespace */
8291
8292 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8293 {
8294    /* Split all arguments except for the first (ring_offsets) and the last
8295     * (exec) so that the dead channels don't stay live throughout the program.
8296     */
8297    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8298       if (startpgm->definitions[i].regClass().size() > 1) {
8299          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8300                            startpgm->definitions[i].regClass().size());
8301       }
8302    }
8303 }
8304
8305 void handle_bc_optimize(isel_context *ctx)
8306 {
8307    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8308    Builder bld(ctx->program, ctx->block);
8309    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8310    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8311    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8312    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8313    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8314    if (uses_center && uses_centroid) {
8315       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8316                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8317
8318       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8319          Temp new_coord[2];
8320          for (unsigned i = 0; i < 2; i++) {
8321             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8322             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8323             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8324                                     persp_centroid, persp_center, sel);
8325          }
8326          ctx->persp_centroid = bld.tmp(v2);
8327          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8328                     Operand(new_coord[0]), Operand(new_coord[1]));
8329          emit_split_vector(ctx, ctx->persp_centroid, 2);
8330       }
8331
8332       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8333          Temp new_coord[2];
8334          for (unsigned i = 0; i < 2; i++) {
8335             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8336             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8337             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8338                                     linear_centroid, linear_center, sel);
8339          }
8340          ctx->linear_centroid = bld.tmp(v2);
8341          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8342                     Operand(new_coord[0]), Operand(new_coord[1]));
8343          emit_split_vector(ctx, ctx->linear_centroid, 2);
8344       }
8345    }
8346 }
8347
8348 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8349 {
8350    Program *program = ctx->program;
8351
8352    unsigned float_controls = shader->info.float_controls_execution_mode;
8353
8354    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8355       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8356    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8357       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8358                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8359
8360    program->next_fp_mode.must_flush_denorms32 =
8361       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8362    program->next_fp_mode.must_flush_denorms16_64 =
8363       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8364                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8365
8366    program->next_fp_mode.care_about_round32 =
8367       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8368
8369    program->next_fp_mode.care_about_round16_64 =
8370       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8371                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8372
8373    /* default to preserving fp16 and fp64 denorms, since it's free */
8374    if (program->next_fp_mode.must_flush_denorms16_64)
8375       program->next_fp_mode.denorm16_64 = 0;
8376    else
8377       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8378
8379    /* preserving fp32 denorms is expensive, so only do it if asked */
8380    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8381       program->next_fp_mode.denorm32 = fp_denorm_keep;
8382    else
8383       program->next_fp_mode.denorm32 = 0;
8384
8385    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8386       program->next_fp_mode.round32 = fp_round_tz;
8387    else
8388       program->next_fp_mode.round32 = fp_round_ne;
8389
8390    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8391       program->next_fp_mode.round16_64 = fp_round_tz;
8392    else
8393       program->next_fp_mode.round16_64 = fp_round_ne;
8394
8395    ctx->block->fp_mode = program->next_fp_mode;
8396 }
8397
8398 void select_program(Program *program,
8399                     unsigned shader_count,
8400                     struct nir_shader *const *shaders,
8401                     ac_shader_config* config,
8402                     struct radv_shader_args *args)
8403 {
8404    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
8405
8406    for (unsigned i = 0; i < shader_count; i++) {
8407       nir_shader *nir = shaders[i];
8408       init_context(&ctx, nir);
8409
8410       setup_fp_mode(&ctx, nir);
8411
8412       if (!i) {
8413          /* needs to be after init_context() for FS */
8414          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8415          append_logical_start(ctx.block);
8416          split_arguments(&ctx, startpgm);
8417       }
8418
8419       if_context ic;
8420       if (shader_count >= 2) {
8421          Builder bld(ctx.program, ctx.block);
8422          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
8423          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8424          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8425
8426          begin_divergent_if_then(&ctx, &ic, cond);
8427       }
8428
8429       if (i) {
8430          Builder bld(ctx.program, ctx.block);
8431          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
8432          bld.sopp(aco_opcode::s_barrier);
8433       }
8434
8435       if (ctx.stage == fragment_fs)
8436          handle_bc_optimize(&ctx);
8437
8438       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8439       visit_cf_list(&ctx, &func->body);
8440
8441       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8442          emit_streamout(&ctx, 0);
8443
8444       if (ctx.stage == vertex_vs)
8445          create_vs_exports(&ctx);
8446
8447       if (shader_count >= 2) {
8448          begin_divergent_if_else(&ctx, &ic);
8449          end_divergent_if(&ctx, &ic);
8450       }
8451
8452       ralloc_free(ctx.divergent_vals);
8453    }
8454
8455    program->config->float_mode = program->blocks[0].fp_mode.val;
8456
8457    append_logical_end(ctx.block);
8458    ctx.block->kind |= block_kind_uniform;
8459    Builder bld(ctx.program, ctx.block);
8460    if (ctx.program->wb_smem_l1_on_end)
8461       bld.smem(aco_opcode::s_dcache_wb, false);
8462    bld.sopp(aco_opcode::s_endpgm);
8463
8464    /* cleanup CFG */
8465    for (Block& BB : program->blocks) {
8466       for (unsigned idx : BB.linear_preds)
8467          program->blocks[idx].linear_succs.emplace_back(BB.index);
8468       for (unsigned idx : BB.logical_preds)
8469          program->blocks[idx].logical_succs.emplace_back(BB.index);
8470    }
8471 }
8472 }