src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool then_branch_divergent;
  89    Block BB_invert;
  90    Block BB_endif;
  91 };
  92
  93 static void visit_cf_list(struct isel_context *ctx,
  94                           struct exec_list *list);
  95
  96 static void add_logical_edge(unsigned pred_idx, Block *succ)
  97 {
  98    succ->logical_preds.emplace_back(pred_idx);
  99 }
 100
 101
 102 static void add_linear_edge(unsigned pred_idx, Block *succ)
 103 {
 104    succ->linear_preds.emplace_back(pred_idx);
 105 }
 106
 107 static void add_edge(unsigned pred_idx, Block *succ)
 108 {
 109    add_logical_edge(pred_idx, succ);
 110    add_linear_edge(pred_idx, succ);
 111 }
 112
 113 static void append_logical_start(Block *b)
 114 {
 115    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 116 }
 117
 118 static void append_logical_end(Block *b)
 119 {
 120    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 121 }
 122
 123 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 124 {
 125    assert(ctx->allocated[def->index].id());
 126    return ctx->allocated[def->index];
 127 }
 128
 129 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 130                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 131 {
 132    Builder bld(ctx->program, ctx->block);
 133    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 134    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 135
 136    if (ctx->program->wave_size == 32) {
 137       return thread_id_lo;
 138    } else {
 139       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 140       return thread_id_hi;
 141    }
 142 }
 143
 144 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 145 {
 146    Builder bld(ctx->program, ctx->block);
 147
 148    if (!dst.id())
 149       dst = bld.tmp(src.regClass());
 150
 151    assert(src.size() == dst.size());
 152
 153    if (ctx->stage != fragment_fs) {
 154       if (!dst.id())
 155          return src;
 156
 157       bld.copy(Definition(dst), src);
 158       return dst;
 159    }
 160
 161    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 162    ctx->program->needs_wqm |= program_needs_wqm;
 163    return dst;
 164 }
 165
 166 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 167 {
 168    if (index.regClass() == s1)
 169       return bld.readlane(bld.def(s1), data, index);
 170
 171    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 172
 173    /* Currently not implemented on GFX6-7 */
 174    assert(ctx->options->chip_class >= GFX8);
 175
 176    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 177       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 178    }
 179
 180    /* GFX10, wave64 mode:
 181     * The bpermute instruction is limited to half-wave operation, which means that it can't
 182     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 183     * emulate it here.
 184     */
 185    if (!ctx->has_gfx10_wave64_bpermute) {
 186       ctx->has_gfx10_wave64_bpermute = true;
 187       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 188       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 189    }
 190
 191    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 192    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 193    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 194    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 195
 196    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 197                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 198 }
 199
 200 Temp as_vgpr(isel_context *ctx, Temp val)
 201 {
 202    if (val.type() == RegType::sgpr) {
 203       Builder bld(ctx->program, ctx->block);
 204       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 205    }
 206    assert(val.type() == RegType::vgpr);
 207    return val;
 208 }
 209
 210 //assumes a != 0xffffffff
 211 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 212 {
 213    assert(b != 0);
 214    Builder bld(ctx->program, ctx->block);
 215
 216    if (util_is_power_of_two_or_zero(b)) {
 217       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 218       return;
 219    }
 220
 221    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 222
 223    assert(info.multiplier <= 0xffffffff);
 224
 225    bool pre_shift = info.pre_shift != 0;
 226    bool increment = info.increment != 0;
 227    bool multiply = true;
 228    bool post_shift = info.post_shift != 0;
 229
 230    if (!pre_shift && !increment && !multiply && !post_shift) {
 231       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 232       return;
 233    }
 234
 235    Temp pre_shift_dst = a;
 236    if (pre_shift) {
 237       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 238       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 239    }
 240
 241    Temp increment_dst = pre_shift_dst;
 242    if (increment) {
 243       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 244       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 245    }
 246
 247    Temp multiply_dst = increment_dst;
 248    if (multiply) {
 249       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 250       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 251                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 252    }
 253
 254    if (post_shift) {
 255       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 256    }
 257 }
 258
 259 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 260 {
 261    Builder bld(ctx->program, ctx->block);
 262    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 263 }
 264
 265
 266 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 267 {
 268    /* no need to extract the whole vector */
 269    if (src.regClass() == dst_rc) {
 270       assert(idx == 0);
 271       return src;
 272    }
 273    assert(src.size() > idx);
 274    Builder bld(ctx->program, ctx->block);
 275    auto it = ctx->allocated_vec.find(src.id());
 276    /* the size check needs to be early because elements other than 0 may be garbage */
 277    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 278       if (it->second[idx].regClass() == dst_rc) {
 279          return it->second[idx];
 280       } else {
 281          assert(dst_rc.size() == it->second[idx].regClass().size());
 282          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 283          return bld.copy(bld.def(dst_rc), it->second[idx]);
 284       }
 285    }
 286
 287    if (src.size() == dst_rc.size()) {
 288       assert(idx == 0);
 289       return bld.copy(bld.def(dst_rc), src);
 290    } else {
 291       Temp dst = bld.tmp(dst_rc);
 292       emit_extract_vector(ctx, src, idx, dst);
 293       return dst;
 294    }
 295 }
 296
 297 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 298 {
 299    if (num_components == 1)
 300       return;
 301    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 302       return;
 303    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 304    split->operands[0] = Operand(vec_src);
 305    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 306    for (unsigned i = 0; i < num_components; i++) {
 307       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 308       split->definitions[i] = Definition(elems[i]);
 309    }
 310    ctx->block->instructions.emplace_back(std::move(split));
 311    ctx->allocated_vec.emplace(vec_src.id(), elems);
 312 }
 313
 314 /* This vector expansion uses a mask to determine which elements in the new vector
 315  * come from the original vector. The other elements are undefined. */
 316 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 317 {
 318    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 319
 320    if (vec_src == dst)
 321       return;
 322
 323    Builder bld(ctx->program, ctx->block);
 324    if (num_components == 1) {
 325       if (dst.type() == RegType::sgpr)
 326          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 327       else
 328          bld.copy(Definition(dst), vec_src);
 329       return;
 330    }
 331
 332    unsigned component_size = dst.size() / num_components;
 333    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 334
 335    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 336    vec->definitions[0] = Definition(dst);
 337    unsigned k = 0;
 338    for (unsigned i = 0; i < num_components; i++) {
 339       if (mask & (1 << i)) {
 340          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 341          if (dst.type() == RegType::sgpr)
 342             src = bld.as_uniform(src);
 343          vec->operands[i] = Operand(src);
 344       } else {
 345          vec->operands[i] = Operand(0u);
 346       }
 347       elems[i] = vec->operands[i].getTemp();
 348    }
 349    ctx->block->instructions.emplace_back(std::move(vec));
 350    ctx->allocated_vec.emplace(dst.id(), elems);
 351 }
 352
 353 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 354 {
 355    Builder bld(ctx->program, ctx->block);
 356    if (!dst.id())
 357       dst = bld.tmp(bld.lm);
 358
 359    assert(val.regClass() == s1);
 360    assert(dst.regClass() == bld.lm);
 361
 362    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 363 }
 364
 365 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 366 {
 367    Builder bld(ctx->program, ctx->block);
 368    if (!dst.id())
 369       dst = bld.tmp(s1);
 370
 371    assert(val.regClass() == bld.lm);
 372    assert(dst.regClass() == s1);
 373
 374    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 375    Temp tmp = bld.tmp(s1);
 376    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 377    return emit_wqm(ctx, tmp, dst);
 378 }
 379
 380 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 381 {
 382    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 383       return get_ssa_temp(ctx, src.src.ssa);
 384
 385    if (src.src.ssa->num_components == size) {
 386       bool identity_swizzle = true;
 387       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 388          if (src.swizzle[i] != i)
 389             identity_swizzle = false;
 390       }
 391       if (identity_swizzle)
 392          return get_ssa_temp(ctx, src.src.ssa);
 393    }
 394
 395    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 396    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 397    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 398    assert(vec.size() % elem_size == 0);
 399
 400    RegClass elem_rc = RegClass(vec.type(), elem_size);
 401    if (size == 1) {
 402       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 403    } else {
 404       assert(size <= 4);
 405       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 406       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 407       for (unsigned i = 0; i < size; ++i) {
 408          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 409          vec_instr->operands[i] = Operand{elems[i]};
 410       }
 411       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 412       vec_instr->definitions[0] = Definition(dst);
 413       ctx->block->instructions.emplace_back(std::move(vec_instr));
 414       ctx->allocated_vec.emplace(dst.id(), elems);
 415       return dst;
 416    }
 417 }
 418
 419 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 420 {
 421    if (ptr.size() == 2)
 422       return ptr;
 423    Builder bld(ctx->program, ctx->block);
 424    if (ptr.type() == RegType::vgpr)
 425       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 426    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 427                      ptr, Operand((unsigned)ctx->options->address32_hi));
 428 }
 429
 430 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 431 {
 432    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 433    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 434    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 435    sop2->definitions[0] = Definition(dst);
 436    if (writes_scc)
 437       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 438    ctx->block->instructions.emplace_back(std::move(sop2));
 439 }
 440
 441 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 442                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 443 {
 444    Builder bld(ctx->program, ctx->block);
 445    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 446    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 447    if (src1.type() == RegType::sgpr) {
 448       if (commutative && src0.type() == RegType::vgpr) {
 449          Temp t = src0;
 450          src0 = src1;
 451          src1 = t;
 452       } else if (src0.type() == RegType::vgpr &&
 453                  op != aco_opcode::v_madmk_f32 &&
 454                  op != aco_opcode::v_madak_f32 &&
 455                  op != aco_opcode::v_madmk_f16 &&
 456                  op != aco_opcode::v_madak_f16) {
 457          /* If the instruction is not commutative, we emit a VOP3A instruction */
 458          bld.vop2_e64(op, Definition(dst), src0, src1);
 459          return;
 460       } else {
 461          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 462       }
 463    }
 464
 465    if (flush_denorms && ctx->program->chip_class < GFX9) {
 466       assert(dst.size() == 1);
 467       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 468       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 469    } else {
 470       bld.vop2(op, Definition(dst), src0, src1);
 471    }
 472 }
 473
 474 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 475                             bool flush_denorms = false)
 476 {
 477    Temp src0 = get_alu_src(ctx, instr->src[0]);
 478    Temp src1 = get_alu_src(ctx, instr->src[1]);
 479    Temp src2 = get_alu_src(ctx, instr->src[2]);
 480
 481    /* ensure that the instruction has at most 1 sgpr operand
 482     * The optimizer will inline constants for us */
 483    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 484       src0 = as_vgpr(ctx, src0);
 485    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 486       src1 = as_vgpr(ctx, src1);
 487    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 488       src2 = as_vgpr(ctx, src2);
 489
 490    Builder bld(ctx->program, ctx->block);
 491    if (flush_denorms && ctx->program->chip_class < GFX9) {
 492       assert(dst.size() == 1);
 493       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 494       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 495    } else {
 496       bld.vop3(op, Definition(dst), src0, src1, src2);
 497    }
 498 }
 499
 500 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 501 {
 502    Builder bld(ctx->program, ctx->block);
 503    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 504 }
 505
 506 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 507 {
 508    Temp src0 = get_alu_src(ctx, instr->src[0]);
 509    Temp src1 = get_alu_src(ctx, instr->src[1]);
 510    assert(src0.size() == src1.size());
 511
 512    aco_ptr<Instruction> vopc;
 513    if (src1.type() == RegType::sgpr) {
 514       if (src0.type() == RegType::vgpr) {
 515          /* to swap the operands, we might also have to change the opcode */
 516          switch (op) {
 517             case aco_opcode::v_cmp_lt_f32:
 518                op = aco_opcode::v_cmp_gt_f32;
 519                break;
 520             case aco_opcode::v_cmp_ge_f32:
 521                op = aco_opcode::v_cmp_le_f32;
 522                break;
 523             case aco_opcode::v_cmp_lt_i32:
 524                op = aco_opcode::v_cmp_gt_i32;
 525                break;
 526             case aco_opcode::v_cmp_ge_i32:
 527                op = aco_opcode::v_cmp_le_i32;
 528                break;
 529             case aco_opcode::v_cmp_lt_u32:
 530                op = aco_opcode::v_cmp_gt_u32;
 531                break;
 532             case aco_opcode::v_cmp_ge_u32:
 533                op = aco_opcode::v_cmp_le_u32;
 534                break;
 535             case aco_opcode::v_cmp_lt_f64:
 536                op = aco_opcode::v_cmp_gt_f64;
 537                break;
 538             case aco_opcode::v_cmp_ge_f64:
 539                op = aco_opcode::v_cmp_le_f64;
 540                break;
 541             case aco_opcode::v_cmp_lt_i64:
 542                op = aco_opcode::v_cmp_gt_i64;
 543                break;
 544             case aco_opcode::v_cmp_ge_i64:
 545                op = aco_opcode::v_cmp_le_i64;
 546                break;
 547             case aco_opcode::v_cmp_lt_u64:
 548                op = aco_opcode::v_cmp_gt_u64;
 549                break;
 550             case aco_opcode::v_cmp_ge_u64:
 551                op = aco_opcode::v_cmp_le_u64;
 552                break;
 553             default: /* eq and ne are commutative */
 554                break;
 555          }
 556          Temp t = src0;
 557          src0 = src1;
 558          src1 = t;
 559       } else {
 560          src1 = as_vgpr(ctx, src1);
 561       }
 562    }
 563
 564    Builder bld(ctx->program, ctx->block);
 565    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 566 }
 567
 568 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 569 {
 570    Temp src0 = get_alu_src(ctx, instr->src[0]);
 571    Temp src1 = get_alu_src(ctx, instr->src[1]);
 572    Builder bld(ctx->program, ctx->block);
 573
 574    assert(dst.regClass() == bld.lm);
 575    assert(src0.type() == RegType::sgpr);
 576    assert(src1.type() == RegType::sgpr);
 577    assert(src0.regClass() == src1.regClass());
 578
 579    /* Emit the SALU comparison instruction */
 580    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 581    /* Turn the result into a per-lane bool */
 582    bool_to_vector_condition(ctx, cmp, dst);
 583 }
 584
 585 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 586                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 587 {
 588    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 589    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 590    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 591    bool use_valu = s_op == aco_opcode::num_opcodes ||
 592                    divergent_vals ||
 593                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 594                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 595    aco_opcode op = use_valu ? v_op : s_op;
 596    assert(op != aco_opcode::num_opcodes);
 597    assert(dst.regClass() == ctx->program->lane_mask);
 598
 599    if (use_valu)
 600       emit_vopc_instruction(ctx, instr, op, dst);
 601    else
 602       emit_sopc_instruction(ctx, instr, op, dst);
 603 }
 604
 605 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 606 {
 607    Builder bld(ctx->program, ctx->block);
 608    Temp src0 = get_alu_src(ctx, instr->src[0]);
 609    Temp src1 = get_alu_src(ctx, instr->src[1]);
 610
 611    assert(dst.regClass() == bld.lm);
 612    assert(src0.regClass() == bld.lm);
 613    assert(src1.regClass() == bld.lm);
 614
 615    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 616 }
 617
 618 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 619 {
 620    Builder bld(ctx->program, ctx->block);
 621    Temp cond = get_alu_src(ctx, instr->src[0]);
 622    Temp then = get_alu_src(ctx, instr->src[1]);
 623    Temp els = get_alu_src(ctx, instr->src[2]);
 624
 625    assert(cond.regClass() == bld.lm);
 626
 627    if (dst.type() == RegType::vgpr) {
 628       aco_ptr<Instruction> bcsel;
 629       if (dst.size() == 1) {
 630          then = as_vgpr(ctx, then);
 631          els = as_vgpr(ctx, els);
 632
 633          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 634       } else if (dst.size() == 2) {
 635          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 637          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 638          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 639
 640          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 641          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 642
 643          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 644       } else {
 645          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 646          nir_print_instr(&instr->instr, stderr);
 647          fprintf(stderr, "\n");
 648       }
 649       return;
 650    }
 651
 652    if (instr->dest.dest.ssa.bit_size == 1) {
 653       assert(dst.regClass() == bld.lm);
 654       assert(then.regClass() == bld.lm);
 655       assert(els.regClass() == bld.lm);
 656    }
 657
 658    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 659       if (dst.regClass() == s1 || dst.regClass() == s2) {
 660          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 661          assert(dst.size() == then.size());
 662          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 663          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 664       } else {
 665          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 666          nir_print_instr(&instr->instr, stderr);
 667          fprintf(stderr, "\n");
 668       }
 669       return;
 670    }
 671
 672    /* divergent boolean bcsel
 673     * this implements bcsel on bools: dst = s0 ? s1 : s2
 674     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 675    assert(instr->dest.dest.ssa.bit_size == 1);
 676
 677    if (cond.id() != then.id())
 678       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 679
 680    if (cond.id() == els.id())
 681       bld.sop1(Builder::s_mov, Definition(dst), then);
 682    else
 683       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 684                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 685 }
 686
 687 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 688                     aco_opcode op, uint32_t undo)
 689 {
 690    /* multiply by 16777216 to handle denormals */
 691    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 692                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 693    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 694    scaled = bld.vop1(op, bld.def(v1), scaled);
 695    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 696
 697    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 698
 699    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 700 }
 701
 702 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 703 {
 704    if (ctx->block->fp_mode.denorm32 == 0) {
 705       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 706       return;
 707    }
 708
 709    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 710 }
 711
 712 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 713 {
 714    if (ctx->block->fp_mode.denorm32 == 0) {
 715       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 716       return;
 717    }
 718
 719    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 720 }
 721
 722 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 723 {
 724    if (ctx->block->fp_mode.denorm32 == 0) {
 725       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 726       return;
 727    }
 728
 729    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 730 }
 731
 732 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 733 {
 734    if (ctx->block->fp_mode.denorm32 == 0) {
 735       bld.vop1(aco_opcode::v_log_f32, dst, val);
 736       return;
 737    }
 738
 739    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 740 }
 741
 742 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 743 {
 744    if (ctx->options->chip_class >= GFX7)
 745       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 746
 747    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 748    /* TODO: create more efficient code! */
 749    if (val.type() == RegType::sgpr)
 750       val = as_vgpr(ctx, val);
 751
 752    /* Split the input value. */
 753    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 754    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 755
 756    /* Extract the exponent and compute the unbiased value. */
 757    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 758
 759    /* Extract the fractional part. */
 760    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 761    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 762
 763    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 764    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 765
 766    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 767    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 768    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 769    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 770    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 771
 772    /* Get the sign bit. */
 773    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 774
 775    /* Decide the operation to apply depending on the unbiased exponent. */
 776    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 777    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 778    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 779    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 780    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 781    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 782
 783    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 784 }
 785
 786 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 787 {
 788    if (ctx->options->chip_class >= GFX7)
 789       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 790
 791    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 792    Temp src0 = as_vgpr(ctx, val);
 793
 794    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 795    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 796
 797    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 798    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 799    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 800
 801    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 803    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 804    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 805
 806    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 807    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 808
 809    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 810
 811    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 812    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 813
 814    return add->definitions[0].getTemp();
 815 }
 816
 817 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 818 {
 819    if (!instr->dest.dest.is_ssa) {
 820       fprintf(stderr, "nir alu dst not in ssa: ");
 821       nir_print_instr(&instr->instr, stderr);
 822       fprintf(stderr, "\n");
 823       abort();
 824    }
 825    Builder bld(ctx->program, ctx->block);
 826    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 827    switch(instr->op) {
 828    case nir_op_vec2:
 829    case nir_op_vec3:
 830    case nir_op_vec4: {
 831       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 832       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 833       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 834          elems[i] = get_alu_src(ctx, instr->src[i]);
 835          vec->operands[i] = Operand{elems[i]};
 836       }
 837       vec->definitions[0] = Definition(dst);
 838       ctx->block->instructions.emplace_back(std::move(vec));
 839       ctx->allocated_vec.emplace(dst.id(), elems);
 840       break;
 841    }
 842    case nir_op_mov: {
 843       Temp src = get_alu_src(ctx, instr->src[0]);
 844       aco_ptr<Instruction> mov;
 845       if (dst.type() == RegType::sgpr) {
 846          if (src.type() == RegType::vgpr)
 847             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 848          else if (src.regClass() == s1)
 849             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 850          else if (src.regClass() == s2)
 851             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 852          else
 853             unreachable("wrong src register class for nir_op_imov");
 854       } else if (dst.regClass() == v1) {
 855          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 856       } else if (dst.regClass() == v2) {
 857          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 858       } else {
 859          nir_print_instr(&instr->instr, stderr);
 860          unreachable("Should have been lowered to scalar.");
 861       }
 862       break;
 863    }
 864    case nir_op_inot: {
 865       Temp src = get_alu_src(ctx, instr->src[0]);
 866       if (instr->dest.dest.ssa.bit_size == 1) {
 867          assert(src.regClass() == bld.lm);
 868          assert(dst.regClass() == bld.lm);
 869          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 870          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 871          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 872       } else if (dst.regClass() == v1) {
 873          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 874       } else if (dst.type() == RegType::sgpr) {
 875          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 876          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 877       } else {
 878          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 879          nir_print_instr(&instr->instr, stderr);
 880          fprintf(stderr, "\n");
 881       }
 882       break;
 883    }
 884    case nir_op_ineg: {
 885       Temp src = get_alu_src(ctx, instr->src[0]);
 886       if (dst.regClass() == v1) {
 887          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 888       } else if (dst.regClass() == s1) {
 889          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 890       } else if (dst.size() == 2) {
 891          Temp src0 = bld.tmp(dst.type(), 1);
 892          Temp src1 = bld.tmp(dst.type(), 1);
 893          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 894
 895          if (dst.regClass() == s2) {
 896             Temp carry = bld.tmp(s1);
 897             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 898             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 899             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 900          } else {
 901             Temp lower = bld.tmp(v1);
 902             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 903             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 904             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 905          }
 906       } else {
 907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 908          nir_print_instr(&instr->instr, stderr);
 909          fprintf(stderr, "\n");
 910       }
 911       break;
 912    }
 913    case nir_op_iabs: {
 914       if (dst.regClass() == s1) {
 915          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 916       } else if (dst.regClass() == v1) {
 917          Temp src = get_alu_src(ctx, instr->src[0]);
 918          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 919       } else {
 920          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 921          nir_print_instr(&instr->instr, stderr);
 922          fprintf(stderr, "\n");
 923       }
 924       break;
 925    }
 926    case nir_op_isign: {
 927       Temp src = get_alu_src(ctx, instr->src[0]);
 928       if (dst.regClass() == s1) {
 929          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 930          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 931          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 932       } else if (dst.regClass() == s2) {
 933          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 934          Temp neqz;
 935          if (ctx->program->chip_class >= GFX8)
 936             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 937          else
 938             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 939          /* SCC gets zero-extended to 64 bit */
 940          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 941       } else if (dst.regClass() == v1) {
 942          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 943          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 944          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 945       } else if (dst.regClass() == v2) {
 946          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 947          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 948          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 949          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 950          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 951          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_imax: {
 960       if (dst.regClass() == v1) {
 961          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 962       } else if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 964       } else {
 965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 966          nir_print_instr(&instr->instr, stderr);
 967          fprintf(stderr, "\n");
 968       }
 969       break;
 970    }
 971    case nir_op_umax: {
 972       if (dst.regClass() == v1) {
 973          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 974       } else if (dst.regClass() == s1) {
 975          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 976       } else {
 977          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 978          nir_print_instr(&instr->instr, stderr);
 979          fprintf(stderr, "\n");
 980       }
 981       break;
 982    }
 983    case nir_op_imin: {
 984       if (dst.regClass() == v1) {
 985          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 986       } else if (dst.regClass() == s1) {
 987          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 988       } else {
 989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 990          nir_print_instr(&instr->instr, stderr);
 991          fprintf(stderr, "\n");
 992       }
 993       break;
 994    }
 995    case nir_op_umin: {
 996       if (dst.regClass() == v1) {
 997          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 998       } else if (dst.regClass() == s1) {
 999          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1000       } else {
1001          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1002          nir_print_instr(&instr->instr, stderr);
1003          fprintf(stderr, "\n");
1004       }
1005       break;
1006    }
1007    case nir_op_ior: {
1008       if (instr->dest.dest.ssa.bit_size == 1) {
1009          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1010       } else if (dst.regClass() == v1) {
1011          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1012       } else if (dst.regClass() == s1) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1014       } else if (dst.regClass() == s2) {
1015          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1016       } else {
1017          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1018          nir_print_instr(&instr->instr, stderr);
1019          fprintf(stderr, "\n");
1020       }
1021       break;
1022    }
1023    case nir_op_iand: {
1024       if (instr->dest.dest.ssa.bit_size == 1) {
1025          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1026       } else if (dst.regClass() == v1) {
1027          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1028       } else if (dst.regClass() == s1) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1030       } else if (dst.regClass() == s2) {
1031          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1032       } else {
1033          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1034          nir_print_instr(&instr->instr, stderr);
1035          fprintf(stderr, "\n");
1036       }
1037       break;
1038    }
1039    case nir_op_ixor: {
1040       if (instr->dest.dest.ssa.bit_size == 1) {
1041          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1042       } else if (dst.regClass() == v1) {
1043          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1044       } else if (dst.regClass() == s1) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1046       } else if (dst.regClass() == s2) {
1047          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1048       } else {
1049          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1050          nir_print_instr(&instr->instr, stderr);
1051          fprintf(stderr, "\n");
1052       }
1053       break;
1054    }
1055    case nir_op_ushr: {
1056       if (dst.regClass() == v1) {
1057          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1058       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1059          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1060                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1061       } else if (dst.regClass() == v2) {
1062          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1063                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1064       } else if (dst.regClass() == s2) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1066       } else if (dst.regClass() == s1) {
1067          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1068       } else {
1069          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1070          nir_print_instr(&instr->instr, stderr);
1071          fprintf(stderr, "\n");
1072       }
1073       break;
1074    }
1075    case nir_op_ishl: {
1076       if (dst.regClass() == v1) {
1077          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1078       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1079          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1080                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1081       } else if (dst.regClass() == v2) {
1082          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1083                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1084       } else if (dst.regClass() == s1) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1086       } else if (dst.regClass() == s2) {
1087          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1088       } else {
1089          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1090          nir_print_instr(&instr->instr, stderr);
1091          fprintf(stderr, "\n");
1092       }
1093       break;
1094    }
1095    case nir_op_ishr: {
1096       if (dst.regClass() == v1) {
1097          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1098       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1099          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1100                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1101       } else if (dst.regClass() == v2) {
1102          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1103                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1104       } else if (dst.regClass() == s1) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1106       } else if (dst.regClass() == s2) {
1107          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1108       } else {
1109          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1110          nir_print_instr(&instr->instr, stderr);
1111          fprintf(stderr, "\n");
1112       }
1113       break;
1114    }
1115    case nir_op_find_lsb: {
1116       Temp src = get_alu_src(ctx, instr->src[0]);
1117       if (src.regClass() == s1) {
1118          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1119       } else if (src.regClass() == v1) {
1120          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1121       } else if (src.regClass() == s2) {
1122          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_ufind_msb:
1131    case nir_op_ifind_msb: {
1132       Temp src = get_alu_src(ctx, instr->src[0]);
1133       if (src.regClass() == s1 || src.regClass() == s2) {
1134          aco_opcode op = src.regClass() == s2 ?
1135                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1136                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1137          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1138
1139          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1140                                         Operand(src.size() * 32u - 1u), msb_rev);
1141          Temp msb = sub.def(0).getTemp();
1142          Temp carry = sub.def(1).getTemp();
1143
1144          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1145       } else if (src.regClass() == v1) {
1146          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1147          Temp msb_rev = bld.tmp(v1);
1148          emit_vop1_instruction(ctx, instr, op, msb_rev);
1149          Temp msb = bld.tmp(v1);
1150          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1151          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1152       } else {
1153          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1154          nir_print_instr(&instr->instr, stderr);
1155          fprintf(stderr, "\n");
1156       }
1157       break;
1158    }
1159    case nir_op_bitfield_reverse: {
1160       if (dst.regClass() == s1) {
1161          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else if (dst.regClass() == v1) {
1163          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1164       } else {
1165          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1166          nir_print_instr(&instr->instr, stderr);
1167          fprintf(stderr, "\n");
1168       }
1169       break;
1170    }
1171    case nir_op_iadd: {
1172       if (dst.regClass() == s1) {
1173          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1174          break;
1175       }
1176
1177       Temp src0 = get_alu_src(ctx, instr->src[0]);
1178       Temp src1 = get_alu_src(ctx, instr->src[1]);
1179       if (dst.regClass() == v1) {
1180          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1181          break;
1182       }
1183
1184       assert(src0.size() == 2 && src1.size() == 2);
1185       Temp src00 = bld.tmp(src0.type(), 1);
1186       Temp src01 = bld.tmp(dst.type(), 1);
1187       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1188       Temp src10 = bld.tmp(src1.type(), 1);
1189       Temp src11 = bld.tmp(dst.type(), 1);
1190       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1191
1192       if (dst.regClass() == s2) {
1193          Temp carry = bld.tmp(s1);
1194          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1195          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1196          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1197       } else if (dst.regClass() == v2) {
1198          Temp dst0 = bld.tmp(v1);
1199          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1200          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1201          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1202       } else {
1203          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1204          nir_print_instr(&instr->instr, stderr);
1205          fprintf(stderr, "\n");
1206       }
1207       break;
1208    }
1209    case nir_op_uadd_sat: {
1210       Temp src0 = get_alu_src(ctx, instr->src[0]);
1211       Temp src1 = get_alu_src(ctx, instr->src[1]);
1212       if (dst.regClass() == s1) {
1213          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1214          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1215                   src0, src1);
1216          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1217       } else if (dst.regClass() == v1) {
1218          if (ctx->options->chip_class >= GFX9) {
1219             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1220             add->operands[0] = Operand(src0);
1221             add->operands[1] = Operand(src1);
1222             add->definitions[0] = Definition(dst);
1223             add->clamp = 1;
1224             ctx->block->instructions.emplace_back(std::move(add));
1225          } else {
1226             if (src1.regClass() != v1)
1227                std::swap(src0, src1);
1228             assert(src1.regClass() == v1);
1229             Temp tmp = bld.tmp(v1);
1230             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1231             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1232          }
1233       } else {
1234          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1235          nir_print_instr(&instr->instr, stderr);
1236          fprintf(stderr, "\n");
1237       }
1238       break;
1239    }
1240    case nir_op_uadd_carry: {
1241       Temp src0 = get_alu_src(ctx, instr->src[0]);
1242       Temp src1 = get_alu_src(ctx, instr->src[1]);
1243       if (dst.regClass() == s1) {
1244          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1245          break;
1246       }
1247       if (dst.regClass() == v1) {
1248          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1249          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1250          break;
1251       }
1252
1253       Temp src00 = bld.tmp(src0.type(), 1);
1254       Temp src01 = bld.tmp(dst.type(), 1);
1255       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1256       Temp src10 = bld.tmp(src1.type(), 1);
1257       Temp src11 = bld.tmp(dst.type(), 1);
1258       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1259       if (dst.regClass() == s2) {
1260          Temp carry = bld.tmp(s1);
1261          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1262          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1263          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1264       } else if (dst.regClass() == v2) {
1265          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1266          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1267          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1268          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1269       } else {
1270          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1271          nir_print_instr(&instr->instr, stderr);
1272          fprintf(stderr, "\n");
1273       }
1274       break;
1275    }
1276    case nir_op_isub: {
1277       if (dst.regClass() == s1) {
1278          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1279          break;
1280       }
1281
1282       Temp src0 = get_alu_src(ctx, instr->src[0]);
1283       Temp src1 = get_alu_src(ctx, instr->src[1]);
1284       if (dst.regClass() == v1) {
1285          bld.vsub32(Definition(dst), src0, src1);
1286          break;
1287       }
1288
1289       Temp src00 = bld.tmp(src0.type(), 1);
1290       Temp src01 = bld.tmp(dst.type(), 1);
1291       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1292       Temp src10 = bld.tmp(src1.type(), 1);
1293       Temp src11 = bld.tmp(dst.type(), 1);
1294       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1295       if (dst.regClass() == s2) {
1296          Temp carry = bld.tmp(s1);
1297          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1298          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1299          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1300       } else if (dst.regClass() == v2) {
1301          Temp lower = bld.tmp(v1);
1302          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1303          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1304          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1305       } else {
1306          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1307          nir_print_instr(&instr->instr, stderr);
1308          fprintf(stderr, "\n");
1309       }
1310       break;
1311    }
1312    case nir_op_usub_borrow: {
1313       Temp src0 = get_alu_src(ctx, instr->src[0]);
1314       Temp src1 = get_alu_src(ctx, instr->src[1]);
1315       if (dst.regClass() == s1) {
1316          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1317          break;
1318       } else if (dst.regClass() == v1) {
1319          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1320          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1321          break;
1322       }
1323
1324       Temp src00 = bld.tmp(src0.type(), 1);
1325       Temp src01 = bld.tmp(dst.type(), 1);
1326       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1327       Temp src10 = bld.tmp(src1.type(), 1);
1328       Temp src11 = bld.tmp(dst.type(), 1);
1329       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1330       if (dst.regClass() == s2) {
1331          Temp borrow = bld.tmp(s1);
1332          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1333          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1334          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1335       } else if (dst.regClass() == v2) {
1336          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1337          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1338          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1339          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1340       } else {
1341          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1342          nir_print_instr(&instr->instr, stderr);
1343          fprintf(stderr, "\n");
1344       }
1345       break;
1346    }
1347    case nir_op_imul: {
1348       if (dst.regClass() == v1) {
1349          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1350                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1351       } else if (dst.regClass() == s1) {
1352          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1353       } else {
1354          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1355          nir_print_instr(&instr->instr, stderr);
1356          fprintf(stderr, "\n");
1357       }
1358       break;
1359    }
1360    case nir_op_umul_high: {
1361       if (dst.regClass() == v1) {
1362          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1364          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1365       } else if (dst.regClass() == s1) {
1366          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1367                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1368          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1369       } else {
1370          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1371          nir_print_instr(&instr->instr, stderr);
1372          fprintf(stderr, "\n");
1373       }
1374       break;
1375    }
1376    case nir_op_imul_high: {
1377       if (dst.regClass() == v1) {
1378          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1380          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1381       } else if (dst.regClass() == s1) {
1382          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1383                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1384          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1385       } else {
1386          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1387          nir_print_instr(&instr->instr, stderr);
1388          fprintf(stderr, "\n");
1389       }
1390       break;
1391    }
1392    case nir_op_fmul: {
1393       if (dst.size() == 1) {
1394          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1395       } else if (dst.size() == 2) {
1396          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1397                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1398       } else {
1399          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1400          nir_print_instr(&instr->instr, stderr);
1401          fprintf(stderr, "\n");
1402       }
1403       break;
1404    }
1405    case nir_op_fadd: {
1406       if (dst.size() == 1) {
1407          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1408       } else if (dst.size() == 2) {
1409          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1410                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1411       } else {
1412          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1413          nir_print_instr(&instr->instr, stderr);
1414          fprintf(stderr, "\n");
1415       }
1416       break;
1417    }
1418    case nir_op_fsub: {
1419       Temp src0 = get_alu_src(ctx, instr->src[0]);
1420       Temp src1 = get_alu_src(ctx, instr->src[1]);
1421       if (dst.size() == 1) {
1422          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1424          else
1425             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1426       } else if (dst.size() == 2) {
1427          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1428                                      get_alu_src(ctx, instr->src[0]),
1429                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1430          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1431          sub->neg[1] = true;
1432       } else {
1433          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1434          nir_print_instr(&instr->instr, stderr);
1435          fprintf(stderr, "\n");
1436       }
1437       break;
1438    }
1439    case nir_op_fmax: {
1440       if (dst.size() == 1) {
1441          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1442       } else if (dst.size() == 2) {
1443          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1444             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1445                                 get_alu_src(ctx, instr->src[0]),
1446                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1447             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1448          } else {
1449             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1450                      get_alu_src(ctx, instr->src[0]),
1451                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1452          }
1453       } else {
1454          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1455          nir_print_instr(&instr->instr, stderr);
1456          fprintf(stderr, "\n");
1457       }
1458       break;
1459    }
1460    case nir_op_fmin: {
1461       if (dst.size() == 1) {
1462          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1463       } else if (dst.size() == 2) {
1464          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1465             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1466                                 get_alu_src(ctx, instr->src[0]),
1467                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1468             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1469          } else {
1470             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1471                      get_alu_src(ctx, instr->src[0]),
1472                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1473          }
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_fmax3: {
1482       if (dst.size() == 1) {
1483          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fmin3: {
1492       if (dst.size() == 1) {
1493          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1494       } else {
1495          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1496          nir_print_instr(&instr->instr, stderr);
1497          fprintf(stderr, "\n");
1498       }
1499       break;
1500    }
1501    case nir_op_fmed3: {
1502       if (dst.size() == 1) {
1503          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1504       } else {
1505          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1506          nir_print_instr(&instr->instr, stderr);
1507          fprintf(stderr, "\n");
1508       }
1509       break;
1510    }
1511    case nir_op_umax3: {
1512       if (dst.size() == 1) {
1513          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1514       } else {
1515          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1516          nir_print_instr(&instr->instr, stderr);
1517          fprintf(stderr, "\n");
1518       }
1519       break;
1520    }
1521    case nir_op_umin3: {
1522       if (dst.size() == 1) {
1523          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1524       } else {
1525          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1526          nir_print_instr(&instr->instr, stderr);
1527          fprintf(stderr, "\n");
1528       }
1529       break;
1530    }
1531    case nir_op_umed3: {
1532       if (dst.size() == 1) {
1533          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1534       } else {
1535          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1536          nir_print_instr(&instr->instr, stderr);
1537          fprintf(stderr, "\n");
1538       }
1539       break;
1540    }
1541    case nir_op_imax3: {
1542       if (dst.size() == 1) {
1543          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_imin3: {
1552       if (dst.size() == 1) {
1553          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1554       } else {
1555          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1556          nir_print_instr(&instr->instr, stderr);
1557          fprintf(stderr, "\n");
1558       }
1559       break;
1560    }
1561    case nir_op_imed3: {
1562       if (dst.size() == 1) {
1563          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1564       } else {
1565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1566          nir_print_instr(&instr->instr, stderr);
1567          fprintf(stderr, "\n");
1568       }
1569       break;
1570    }
1571    case nir_op_cube_face_coord: {
1572       Temp in = get_alu_src(ctx, instr->src[0], 3);
1573       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1574                       emit_extract_vector(ctx, in, 1, v1),
1575                       emit_extract_vector(ctx, in, 2, v1) };
1576       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1577       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1578       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1579       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1580       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1581       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1582       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1583       break;
1584    }
1585    case nir_op_cube_face_index: {
1586       Temp in = get_alu_src(ctx, instr->src[0], 3);
1587       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1588                       emit_extract_vector(ctx, in, 1, v1),
1589                       emit_extract_vector(ctx, in, 2, v1) };
1590       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1591       break;
1592    }
1593    case nir_op_bcsel: {
1594       emit_bcsel(ctx, instr, dst);
1595       break;
1596    }
1597    case nir_op_frsq: {
1598       if (dst.size() == 1) {
1599          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1600       } else if (dst.size() == 2) {
1601          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1602       } else {
1603          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1604          nir_print_instr(&instr->instr, stderr);
1605          fprintf(stderr, "\n");
1606       }
1607       break;
1608    }
1609    case nir_op_fneg: {
1610       Temp src = get_alu_src(ctx, instr->src[0]);
1611       if (dst.size() == 1) {
1612          if (ctx->block->fp_mode.must_flush_denorms32)
1613             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1614          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1615       } else if (dst.size() == 2) {
1616          if (ctx->block->fp_mode.must_flush_denorms16_64)
1617             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1618          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1619          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1620          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1621          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1622       } else {
1623          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1624          nir_print_instr(&instr->instr, stderr);
1625          fprintf(stderr, "\n");
1626       }
1627       break;
1628    }
1629    case nir_op_fabs: {
1630       Temp src = get_alu_src(ctx, instr->src[0]);
1631       if (dst.size() == 1) {
1632          if (ctx->block->fp_mode.must_flush_denorms32)
1633             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1634          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1635       } else if (dst.size() == 2) {
1636          if (ctx->block->fp_mode.must_flush_denorms16_64)
1637             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1638          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1639          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1640          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1642       } else {
1643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1644          nir_print_instr(&instr->instr, stderr);
1645          fprintf(stderr, "\n");
1646       }
1647       break;
1648    }
1649    case nir_op_fsat: {
1650       Temp src = get_alu_src(ctx, instr->src[0]);
1651       if (dst.size() == 1) {
1652          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1653          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1654          // TODO: confirm that this holds under any circumstances
1655       } else if (dst.size() == 2) {
1656          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1657          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1658          vop3->clamp = true;
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_flog2: {
1667       if (dst.size() == 1) {
1668          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_frcp: {
1677       if (dst.size() == 1) {
1678          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1679       } else if (dst.size() == 2) {
1680          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1681       } else {
1682          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1683          nir_print_instr(&instr->instr, stderr);
1684          fprintf(stderr, "\n");
1685       }
1686       break;
1687    }
1688    case nir_op_fexp2: {
1689       if (dst.size() == 1) {
1690          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1691       } else {
1692          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1693          nir_print_instr(&instr->instr, stderr);
1694          fprintf(stderr, "\n");
1695       }
1696       break;
1697    }
1698    case nir_op_fsqrt: {
1699       if (dst.size() == 1) {
1700          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1701       } else if (dst.size() == 2) {
1702          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1703       } else {
1704          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1705          nir_print_instr(&instr->instr, stderr);
1706          fprintf(stderr, "\n");
1707       }
1708       break;
1709    }
1710    case nir_op_ffract: {
1711       if (dst.size() == 1) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1713       } else if (dst.size() == 2) {
1714          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1715       } else {
1716          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1717          nir_print_instr(&instr->instr, stderr);
1718          fprintf(stderr, "\n");
1719       }
1720       break;
1721    }
1722    case nir_op_ffloor: {
1723       if (dst.size() == 1) {
1724          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1725       } else if (dst.size() == 2) {
1726          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1727       } else {
1728          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1729          nir_print_instr(&instr->instr, stderr);
1730          fprintf(stderr, "\n");
1731       }
1732       break;
1733    }
1734    case nir_op_fceil: {
1735       if (dst.size() == 1) {
1736          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1737       } else if (dst.size() == 2) {
1738          if (ctx->options->chip_class >= GFX7) {
1739             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1740          } else {
1741             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1742             Temp src0 = get_alu_src(ctx, instr->src[0]);
1743
1744             /* trunc = trunc(src0)
1745              * if (src0 > 0.0 && src0 != trunc)
1746              *    trunc += 1.0
1747              */
1748             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1749             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1750             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1751             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1752             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1753             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1754             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1755          }
1756       } else {
1757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1758          nir_print_instr(&instr->instr, stderr);
1759          fprintf(stderr, "\n");
1760       }
1761       break;
1762    }
1763    case nir_op_ftrunc: {
1764       if (dst.size() == 1) {
1765          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1766       } else if (dst.size() == 2) {
1767          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1768       } else {
1769          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1770          nir_print_instr(&instr->instr, stderr);
1771          fprintf(stderr, "\n");
1772       }
1773       break;
1774    }
1775    case nir_op_fround_even: {
1776       if (dst.size() == 1) {
1777          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1778       } else if (dst.size() == 2) {
1779          if (ctx->options->chip_class >= GFX7) {
1780             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1781          } else {
1782             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1783             Temp src0 = get_alu_src(ctx, instr->src[0]);
1784
1785             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1786             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1787
1788             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1789             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1790             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1791             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1792             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1793             tmp = sub->definitions[0].getTemp();
1794
1795             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1796             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1797             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1798             Temp cond = vop3->definitions[0].getTemp();
1799
1800             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1801             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1802             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1803             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1804
1805             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1806          }
1807       } else {
1808          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1809          nir_print_instr(&instr->instr, stderr);
1810          fprintf(stderr, "\n");
1811       }
1812       break;
1813    }
1814    case nir_op_fsin:
1815    case nir_op_fcos: {
1816       Temp src = get_alu_src(ctx, instr->src[0]);
1817       aco_ptr<Instruction> norm;
1818       if (dst.size() == 1) {
1819          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1820          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1821
1822          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1823          if (ctx->options->chip_class < GFX9)
1824             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1825
1826          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1827          bld.vop1(opcode, Definition(dst), tmp);
1828       } else {
1829          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1830          nir_print_instr(&instr->instr, stderr);
1831          fprintf(stderr, "\n");
1832       }
1833       break;
1834    }
1835    case nir_op_ldexp: {
1836       if (dst.size() == 1) {
1837          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1838                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1839                   get_alu_src(ctx, instr->src[1]));
1840       } else if (dst.size() == 2) {
1841          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1842                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1843                   get_alu_src(ctx, instr->src[1]));
1844       } else {
1845          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1846          nir_print_instr(&instr->instr, stderr);
1847          fprintf(stderr, "\n");
1848       }
1849       break;
1850    }
1851    case nir_op_frexp_sig: {
1852       if (dst.size() == 1) {
1853          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1854                   get_alu_src(ctx, instr->src[0]));
1855       } else if (dst.size() == 2) {
1856          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1857                   get_alu_src(ctx, instr->src[0]));
1858       } else {
1859          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1860          nir_print_instr(&instr->instr, stderr);
1861          fprintf(stderr, "\n");
1862       }
1863       break;
1864    }
1865    case nir_op_frexp_exp: {
1866       if (instr->src[0].src.ssa->bit_size == 32) {
1867          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1868                   get_alu_src(ctx, instr->src[0]));
1869       } else if (instr->src[0].src.ssa->bit_size == 64) {
1870          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1871                   get_alu_src(ctx, instr->src[0]));
1872       } else {
1873          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1874          nir_print_instr(&instr->instr, stderr);
1875          fprintf(stderr, "\n");
1876       }
1877       break;
1878    }
1879    case nir_op_fsign: {
1880       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1881       if (dst.size() == 1) {
1882          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1884          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1885          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1886       } else if (dst.size() == 2) {
1887          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1888          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1889          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1890
1891          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1892          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1893          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1894
1895          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1896       } else {
1897          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1898          nir_print_instr(&instr->instr, stderr);
1899          fprintf(stderr, "\n");
1900       }
1901       break;
1902    }
1903    case nir_op_f2f32: {
1904       if (instr->src[0].src.ssa->bit_size == 64) {
1905          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1906       } else {
1907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1908          nir_print_instr(&instr->instr, stderr);
1909          fprintf(stderr, "\n");
1910       }
1911       break;
1912    }
1913    case nir_op_f2f64: {
1914       if (instr->src[0].src.ssa->bit_size == 32) {
1915          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1916       } else {
1917          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1918          nir_print_instr(&instr->instr, stderr);
1919          fprintf(stderr, "\n");
1920       }
1921       break;
1922    }
1923    case nir_op_i2f32: {
1924       assert(dst.size() == 1);
1925       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1926       break;
1927    }
1928    case nir_op_i2f64: {
1929       if (instr->src[0].src.ssa->bit_size == 32) {
1930          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1931       } else if (instr->src[0].src.ssa->bit_size == 64) {
1932          Temp src = get_alu_src(ctx, instr->src[0]);
1933          RegClass rc = RegClass(src.type(), 1);
1934          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1935          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1936          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1937          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1938          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1939          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1940
1941       } else {
1942          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1943          nir_print_instr(&instr->instr, stderr);
1944          fprintf(stderr, "\n");
1945       }
1946       break;
1947    }
1948    case nir_op_u2f32: {
1949       assert(dst.size() == 1);
1950       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1951       break;
1952    }
1953    case nir_op_u2f64: {
1954       if (instr->src[0].src.ssa->bit_size == 32) {
1955          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1956       } else if (instr->src[0].src.ssa->bit_size == 64) {
1957          Temp src = get_alu_src(ctx, instr->src[0]);
1958          RegClass rc = RegClass(src.type(), 1);
1959          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1960          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1961          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1962          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1963          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1964          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1965       } else {
1966          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1967          nir_print_instr(&instr->instr, stderr);
1968          fprintf(stderr, "\n");
1969       }
1970       break;
1971    }
1972    case nir_op_f2i32: {
1973       Temp src = get_alu_src(ctx, instr->src[0]);
1974       if (instr->src[0].src.ssa->bit_size == 32) {
1975          if (dst.type() == RegType::vgpr)
1976             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1977          else
1978             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1979                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1980
1981       } else if (instr->src[0].src.ssa->bit_size == 64) {
1982          if (dst.type() == RegType::vgpr)
1983             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1984          else
1985             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1986                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1987
1988       } else {
1989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1990          nir_print_instr(&instr->instr, stderr);
1991          fprintf(stderr, "\n");
1992       }
1993       break;
1994    }
1995    case nir_op_f2u32: {
1996       Temp src = get_alu_src(ctx, instr->src[0]);
1997       if (instr->src[0].src.ssa->bit_size == 32) {
1998          if (dst.type() == RegType::vgpr)
1999             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2000          else
2001             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2002                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2003
2004       } else if (instr->src[0].src.ssa->bit_size == 64) {
2005          if (dst.type() == RegType::vgpr)
2006             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2007          else
2008             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2009                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2010
2011       } else {
2012          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2013          nir_print_instr(&instr->instr, stderr);
2014          fprintf(stderr, "\n");
2015       }
2016       break;
2017    }
2018    case nir_op_f2i64: {
2019       Temp src = get_alu_src(ctx, instr->src[0]);
2020       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2021          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2022          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2023          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2024          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2025          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2026          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2027          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2028          Temp new_exponent = bld.tmp(v1);
2029          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2030          if (ctx->program->chip_class >= GFX8)
2031             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2032          else
2033             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2034          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2035          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2036          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2037          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2038          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2039          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2040          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2041          Temp new_lower = bld.tmp(v1);
2042          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2043          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2044          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2045
2046       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2047          if (src.type() == RegType::vgpr)
2048             src = bld.as_uniform(src);
2049          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2050          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2051          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2052          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2053          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2054          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2055          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2056          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2057          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2058          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2059          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2060          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2061          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2062          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2063          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2064          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2065          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2066          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2067          Temp borrow = bld.tmp(s1);
2068          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2069          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2070          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2071
2072       } else if (instr->src[0].src.ssa->bit_size == 64) {
2073          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2074          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2075          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2076          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2077          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2078          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2079          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2080          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2081          if (dst.type() == RegType::sgpr) {
2082             lower = bld.as_uniform(lower);
2083             upper = bld.as_uniform(upper);
2084          }
2085          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2086
2087       } else {
2088          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2089          nir_print_instr(&instr->instr, stderr);
2090          fprintf(stderr, "\n");
2091       }
2092       break;
2093    }
2094    case nir_op_f2u64: {
2095       Temp src = get_alu_src(ctx, instr->src[0]);
2096       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2097          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2098          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2099          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2100          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2101          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2102          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2103          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2104          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2105          Temp new_exponent = bld.tmp(v1);
2106          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2107          if (ctx->program->chip_class >= GFX8)
2108             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2109          else
2110             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2111          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2112          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2114          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2115          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2116          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2117          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2118
2119       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2120          if (src.type() == RegType::vgpr)
2121             src = bld.as_uniform(src);
2122          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2123          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2124          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2125          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2126          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2127          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2128          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2129          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2130          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2131          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2132          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2133          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2134          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2135          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2136          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2137          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2138          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2139          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2140
2141       } else if (instr->src[0].src.ssa->bit_size == 64) {
2142          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2143          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2144          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2145          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2146          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2147          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2148          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2149          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2150          if (dst.type() == RegType::sgpr) {
2151             lower = bld.as_uniform(lower);
2152             upper = bld.as_uniform(upper);
2153          }
2154          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2155
2156       } else {
2157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2158          nir_print_instr(&instr->instr, stderr);
2159          fprintf(stderr, "\n");
2160       }
2161       break;
2162    }
2163    case nir_op_b2f32: {
2164       Temp src = get_alu_src(ctx, instr->src[0]);
2165       assert(src.regClass() == bld.lm);
2166
2167       if (dst.regClass() == s1) {
2168          src = bool_to_scalar_condition(ctx, src);
2169          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2170       } else if (dst.regClass() == v1) {
2171          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2172       } else {
2173          unreachable("Wrong destination register class for nir_op_b2f32.");
2174       }
2175       break;
2176    }
2177    case nir_op_b2f64: {
2178       Temp src = get_alu_src(ctx, instr->src[0]);
2179       assert(src.regClass() == bld.lm);
2180
2181       if (dst.regClass() == s2) {
2182          src = bool_to_scalar_condition(ctx, src);
2183          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2184       } else if (dst.regClass() == v2) {
2185          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2186          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2187          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2188       } else {
2189          unreachable("Wrong destination register class for nir_op_b2f64.");
2190       }
2191       break;
2192    }
2193    case nir_op_i2i32: {
2194       Temp src = get_alu_src(ctx, instr->src[0]);
2195       if (instr->src[0].src.ssa->bit_size == 64) {
2196          /* we can actually just say dst = src, as it would map the lower register */
2197          emit_extract_vector(ctx, src, 0, dst);
2198       } else {
2199          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2200          nir_print_instr(&instr->instr, stderr);
2201          fprintf(stderr, "\n");
2202       }
2203       break;
2204    }
2205    case nir_op_u2u32: {
2206       Temp src = get_alu_src(ctx, instr->src[0]);
2207       if (instr->src[0].src.ssa->bit_size == 16) {
2208          if (dst.regClass() == s1) {
2209             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2210          } else {
2211             // TODO: do better with SDWA
2212             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2213          }
2214       } else if (instr->src[0].src.ssa->bit_size == 64) {
2215          /* we can actually just say dst = src, as it would map the lower register */
2216          emit_extract_vector(ctx, src, 0, dst);
2217       } else {
2218          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2219          nir_print_instr(&instr->instr, stderr);
2220          fprintf(stderr, "\n");
2221       }
2222       break;
2223    }
2224    case nir_op_i2i64: {
2225       Temp src = get_alu_src(ctx, instr->src[0]);
2226       if (src.regClass() == s1) {
2227          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2228          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2229       } else if (src.regClass() == v1) {
2230          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2231          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2232       } else {
2233          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2234          nir_print_instr(&instr->instr, stderr);
2235          fprintf(stderr, "\n");
2236       }
2237       break;
2238    }
2239    case nir_op_u2u64: {
2240       Temp src = get_alu_src(ctx, instr->src[0]);
2241       if (instr->src[0].src.ssa->bit_size == 32) {
2242          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2243       } else {
2244          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2245          nir_print_instr(&instr->instr, stderr);
2246          fprintf(stderr, "\n");
2247       }
2248       break;
2249    }
2250    case nir_op_b2i32: {
2251       Temp src = get_alu_src(ctx, instr->src[0]);
2252       assert(src.regClass() == bld.lm);
2253
2254       if (dst.regClass() == s1) {
2255          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2256          bool_to_scalar_condition(ctx, src, dst);
2257       } else if (dst.regClass() == v1) {
2258          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2259       } else {
2260          unreachable("Invalid register class for b2i32");
2261       }
2262       break;
2263    }
2264    case nir_op_i2b1: {
2265       Temp src = get_alu_src(ctx, instr->src[0]);
2266       assert(dst.regClass() == bld.lm);
2267
2268       if (src.type() == RegType::vgpr) {
2269          assert(src.regClass() == v1 || src.regClass() == v2);
2270          assert(dst.regClass() == bld.lm);
2271          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2272                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2273       } else {
2274          assert(src.regClass() == s1 || src.regClass() == s2);
2275          Temp tmp;
2276          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2277             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2278          } else {
2279             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2280                            bld.scc(bld.def(s1)), Operand(0u), src);
2281          }
2282          bool_to_vector_condition(ctx, tmp, dst);
2283       }
2284       break;
2285    }
2286    case nir_op_pack_64_2x32_split: {
2287       Temp src0 = get_alu_src(ctx, instr->src[0]);
2288       Temp src1 = get_alu_src(ctx, instr->src[1]);
2289
2290       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2291       break;
2292    }
2293    case nir_op_unpack_64_2x32_split_x:
2294       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2295       break;
2296    case nir_op_unpack_64_2x32_split_y:
2297       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2298       break;
2299    case nir_op_pack_half_2x16: {
2300       Temp src = get_alu_src(ctx, instr->src[0], 2);
2301
2302       if (dst.regClass() == v1) {
2303          Temp src0 = bld.tmp(v1);
2304          Temp src1 = bld.tmp(v1);
2305          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2306          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2307             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2308          else
2309             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2310                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2311                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2312       } else {
2313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2314          nir_print_instr(&instr->instr, stderr);
2315          fprintf(stderr, "\n");
2316       }
2317       break;
2318    }
2319    case nir_op_unpack_half_2x16_split_x: {
2320       if (dst.regClass() == v1) {
2321          Builder bld(ctx->program, ctx->block);
2322          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2323       } else {
2324          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2325          nir_print_instr(&instr->instr, stderr);
2326          fprintf(stderr, "\n");
2327       }
2328       break;
2329    }
2330    case nir_op_unpack_half_2x16_split_y: {
2331       if (dst.regClass() == v1) {
2332          Builder bld(ctx->program, ctx->block);
2333          /* TODO: use SDWA here */
2334          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2335                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2336       } else {
2337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2338          nir_print_instr(&instr->instr, stderr);
2339          fprintf(stderr, "\n");
2340       }
2341       break;
2342    }
2343    case nir_op_fquantize2f16: {
2344       Temp src = get_alu_src(ctx, instr->src[0]);
2345       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2346       Temp f32, cmp_res;
2347
2348       if (ctx->program->chip_class >= GFX8) {
2349          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2350          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2351          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2352       } else {
2353          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2354           * so compare the result and flush to 0 if it's smaller.
2355           */
2356          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2357          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2358          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2359          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2360          cmp_res = vop3->definitions[0].getTemp();
2361       }
2362
2363       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2364          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2366       } else {
2367          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2368       }
2369       break;
2370    }
2371    case nir_op_bfm: {
2372       Temp bits = get_alu_src(ctx, instr->src[0]);
2373       Temp offset = get_alu_src(ctx, instr->src[1]);
2374
2375       if (dst.regClass() == s1) {
2376          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2377       } else if (dst.regClass() == v1) {
2378          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2379       } else {
2380          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2381          nir_print_instr(&instr->instr, stderr);
2382          fprintf(stderr, "\n");
2383       }
2384       break;
2385    }
2386    case nir_op_bitfield_select: {
2387       /* (mask & insert) | (~mask & base) */
2388       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2389       Temp insert = get_alu_src(ctx, instr->src[1]);
2390       Temp base = get_alu_src(ctx, instr->src[2]);
2391
2392       /* dst = (insert & bitmask) | (base & ~bitmask) */
2393       if (dst.regClass() == s1) {
2394          aco_ptr<Instruction> sop2;
2395          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2396          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2397          Operand lhs;
2398          if (const_insert && const_bitmask) {
2399             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2400          } else {
2401             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2402             lhs = Operand(insert);
2403          }
2404
2405          Operand rhs;
2406          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2407          if (const_base && const_bitmask) {
2408             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2409          } else {
2410             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2411             rhs = Operand(base);
2412          }
2413
2414          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2415
2416       } else if (dst.regClass() == v1) {
2417          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2418             base = as_vgpr(ctx, base);
2419          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2420             insert = as_vgpr(ctx, insert);
2421
2422          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2423
2424       } else {
2425          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2426          nir_print_instr(&instr->instr, stderr);
2427          fprintf(stderr, "\n");
2428       }
2429       break;
2430    }
2431    case nir_op_ubfe:
2432    case nir_op_ibfe: {
2433       Temp base = get_alu_src(ctx, instr->src[0]);
2434       Temp offset = get_alu_src(ctx, instr->src[1]);
2435       Temp bits = get_alu_src(ctx, instr->src[2]);
2436
2437       if (dst.type() == RegType::sgpr) {
2438          Operand extract;
2439          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2440          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2441          if (const_offset && const_bits) {
2442             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2443             extract = Operand(const_extract);
2444          } else {
2445             Operand width;
2446             if (const_bits) {
2447                width = Operand(const_bits->u32 << 16);
2448             } else {
2449                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2450             }
2451             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2452          }
2453
2454          aco_opcode opcode;
2455          if (dst.regClass() == s1) {
2456             if (instr->op == nir_op_ubfe)
2457                opcode = aco_opcode::s_bfe_u32;
2458             else
2459                opcode = aco_opcode::s_bfe_i32;
2460          } else if (dst.regClass() == s2) {
2461             if (instr->op == nir_op_ubfe)
2462                opcode = aco_opcode::s_bfe_u64;
2463             else
2464                opcode = aco_opcode::s_bfe_i64;
2465          } else {
2466             unreachable("Unsupported BFE bit size");
2467          }
2468
2469          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2470
2471       } else {
2472          aco_opcode opcode;
2473          if (dst.regClass() == v1) {
2474             if (instr->op == nir_op_ubfe)
2475                opcode = aco_opcode::v_bfe_u32;
2476             else
2477                opcode = aco_opcode::v_bfe_i32;
2478          } else {
2479             unreachable("Unsupported BFE bit size");
2480          }
2481
2482          emit_vop3a_instruction(ctx, instr, opcode, dst);
2483       }
2484       break;
2485    }
2486    case nir_op_bit_count: {
2487       Temp src = get_alu_src(ctx, instr->src[0]);
2488       if (src.regClass() == s1) {
2489          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2490       } else if (src.regClass() == v1) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2492       } else if (src.regClass() == v2) {
2493          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2494                   emit_extract_vector(ctx, src, 1, v1),
2495                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2496                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2497       } else if (src.regClass() == s2) {
2498          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2499       } else {
2500          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2501          nir_print_instr(&instr->instr, stderr);
2502          fprintf(stderr, "\n");
2503       }
2504       break;
2505    }
2506    case nir_op_flt: {
2507       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2508       break;
2509    }
2510    case nir_op_fge: {
2511       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2512       break;
2513    }
2514    case nir_op_feq: {
2515       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2516       break;
2517    }
2518    case nir_op_fne: {
2519       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2520       break;
2521    }
2522    case nir_op_ilt: {
2523       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2524       break;
2525    }
2526    case nir_op_ige: {
2527       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2528       break;
2529    }
2530    case nir_op_ieq: {
2531       if (instr->src[0].src.ssa->bit_size == 1)
2532          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2533       else
2534          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2535                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2536       break;
2537    }
2538    case nir_op_ine: {
2539       if (instr->src[0].src.ssa->bit_size == 1)
2540          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2541       else
2542          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2543                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2544       break;
2545    }
2546    case nir_op_ult: {
2547       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2548       break;
2549    }
2550    case nir_op_uge: {
2551       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2552       break;
2553    }
2554    case nir_op_fddx:
2555    case nir_op_fddy:
2556    case nir_op_fddx_fine:
2557    case nir_op_fddy_fine:
2558    case nir_op_fddx_coarse:
2559    case nir_op_fddy_coarse: {
2560       Temp src = get_alu_src(ctx, instr->src[0]);
2561       uint16_t dpp_ctrl1, dpp_ctrl2;
2562       if (instr->op == nir_op_fddx_fine) {
2563          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2564          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2565       } else if (instr->op == nir_op_fddy_fine) {
2566          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2567          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2568       } else {
2569          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2570          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2571             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2572          else
2573             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2574       }
2575
2576       Temp tmp;
2577       if (ctx->program->chip_class >= GFX8) {
2578          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2579          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2580       } else {
2581          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2582          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2583          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2584       }
2585       emit_wqm(ctx, tmp, dst, true);
2586       break;
2587    }
2588    default:
2589       fprintf(stderr, "Unknown NIR ALU instr: ");
2590       nir_print_instr(&instr->instr, stderr);
2591       fprintf(stderr, "\n");
2592    }
2593 }
2594
2595 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2596 {
2597    Temp dst = get_ssa_temp(ctx, &instr->def);
2598
2599    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2600    // which get truncated the lsb if double and msb if int
2601    // for now, we only use s_mov_b64 with 64bit inline constants
2602    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2603    assert(dst.type() == RegType::sgpr);
2604
2605    Builder bld(ctx->program, ctx->block);
2606
2607    if (instr->def.bit_size == 1) {
2608       assert(dst.regClass() == bld.lm);
2609       int val = instr->value[0].b ? -1 : 0;
2610       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2611       bld.sop1(Builder::s_mov, Definition(dst), op);
2612    } else if (dst.size() == 1) {
2613       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2614    } else {
2615       assert(dst.size() != 1);
2616       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2617       if (instr->def.bit_size == 64)
2618          for (unsigned i = 0; i < dst.size(); i++)
2619             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2620       else {
2621          for (unsigned i = 0; i < dst.size(); i++)
2622             vec->operands[i] = Operand{instr->value[i].u32};
2623       }
2624       vec->definitions[0] = Definition(dst);
2625       ctx->block->instructions.emplace_back(std::move(vec));
2626    }
2627 }
2628
2629 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2630 {
2631    uint32_t new_mask = 0;
2632    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2633       if (mask & (1u << i))
2634          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2635    return new_mask;
2636 }
2637
2638 Operand load_lds_size_m0(isel_context *ctx)
2639 {
2640    /* TODO: m0 does not need to be initialized on GFX9+ */
2641    Builder bld(ctx->program, ctx->block);
2642    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2643 }
2644
2645 Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2646               Temp address, unsigned base_offset, unsigned align)
2647 {
2648    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2649
2650    Builder bld(ctx->program, ctx->block);
2651
2652    Operand m = load_lds_size_m0(ctx);
2653
2654    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2655    unsigned bytes_read = 0;
2656    unsigned result_size = 0;
2657    unsigned total_bytes = num_components * elem_size_bytes;
2658    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2659    bool large_ds_read = ctx->options->chip_class >= GFX7;
2660    bool usable_read2 = ctx->options->chip_class >= GFX7;
2661
2662    while (bytes_read < total_bytes) {
2663       unsigned todo = total_bytes - bytes_read;
2664       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2665       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2666
2667       aco_opcode op = aco_opcode::last_opcode;
2668       bool read2 = false;
2669       if (todo >= 16 && aligned16 && large_ds_read) {
2670          op = aco_opcode::ds_read_b128;
2671          todo = 16;
2672       } else if (todo >= 16 && aligned8 && usable_read2) {
2673          op = aco_opcode::ds_read2_b64;
2674          read2 = true;
2675          todo = 16;
2676       } else if (todo >= 12 && aligned16 && large_ds_read) {
2677          op = aco_opcode::ds_read_b96;
2678          todo = 12;
2679       } else if (todo >= 8 && aligned8) {
2680          op = aco_opcode::ds_read_b64;
2681          todo = 8;
2682       } else if (todo >= 8 && usable_read2) {
2683          op = aco_opcode::ds_read2_b32;
2684          read2 = true;
2685          todo = 8;
2686       } else if (todo >= 4) {
2687          op = aco_opcode::ds_read_b32;
2688          todo = 4;
2689       } else {
2690          assert(false);
2691       }
2692       assert(todo % elem_size_bytes == 0);
2693       unsigned num_elements = todo / elem_size_bytes;
2694       unsigned offset = base_offset + bytes_read;
2695       unsigned max_offset = read2 ? 1019 : 65535;
2696
2697       Temp address_offset = address;
2698       if (offset > max_offset) {
2699          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2700          offset = bytes_read;
2701       }
2702       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2703
2704       Temp res;
2705       if (num_components == 1 && dst.type() == RegType::vgpr)
2706          res = dst;
2707       else
2708          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2709
2710       if (read2)
2711          res = bld.ds(op, Definition(res), address_offset, m, offset / (todo / 2), (offset / (todo / 2)) + 1);
2712       else
2713          res = bld.ds(op, Definition(res), address_offset, m, offset);
2714
2715       if (num_components == 1) {
2716          assert(todo == total_bytes);
2717          if (dst.type() == RegType::sgpr)
2718             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2719          return dst;
2720       }
2721
2722       if (dst.type() == RegType::sgpr) {
2723          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2724          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2725          res = new_res;
2726       }
2727
2728       if (num_elements == 1) {
2729          result[result_size++] = res;
2730       } else {
2731          assert(res != dst && res.size() % num_elements == 0);
2732          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2733          split->operands[0] = Operand(res);
2734          for (unsigned i = 0; i < num_elements; i++)
2735             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2736          ctx->block->instructions.emplace_back(std::move(split));
2737       }
2738
2739       bytes_read += todo;
2740    }
2741
2742    assert(result_size == num_components && result_size > 1);
2743    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2744    for (unsigned i = 0; i < result_size; i++)
2745       vec->operands[i] = Operand(result[i]);
2746    vec->definitions[0] = Definition(dst);
2747    ctx->block->instructions.emplace_back(std::move(vec));
2748    ctx->allocated_vec.emplace(dst.id(), result);
2749
2750    return dst;
2751 }
2752
2753 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2754 {
2755    if (start == 0 && size == data.size())
2756       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2757
2758    unsigned size_hint = 1;
2759    auto it = ctx->allocated_vec.find(data.id());
2760    if (it != ctx->allocated_vec.end())
2761       size_hint = it->second[0].size();
2762    if (size % size_hint || start % size_hint)
2763       size_hint = 1;
2764
2765    start /= size_hint;
2766    size /= size_hint;
2767
2768    Temp elems[size];
2769    for (unsigned i = 0; i < size; i++)
2770       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2771
2772    if (size == 1)
2773       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2774
2775    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2776    for (unsigned i = 0; i < size; i++)
2777       vec->operands[i] = Operand(elems[i]);
2778    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2779    vec->definitions[0] = Definition(res);
2780    ctx->block->instructions.emplace_back(std::move(vec));
2781    return res;
2782 }
2783
2784 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2785 {
2786    Builder bld(ctx->program, ctx->block);
2787    unsigned bytes_written = 0;
2788    bool large_ds_write = ctx->options->chip_class >= GFX7;
2789    bool usable_write2 = ctx->options->chip_class >= GFX7;
2790
2791    while (bytes_written < total_size * 4) {
2792       unsigned todo = total_size * 4 - bytes_written;
2793       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2794       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2795
2796       aco_opcode op = aco_opcode::last_opcode;
2797       bool write2 = false;
2798       unsigned size = 0;
2799       if (todo >= 16 && aligned16 && large_ds_write) {
2800          op = aco_opcode::ds_write_b128;
2801          size = 4;
2802       } else if (todo >= 16 && aligned8 && usable_write2) {
2803          op = aco_opcode::ds_write2_b64;
2804          write2 = true;
2805          size = 4;
2806       } else if (todo >= 12 && aligned16 && large_ds_write) {
2807          op = aco_opcode::ds_write_b96;
2808          size = 3;
2809       } else if (todo >= 8 && aligned8) {
2810          op = aco_opcode::ds_write_b64;
2811          size = 2;
2812       } else if (todo >= 8 && usable_write2) {
2813          op = aco_opcode::ds_write2_b32;
2814          write2 = true;
2815          size = 2;
2816       } else if (todo >= 4) {
2817          op = aco_opcode::ds_write_b32;
2818          size = 1;
2819       } else {
2820          assert(false);
2821       }
2822
2823       unsigned offset = offset0 + offset1 + bytes_written;
2824       unsigned max_offset = write2 ? 1020 : 65535;
2825       Temp address_offset = address;
2826       if (offset > max_offset) {
2827          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2828          offset = offset1 + bytes_written;
2829       }
2830       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2831
2832       if (write2) {
2833          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2834          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2835          bld.ds(op, address_offset, val0, val1, m, offset / size / 2, (offset / size / 2) + 1);
2836       } else {
2837          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2838          bld.ds(op, address_offset, val, m, offset);
2839       }
2840
2841       bytes_written += size * 4;
2842    }
2843 }
2844
2845 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2846                Temp address, unsigned base_offset, unsigned align)
2847 {
2848    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2849    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
2850
2851    Operand m = load_lds_size_m0(ctx);
2852
2853    /* we need at most two stores, assuming that the writemask is at most 4 bits wide */
2854    assert(wrmask <= 0x0f);
2855    int start[2], count[2];
2856    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2857    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2858    assert(wrmask == 0);
2859
2860    /* one combined store is sufficient */
2861    if (count[0] == count[1] && (align % elem_size_bytes) == 0 && (base_offset % elem_size_bytes) == 0) {
2862       Builder bld(ctx->program, ctx->block);
2863
2864       Temp address_offset = address;
2865       if ((base_offset / elem_size_bytes) + start[1] > 255) {
2866          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2867          base_offset = 0;
2868       }
2869
2870       assert(count[0] == 1);
2871       RegClass xtract_rc(RegType::vgpr, elem_size_bytes / 4);
2872
2873       Temp val0 = emit_extract_vector(ctx, data, start[0], xtract_rc);
2874       Temp val1 = emit_extract_vector(ctx, data, start[1], xtract_rc);
2875       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2876       base_offset = base_offset / elem_size_bytes;
2877       bld.ds(op, address_offset, val0, val1, m,
2878              base_offset + start[0], base_offset + start[1]);
2879       return;
2880    }
2881
2882    for (unsigned i = 0; i < 2; i++) {
2883       if (count[i] == 0)
2884          continue;
2885
2886       unsigned elem_size_words = elem_size_bytes / 4;
2887       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2888                       base_offset, start[i] * elem_size_bytes, align);
2889    }
2890    return;
2891 }
2892
2893 unsigned calculate_lds_alignment(isel_context *ctx, unsigned const_offset)
2894 {
2895    unsigned align = 16;
2896    if (const_offset)
2897       align = std::min(align, 1u << (ffs(const_offset) - 1));
2898
2899    return align;
2900 }
2901
2902
2903 Temp create_vec_from_array(isel_context *ctx, Temp arr[], unsigned cnt, RegType reg_type, unsigned split_cnt = 0u, Temp dst = Temp())
2904 {
2905    Builder bld(ctx->program, ctx->block);
2906
2907    if (!dst.id())
2908       dst = bld.tmp(RegClass(reg_type, cnt * arr[0].size()));
2909
2910    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
2911    aco_ptr<Pseudo_instruction> instr {create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, cnt, 1)};
2912    instr->definitions[0] = Definition(dst);
2913
2914    for (unsigned i = 0; i < cnt; ++i) {
2915       assert(arr[i].size() == arr[0].size());
2916       allocated_vec[i] = arr[i];
2917       instr->operands[i] = Operand(arr[i]);
2918    }
2919
2920    bld.insert(std::move(instr));
2921
2922    if (split_cnt)
2923       emit_split_vector(ctx, dst, split_cnt);
2924    else
2925       ctx->allocated_vec.emplace(dst.id(), allocated_vec); /* emit_split_vector already does this */
2926
2927    return dst;
2928 }
2929
2930 inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, unsigned const_offset)
2931 {
2932    if (const_offset >= 4096) {
2933       unsigned excess_const_offset = const_offset / 4096u * 4096u;
2934       const_offset %= 4096u;
2935
2936       if (!voffset.id())
2937          voffset = bld.copy(bld.def(v1), Operand(excess_const_offset));
2938       else if (unlikely(voffset.regClass() == s1))
2939          voffset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), Operand(excess_const_offset), Operand(voffset));
2940       else if (likely(voffset.regClass() == v1))
2941          voffset = bld.vadd32(bld.def(v1), Operand(voffset), Operand(excess_const_offset));
2942       else
2943          unreachable("Unsupported register class of voffset");
2944    }
2945
2946    return const_offset;
2947 }
2948
2949 void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata,
2950                              unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false)
2951 {
2952    assert(vdata.id());
2953    assert(vdata.size() != 3 || ctx->program->chip_class != GFX6);
2954    assert(vdata.size() >= 1 && vdata.size() <= 4);
2955
2956    Builder bld(ctx->program, ctx->block);
2957    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_store_dword + vdata.size() - 1);
2958    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
2959
2960    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
2961    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
2962    Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset,
2963                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
2964                                  /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc);
2965
2966    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
2967 }
2968
2969 void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset,
2970                                    unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask,
2971                                    bool allow_combining = true, bool reorder = true, bool slc = false)
2972 {
2973    Builder bld(ctx->program, ctx->block);
2974    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
2975    assert(write_mask);
2976
2977    if (elem_size_bytes == 8) {
2978       elem_size_bytes = 4;
2979       write_mask = widen_mask(write_mask, 2);
2980    }
2981
2982    while (write_mask) {
2983       int start = 0;
2984       int count = 0;
2985       u_bit_scan_consecutive_range(&write_mask, &start, &count);
2986       assert(count > 0);
2987       assert(start >= 0);
2988
2989       while (count > 0) {
2990          unsigned sub_count = allow_combining ? MIN2(count, 4) : 1;
2991          unsigned const_offset = (unsigned) start * elem_size_bytes + base_const_offset;
2992
2993          /* GFX6 doesn't have buffer_store_dwordx3, so make sure not to emit that here either. */
2994          if (unlikely(ctx->program->chip_class == GFX6 && sub_count == 3))
2995             sub_count = 2;
2996
2997          Temp elem = extract_subvector(ctx, src, start, sub_count, RegType::vgpr);
2998          emit_single_mubuf_store(ctx, descriptor, voffset, soffset, elem, const_offset, reorder, slc);
2999
3000          count -= sub_count;
3001          start += sub_count;
3002       }
3003
3004       assert(count == 0);
3005    }
3006 }
3007
3008 Temp emit_single_mubuf_load(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset,
3009                             unsigned const_offset, unsigned size_dwords, bool allow_reorder = true)
3010 {
3011    assert(size_dwords != 3 || ctx->program->chip_class != GFX6);
3012    assert(size_dwords >= 1 && size_dwords <= 4);
3013
3014    Builder bld(ctx->program, ctx->block);
3015    Temp vdata = bld.tmp(RegClass(RegType::vgpr, size_dwords));
3016    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_load_dword + size_dwords - 1);
3017    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3018
3019    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3020    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3021    Builder::Result r = bld.mubuf(op, Definition(vdata), Operand(descriptor), voffset_op, soffset_op, const_offset,
3022                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3023                                  /* disable_wqm */ false, /* glc */ true,
3024                                  /* dlc*/ ctx->program->chip_class >= GFX10, /* slc */ false);
3025
3026    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3027
3028    return vdata;
3029 }
3030
3031 void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, Temp soffset,
3032                      unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components,
3033                      unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true)
3034 {
3035    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3036    assert((num_components * elem_size_bytes / 4) == dst.size());
3037    assert(!!stride != allow_combining);
3038
3039    Builder bld(ctx->program, ctx->block);
3040    unsigned split_cnt = num_components;
3041
3042    if (elem_size_bytes == 8) {
3043       elem_size_bytes = 4;
3044       num_components *= 2;
3045    }
3046
3047    if (!stride)
3048       stride = elem_size_bytes;
3049
3050    unsigned load_size = 1;
3051    if (allow_combining) {
3052       if ((num_components % 4) == 0)
3053          load_size = 4;
3054       else if ((num_components % 3) == 0 && ctx->program->chip_class != GFX6)
3055          load_size = 3;
3056       else if ((num_components % 2) == 0)
3057          load_size = 2;
3058    }
3059
3060    unsigned num_loads = num_components / load_size;
3061    std::array<Temp, NIR_MAX_VEC_COMPONENTS> elems;
3062
3063    for (unsigned i = 0; i < num_loads; ++i) {
3064       unsigned const_offset = i * stride * load_size + base_const_offset;
3065       elems[i] = emit_single_mubuf_load(ctx, descriptor, voffset, soffset, const_offset, load_size, allow_reorder);
3066    }
3067
3068    create_vec_from_array(ctx, elems.data(), num_loads, RegType::vgpr, split_cnt, dst);
3069 }
3070
3071 std::pair<Temp, unsigned> offset_add_from_nir(isel_context *ctx, const std::pair<Temp, unsigned> &base_offset, nir_src *off_src, unsigned stride = 1u)
3072 {
3073    Builder bld(ctx->program, ctx->block);
3074    Temp offset = base_offset.first;
3075    unsigned const_offset = base_offset.second;
3076
3077    if (!nir_src_is_const(*off_src)) {
3078       Temp indirect_offset_arg = get_ssa_temp(ctx, off_src->ssa);
3079       Temp with_stride;
3080
3081       /* Calculate indirect offset with stride */
3082       if (likely(indirect_offset_arg.regClass() == v1))
3083          with_stride = bld.v_mul_imm(bld.def(v1), indirect_offset_arg, stride);
3084       else if (indirect_offset_arg.regClass() == s1)
3085          with_stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), indirect_offset_arg);
3086       else
3087          unreachable("Unsupported register class of indirect offset");
3088
3089       /* Add to the supplied base offset */
3090       if (offset.id() == 0)
3091          offset = with_stride;
3092       else if (unlikely(offset.regClass() == s1 && with_stride.regClass() == s1))
3093          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), with_stride, offset);
3094       else if (offset.size() == 1 && with_stride.size() == 1)
3095          offset = bld.vadd32(bld.def(v1), with_stride, offset);
3096       else
3097          unreachable("Unsupported register class of indirect offset");
3098    } else {
3099       unsigned const_offset_arg = nir_src_as_uint(*off_src);
3100       const_offset += const_offset_arg * stride;
3101    }
3102
3103    return std::make_pair(offset, const_offset);
3104 }
3105
3106 std::pair<Temp, unsigned> offset_add(isel_context *ctx, const std::pair<Temp, unsigned> &off1, const std::pair<Temp, unsigned> &off2)
3107 {
3108    Builder bld(ctx->program, ctx->block);
3109    Temp offset;
3110
3111    if (off1.first.id() && off2.first.id()) {
3112       if (unlikely(off1.first.regClass() == s1 && off2.first.regClass() == s1))
3113          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), off1.first, off2.first);
3114       else if (off1.first.size() == 1 && off2.first.size() == 1)
3115          offset = bld.vadd32(bld.def(v1), off1.first, off2.first);
3116       else
3117          unreachable("Unsupported register class of indirect offset");
3118    } else {
3119       offset = off1.first.id() ? off1.first : off2.first;
3120    }
3121
3122    return std::make_pair(offset, off1.second + off2.second);
3123 }
3124
3125 std::pair<Temp, unsigned> offset_mul(isel_context *ctx, const std::pair<Temp, unsigned> &offs, unsigned multiplier)
3126 {
3127    Builder bld(ctx->program, ctx->block);
3128    unsigned const_offset = offs.second * multiplier;
3129
3130    if (!offs.first.id())
3131       return std::make_pair(offs.first, const_offset);
3132
3133    Temp offset = unlikely(offs.first.regClass() == s1)
3134                  ? bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(multiplier), offs.first)
3135                  : bld.v_mul_imm(bld.def(v1), offs.first, multiplier);
3136
3137    return std::make_pair(offset, const_offset);
3138 }
3139
3140 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride, unsigned component_stride)
3141 {
3142    Builder bld(ctx->program, ctx->block);
3143
3144    /* base is the driver_location, which is already multiplied by 4, so is in dwords */
3145    unsigned const_offset = nir_intrinsic_base(instr) * base_stride;
3146    /* component is in bytes */
3147    const_offset += nir_intrinsic_component(instr) * component_stride;
3148
3149    /* offset should be interpreted in relation to the base, so the instruction effectively reads/writes another input/output when it has an offset */
3150    nir_src *off_src = nir_get_io_offset_src(instr);
3151    return offset_add_from_nir(ctx, std::make_pair(Temp(), const_offset), off_src, 4u * base_stride);
3152 }
3153
3154 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned stride = 1u)
3155 {
3156    return get_intrinsic_io_basic_offset(ctx, instr, stride, stride);
3157 }
3158
3159 Temp get_tess_rel_patch_id(isel_context *ctx)
3160 {
3161    Builder bld(ctx->program, ctx->block);
3162
3163    switch (ctx->shader->info.stage) {
3164    case MESA_SHADER_TESS_CTRL:
3165       return bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffu),
3166                       get_arg(ctx, ctx->args->ac.tcs_rel_ids));
3167    case MESA_SHADER_TESS_EVAL:
3168       return get_arg(ctx, ctx->args->tes_rel_patch_id);
3169    default:
3170       unreachable("Unsupported stage in get_tess_rel_patch_id");
3171    }
3172 }
3173
3174 std::pair<Temp, unsigned> get_tcs_per_vertex_input_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3175 {
3176    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3177    Builder bld(ctx->program, ctx->block);
3178
3179    uint32_t tcs_in_patch_stride = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 4;
3180    uint32_t tcs_in_vertex_stride = ctx->tcs_num_inputs * 4;
3181
3182    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr);
3183
3184    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3185    offs = offset_add_from_nir(ctx, offs, vertex_index_src, tcs_in_vertex_stride);
3186
3187    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3188    Temp tcs_in_current_patch_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, tcs_in_patch_stride);
3189    offs = offset_add(ctx, offs, std::make_pair(tcs_in_current_patch_offset, 0));
3190
3191    return offset_mul(ctx, offs, 4u);
3192 }
3193
3194 std::pair<Temp, unsigned> get_tcs_output_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, bool per_vertex = false)
3195 {
3196    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3197    Builder bld(ctx->program, ctx->block);
3198
3199    uint32_t input_patch_size = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 16;
3200    uint32_t num_tcs_outputs = util_last_bit64(ctx->args->shader_info->tcs.outputs_written);
3201    uint32_t num_tcs_patch_outputs = util_last_bit64(ctx->args->shader_info->tcs.patch_outputs_written);
3202    uint32_t output_vertex_size = num_tcs_outputs * 16;
3203    uint32_t pervertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3204    uint32_t output_patch_stride = pervertex_output_patch_size + num_tcs_patch_outputs * 16;
3205
3206    std::pair<Temp, unsigned> offs = instr
3207                                     ? get_intrinsic_io_basic_offset(ctx, instr, 4u)
3208                                     : std::make_pair(Temp(), 0u);
3209
3210    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3211    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, output_patch_stride);
3212
3213    if (per_vertex) {
3214       assert(instr);
3215
3216       nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3217       offs = offset_add_from_nir(ctx, offs, vertex_index_src, output_vertex_size);
3218
3219       uint32_t output_patch0_offset = (input_patch_size * ctx->tcs_num_patches);
3220       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_offset));
3221    } else {
3222       uint32_t output_patch0_patch_data_offset = (input_patch_size * ctx->tcs_num_patches + pervertex_output_patch_size);
3223       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_patch_data_offset));
3224    }
3225
3226    return offs;
3227 }
3228
3229 std::pair<Temp, unsigned> get_tcs_per_vertex_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3230 {
3231    Builder bld(ctx->program, ctx->block);
3232
3233    unsigned vertices_per_patch = ctx->shader->info.tess.tcs_vertices_out;
3234    unsigned attr_stride = vertices_per_patch * ctx->tcs_num_patches;
3235
3236    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u);
3237
3238    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3239    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, vertices_per_patch * 16u);
3240    offs = offset_add(ctx, offs, std::make_pair(patch_off, 0u));
3241
3242    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3243    offs = offset_add_from_nir(ctx, offs, vertex_index_src, 16u);
3244
3245    return offs;
3246 }
3247
3248 std::pair<Temp, unsigned> get_tcs_per_patch_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, unsigned const_base_offset = 0u)
3249 {
3250    Builder bld(ctx->program, ctx->block);
3251
3252    unsigned num_tcs_outputs = ctx->shader->info.stage == MESA_SHADER_TESS_CTRL
3253                               ? util_last_bit64(ctx->args->shader_info->tcs.outputs_written)
3254                               : ctx->args->options->key.tes.tcs_num_outputs;
3255
3256    unsigned output_vertex_size = num_tcs_outputs * 16;
3257    unsigned per_vertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3258    unsigned per_patch_data_offset = per_vertex_output_patch_size * ctx->tcs_num_patches;
3259    unsigned attr_stride = ctx->tcs_num_patches;
3260
3261    std::pair<Temp, unsigned> offs = instr
3262                                     ? get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u)
3263                                     : std::make_pair(Temp(), 0u);
3264
3265    if (const_base_offset)
3266       offs.second += const_base_offset * attr_stride;
3267
3268    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3269    Temp patch_off = bld.v_mul_imm(bld.def(v1), rel_patch_id, 16u);
3270    offs = offset_add(ctx, offs, std::make_pair(patch_off, per_patch_data_offset));
3271
3272    return offs;
3273 }
3274
3275 void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr)
3276 {
3277    Builder bld(ctx->program, ctx->block);
3278
3279    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, 4u);
3280    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3281    unsigned write_mask = nir_intrinsic_write_mask(instr);
3282    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
3283
3284    if (ctx->stage == vertex_es) {
3285       /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */
3286       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
3287       Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset);
3288       store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true);
3289    } else {
3290       Temp lds_base;
3291
3292       if (ctx->stage == vertex_geometry_gs) {
3293          /* GFX9+: ES stage is merged into GS, data is passed between them using LDS. */
3294          unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3295          Temp thread_id = emit_mbcnt(ctx, bld.def(v1));
3296          Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
3297          Temp vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), thread_id,
3298                                bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
3299          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
3300       } else if (ctx->stage == vertex_ls || ctx->stage == vertex_tess_control_hs) {
3301          /* GFX6-8: VS runs on LS stage when tessellation is used, but LS shares LDS space with HS.
3302           * GFX9+: LS is merged into HS, but still uses the same LDS layout.
3303           */
3304          unsigned num_tcs_inputs = util_last_bit64(ctx->args->shader_info->vs.ls_outputs_written);
3305          Temp vertex_idx = get_arg(ctx, ctx->args->rel_auto_id);
3306          lds_base = bld.v_mul_imm(bld.def(v1), vertex_idx, num_tcs_inputs * 16u);
3307       } else {
3308          unreachable("Invalid LS or ES stage");
3309       }
3310
3311       offs = offset_add(ctx, offs, std::make_pair(lds_base, 0u));
3312       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3313       store_lds(ctx, elem_size_bytes, src, write_mask, offs.first, offs.second, lds_align);
3314    }
3315 }
3316
3317 void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3318 {
3319    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3320    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3321
3322    Builder bld(ctx->program, ctx->block);
3323
3324    Temp store_val = get_ssa_temp(ctx, instr->src[0].ssa);
3325    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3326    unsigned write_mask = nir_intrinsic_write_mask(instr);
3327
3328    /* TODO: Only write to VMEM if the output is per-vertex or it's per-patch non tess factor */
3329    bool write_to_vmem = true;
3330    /* TODO: Only write to LDS if the output is read by the shader, or it's per-patch tess factor */
3331    bool write_to_lds = true;
3332
3333    if (write_to_vmem) {
3334       std::pair<Temp, unsigned> vmem_offs = per_vertex
3335                                             ? get_tcs_per_vertex_output_vmem_offset(ctx, instr)
3336                                             : get_tcs_per_patch_output_vmem_offset(ctx, instr);
3337
3338       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
3339       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
3340       store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, false, false);
3341    }
3342
3343    if (write_to_lds) {
3344       std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3345       unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3346       store_lds(ctx, elem_size_bytes, store_val, write_mask, lds_offs.first, lds_offs.second, lds_align);
3347    }
3348 }
3349
3350 void visit_load_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3351 {
3352    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3353    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3354
3355    Builder bld(ctx->program, ctx->block);
3356
3357    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3358    std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3359    unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3360    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3361
3362    load_lds(ctx, elem_size_bytes, dst, lds_offs.first, lds_offs.second, lds_align);
3363 }
3364
3365 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
3366 {
3367    if (ctx->stage == vertex_vs ||
3368        ctx->stage == tess_eval_vs ||
3369        ctx->stage == fragment_fs ||
3370        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
3371       unsigned write_mask = nir_intrinsic_write_mask(instr);
3372       unsigned component = nir_intrinsic_component(instr);
3373       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3374       unsigned idx = nir_intrinsic_base(instr) + component;
3375
3376       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3377       if (off_instr->type != nir_instr_type_load_const) {
3378          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3379          nir_print_instr(off_instr, stderr);
3380          fprintf(stderr, "\n");
3381       }
3382       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3383
3384       if (instr->src[0].ssa->bit_size == 64)
3385          write_mask = widen_mask(write_mask, 2);
3386
3387       for (unsigned i = 0; i < 8; ++i) {
3388          if (write_mask & (1 << i)) {
3389             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
3390             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
3391          }
3392          idx++;
3393       }
3394    } else if (ctx->stage == vertex_es ||
3395               ctx->stage == vertex_ls ||
3396               (ctx->stage == vertex_tess_control_hs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3397               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
3398       visit_store_ls_or_es_output(ctx, instr);
3399    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
3400       visit_store_tcs_output(ctx, instr, false);
3401    } else {
3402       unreachable("Shader stage not implemented");
3403    }
3404 }
3405
3406 void visit_load_output(isel_context *ctx, nir_intrinsic_instr *instr)
3407 {
3408    visit_load_tcs_output(ctx, instr, false);
3409 }
3410
3411 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3412 {
3413    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3414    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3415
3416    Builder bld(ctx->program, ctx->block);
3417    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3418    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3419 }
3420
3421 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3422 {
3423    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3424    for (unsigned i = 0; i < num_components; i++)
3425       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3426    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3427       assert(num_components == 4);
3428       Builder bld(ctx->program, ctx->block);
3429       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3430    }
3431
3432    for (Operand& op : vec->operands)
3433       op = op.isUndefined() ? Operand(0u) : op;
3434
3435    vec->definitions[0] = Definition(dst);
3436    ctx->block->instructions.emplace_back(std::move(vec));
3437    emit_split_vector(ctx, dst, num_components);
3438    return;
3439 }
3440
3441 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3442 {
3443    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3444    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3445    unsigned idx = nir_intrinsic_base(instr);
3446    unsigned component = nir_intrinsic_component(instr);
3447    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3448
3449    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3450    if (offset) {
3451       assert(offset->u32 == 0);
3452    } else {
3453       /* the lower 15bit of the prim_mask contain the offset into LDS
3454        * while the upper bits contain the number of prims */
3455       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3456       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3457       Builder bld(ctx->program, ctx->block);
3458       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3459       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3460       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3461       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3462       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3463    }
3464
3465    if (instr->dest.ssa.num_components == 1) {
3466       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3467    } else {
3468       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3469       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3470       {
3471          Temp tmp = {ctx->program->allocateId(), v1};
3472          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3473          vec->operands[i] = Operand(tmp);
3474       }
3475       vec->definitions[0] = Definition(dst);
3476       ctx->block->instructions.emplace_back(std::move(vec));
3477    }
3478 }
3479
3480 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3481                              unsigned offset, unsigned stride, unsigned channels)
3482 {
3483    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3484    if (vtx_info->chan_byte_size != 4 && channels == 3)
3485       return false;
3486    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3487           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3488 }
3489
3490 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3491                               unsigned offset, unsigned stride, unsigned *channels)
3492 {
3493    if (!vtx_info->chan_byte_size) {
3494       *channels = vtx_info->num_channels;
3495       return vtx_info->chan_format;
3496    }
3497
3498    unsigned num_channels = *channels;
3499    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3500       unsigned new_channels = num_channels + 1;
3501       /* first, assume more loads is worse and try using a larger data format */
3502       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3503          new_channels++;
3504          /* don't make the attribute potentially out-of-bounds */
3505          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3506             new_channels = 5;
3507       }
3508
3509       if (new_channels == 5) {
3510          /* then try decreasing load size (at the cost of more loads) */
3511          new_channels = *channels;
3512          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3513             new_channels--;
3514       }
3515
3516       if (new_channels < *channels)
3517          *channels = new_channels;
3518       num_channels = new_channels;
3519    }
3520
3521    switch (vtx_info->chan_format) {
3522    case V_008F0C_BUF_DATA_FORMAT_8:
3523       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3524                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3525    case V_008F0C_BUF_DATA_FORMAT_16:
3526       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3527                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3528    case V_008F0C_BUF_DATA_FORMAT_32:
3529       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3530                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3531    }
3532    unreachable("shouldn't reach here");
3533    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3534 }
3535
3536 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3537  * so we may need to fix it up. */
3538 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3539 {
3540    Builder bld(ctx->program, ctx->block);
3541
3542    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3543       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3544
3545    /* For the integer-like cases, do a natural sign extension.
3546     *
3547     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3548     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3549     * exponent.
3550     */
3551    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3552    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3553
3554    /* Convert back to the right type. */
3555    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3556       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3557       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3558       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3559    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3560       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3561    }
3562
3563    return alpha;
3564 }
3565
3566 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3567 {
3568    Builder bld(ctx->program, ctx->block);
3569    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3570    if (ctx->shader->info.stage == MESA_SHADER_VERTEX) {
3571
3572       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3573       if (off_instr->type != nir_instr_type_load_const) {
3574          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3575          nir_print_instr(off_instr, stderr);
3576          fprintf(stderr, "\n");
3577       }
3578       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3579
3580       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3581
3582       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3583       unsigned component = nir_intrinsic_component(instr);
3584       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3585       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3586       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3587       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3588
3589       unsigned dfmt = attrib_format & 0xf;
3590       unsigned nfmt = (attrib_format >> 4) & 0x7;
3591       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3592
3593       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3594       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3595       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3596       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3597       if (post_shuffle)
3598          num_channels = MAX2(num_channels, 3);
3599
3600       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3601       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3602
3603       Temp index;
3604       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3605          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3606          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3607          if (divisor) {
3608             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3609             if (divisor != 1) {
3610                Temp divided = bld.tmp(v1);
3611                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3612                index = bld.vadd32(bld.def(v1), start_instance, divided);
3613             } else {
3614                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3615             }
3616          } else {
3617             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3618          }
3619       } else {
3620          index = bld.vadd32(bld.def(v1),
3621                             get_arg(ctx, ctx->args->ac.base_vertex),
3622                             get_arg(ctx, ctx->args->ac.vertex_id));
3623       }
3624
3625       Temp channels[num_channels];
3626       unsigned channel_start = 0;
3627       bool direct_fetch = false;
3628
3629       /* skip unused channels at the start */
3630       if (vtx_info->chan_byte_size && !post_shuffle) {
3631          channel_start = ffs(mask) - 1;
3632          for (unsigned i = 0; i < channel_start; i++)
3633             channels[i] = Temp(0, s1);
3634       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3635          num_channels = 3 - (ffs(mask) - 1);
3636       }
3637
3638       /* load channels */
3639       while (channel_start < num_channels) {
3640          unsigned fetch_size = num_channels - channel_start;
3641          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3642          bool expanded = false;
3643
3644          /* use MUBUF when possible to avoid possible alignment issues */
3645          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3646          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3647                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3648                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3649                           vtx_info->chan_byte_size == 4;
3650          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3651          if (!use_mubuf) {
3652             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3653          } else {
3654             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
3655                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
3656                fetch_size = 4;
3657                expanded = true;
3658             }
3659          }
3660
3661          Temp fetch_index = index;
3662          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3663             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3664             fetch_offset = fetch_offset % attrib_stride;
3665          }
3666
3667          Operand soffset(0u);
3668          if (fetch_offset >= 4096) {
3669             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3670             fetch_offset %= 4096;
3671          }
3672
3673          aco_opcode opcode;
3674          switch (fetch_size) {
3675          case 1:
3676             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3677             break;
3678          case 2:
3679             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3680             break;
3681          case 3:
3682             assert(ctx->options->chip_class >= GFX7 ||
3683                    (!use_mubuf && ctx->options->chip_class == GFX6));
3684             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3685             break;
3686          case 4:
3687             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3688             break;
3689          default:
3690             unreachable("Unimplemented load_input vector size");
3691          }
3692
3693          Temp fetch_dst;
3694          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3695              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
3696                            num_channels <= 3)) {
3697             direct_fetch = true;
3698             fetch_dst = dst;
3699          } else {
3700             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3701          }
3702
3703          if (use_mubuf) {
3704             Instruction *mubuf = bld.mubuf(opcode,
3705                                            Definition(fetch_dst), list, fetch_index, soffset,
3706                                            fetch_offset, false, true).instr;
3707             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3708          } else {
3709             Instruction *mtbuf = bld.mtbuf(opcode,
3710                                            Definition(fetch_dst), list, fetch_index, soffset,
3711                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3712             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3713          }
3714
3715          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3716
3717          if (fetch_size == 1) {
3718             channels[channel_start] = fetch_dst;
3719          } else {
3720             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3721                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3722          }
3723
3724          channel_start += fetch_size;
3725       }
3726
3727       if (!direct_fetch) {
3728          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3729                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3730
3731          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3732          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3733          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3734
3735          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3736          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3737          unsigned num_temp = 0;
3738          for (unsigned i = 0; i < dst.size(); i++) {
3739             unsigned idx = i + component;
3740             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3741                Temp channel = channels[swizzle[idx]];
3742                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3743                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3744                vec->operands[i] = Operand(channel);
3745
3746                num_temp++;
3747                elems[i] = channel;
3748             } else if (is_float && idx == 3) {
3749                vec->operands[i] = Operand(0x3f800000u);
3750             } else if (!is_float && idx == 3) {
3751                vec->operands[i] = Operand(1u);
3752             } else {
3753                vec->operands[i] = Operand(0u);
3754             }
3755          }
3756          vec->definitions[0] = Definition(dst);
3757          ctx->block->instructions.emplace_back(std::move(vec));
3758          emit_split_vector(ctx, dst, dst.size());
3759
3760          if (num_temp == dst.size())
3761             ctx->allocated_vec.emplace(dst.id(), elems);
3762       }
3763    } else if (ctx->shader->info.stage == MESA_SHADER_FRAGMENT) {
3764       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
3765       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
3766       if (off_instr->type != nir_instr_type_load_const ||
3767           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3768          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3769          nir_print_instr(off_instr, stderr);
3770          fprintf(stderr, "\n");
3771       }
3772
3773       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3774       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
3775       if (offset) {
3776          assert(offset->u32 == 0);
3777       } else {
3778          /* the lower 15bit of the prim_mask contain the offset into LDS
3779           * while the upper bits contain the number of prims */
3780          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
3781          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3782          Builder bld(ctx->program, ctx->block);
3783          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3784          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3785          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3786          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3787          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3788       }
3789
3790       unsigned idx = nir_intrinsic_base(instr);
3791       unsigned component = nir_intrinsic_component(instr);
3792       unsigned vertex_id = 2; /* P0 */
3793
3794       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
3795          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
3796          switch (src0->u32) {
3797          case 0:
3798             vertex_id = 2; /* P0 */
3799             break;
3800          case 1:
3801             vertex_id = 0; /* P10 */
3802             break;
3803          case 2:
3804             vertex_id = 1; /* P20 */
3805             break;
3806          default:
3807             unreachable("invalid vertex index");
3808          }
3809       }
3810
3811       if (dst.size() == 1) {
3812          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
3813       } else {
3814          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3815          for (unsigned i = 0; i < dst.size(); i++)
3816             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
3817          vec->definitions[0] = Definition(dst);
3818          bld.insert(std::move(vec));
3819       }
3820
3821    } else {
3822       unreachable("Shader stage not implemented");
3823    }
3824 }
3825
3826 std::pair<Temp, unsigned> get_gs_per_vertex_input_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride = 1u)
3827 {
3828    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3829
3830    Builder bld(ctx->program, ctx->block);
3831    nir_src *vertex_src = nir_get_io_vertex_index_src(instr);
3832    Temp vertex_offset;
3833
3834    if (!nir_src_is_const(*vertex_src)) {
3835       /* better code could be created, but this case probably doesn't happen
3836        * much in practice */
3837       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, vertex_src->ssa));
3838       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3839          Temp elem;
3840
3841          if (ctx->stage == vertex_geometry_gs) {
3842             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3843             if (i % 2u)
3844                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3845          } else {
3846             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3847          }
3848
3849          if (vertex_offset.id()) {
3850             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)),
3851                                  Operand(i), indirect_vertex);
3852             vertex_offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), vertex_offset, elem, cond);
3853          } else {
3854             vertex_offset = elem;
3855          }
3856       }
3857
3858       if (ctx->stage == vertex_geometry_gs)
3859          vertex_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), vertex_offset);
3860    } else {
3861       unsigned vertex = nir_src_as_uint(*vertex_src);
3862       if (ctx->stage == vertex_geometry_gs)
3863          vertex_offset = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
3864                                   get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3865                                   Operand((vertex % 2u) * 16u), Operand(16u));
3866       else
3867          vertex_offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3868    }
3869
3870    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, base_stride);
3871    offs = offset_add(ctx, offs, std::make_pair(vertex_offset, 0u));
3872    return offset_mul(ctx, offs, 4u);
3873 }
3874
3875 void visit_load_gs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3876 {
3877    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3878
3879    Builder bld(ctx->program, ctx->block);
3880    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3881    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3882
3883    if (ctx->stage == geometry_gs) {
3884       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr, ctx->program->wave_size);
3885       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3886       load_vmem_mubuf(ctx, dst, ring, offs.first, Temp(), offs.second, elem_size_bytes, instr->dest.ssa.num_components, 4u * ctx->program->wave_size, false, true);
3887    } else if (ctx->stage == vertex_geometry_gs) {
3888       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr);
3889       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3890       load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
3891    } else {
3892       unreachable("Unsupported GS stage.");
3893    }
3894 }
3895
3896 void visit_load_tcs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3897 {
3898    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3899
3900    Builder bld(ctx->program, ctx->block);
3901    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3902    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_input_lds_offset(ctx, instr);
3903    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3904    unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3905
3906    load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
3907 }
3908
3909 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3910 {
3911    switch (ctx->shader->info.stage) {
3912    case MESA_SHADER_GEOMETRY:
3913       visit_load_gs_per_vertex_input(ctx, instr);
3914       break;
3915    case MESA_SHADER_TESS_CTRL:
3916       visit_load_tcs_per_vertex_input(ctx, instr);
3917       break;
3918    default:
3919       unreachable("Unimplemented shader stage");
3920    }
3921 }
3922
3923 void visit_load_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
3924 {
3925    visit_load_tcs_output(ctx, instr, true);
3926 }
3927
3928 void visit_store_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
3929 {
3930    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3931    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3932
3933    visit_store_tcs_output(ctx, instr, true);
3934 }
3935
3936 void visit_load_tess_coord(isel_context *ctx, nir_intrinsic_instr *instr)
3937 {
3938    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
3939
3940    Builder bld(ctx->program, ctx->block);
3941    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3942
3943    Operand tes_u(get_arg(ctx, ctx->args->tes_u));
3944    Operand tes_v(get_arg(ctx, ctx->args->tes_v));
3945    Operand tes_w(0u);
3946
3947    if (ctx->shader->info.tess.primitive_mode == GL_TRIANGLES) {
3948       Temp tmp = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), tes_u, tes_v);
3949       tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0x3f800000u /* 1.0f */), tmp);
3950       tes_w = Operand(tmp);
3951    }
3952
3953    Temp tess_coord = bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tes_u, tes_v, tes_w);
3954    emit_split_vector(ctx, tess_coord, 3);
3955 }
3956
3957 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3958 {
3959    if (ctx->program->info->need_indirect_descriptor_sets) {
3960       Builder bld(ctx->program, ctx->block);
3961       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3962       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3963       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3964    }
3965
3966    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3967 }
3968
3969
3970 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3971 {
3972    Builder bld(ctx->program, ctx->block);
3973    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3974    if (!ctx->divergent_vals[instr->dest.ssa.index])
3975       index = bld.as_uniform(index);
3976    unsigned desc_set = nir_intrinsic_desc_set(instr);
3977    unsigned binding = nir_intrinsic_binding(instr);
3978
3979    Temp desc_ptr;
3980    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3981    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3982    unsigned offset = layout->binding[binding].offset;
3983    unsigned stride;
3984    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3985        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3986       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3987       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3988       offset = pipeline_layout->push_constant_size + 16 * idx;
3989       stride = 16;
3990    } else {
3991       desc_ptr = load_desc_ptr(ctx, desc_set);
3992       stride = layout->binding[binding].size;
3993    }
3994
3995    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3996    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3997    if (stride != 1) {
3998       if (nir_const_index) {
3999          const_index = const_index * stride;
4000       } else if (index.type() == RegType::vgpr) {
4001          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
4002          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
4003       } else {
4004          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
4005       }
4006    }
4007    if (offset) {
4008       if (nir_const_index) {
4009          const_index = const_index + offset;
4010       } else if (index.type() == RegType::vgpr) {
4011          index = bld.vadd32(bld.def(v1), Operand(offset), index);
4012       } else {
4013          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
4014       }
4015    }
4016
4017    if (nir_const_index && const_index == 0) {
4018       index = desc_ptr;
4019    } else if (index.type() == RegType::vgpr) {
4020       index = bld.vadd32(bld.def(v1),
4021                          nir_const_index ? Operand(const_index) : Operand(index),
4022                          Operand(desc_ptr));
4023    } else {
4024       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4025                        nir_const_index ? Operand(const_index) : Operand(index),
4026                        Operand(desc_ptr));
4027    }
4028
4029    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
4030 }
4031
4032 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
4033                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
4034 {
4035    Builder bld(ctx->program, ctx->block);
4036
4037    unsigned num_bytes = dst.size() * 4;
4038    bool dlc = glc && ctx->options->chip_class >= GFX10;
4039
4040    aco_opcode op;
4041    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
4042       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4043       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4044       unsigned const_offset = 0;
4045
4046       Temp lower = Temp();
4047       if (num_bytes > 16) {
4048          assert(num_components == 3 || num_components == 4);
4049          op = aco_opcode::buffer_load_dwordx4;
4050          lower = bld.tmp(v4);
4051          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4052          mubuf->definitions[0] = Definition(lower);
4053          mubuf->operands[0] = Operand(rsrc);
4054          mubuf->operands[1] = vaddr;
4055          mubuf->operands[2] = soffset;
4056          mubuf->offen = (offset.type() == RegType::vgpr);
4057          mubuf->glc = glc;
4058          mubuf->dlc = dlc;
4059          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4060          mubuf->can_reorder = readonly;
4061          bld.insert(std::move(mubuf));
4062          emit_split_vector(ctx, lower, 2);
4063          num_bytes -= 16;
4064          const_offset = 16;
4065       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
4066          /* GFX6 doesn't support loading vec3, expand to vec4. */
4067          num_bytes = 16;
4068       }
4069
4070       switch (num_bytes) {
4071          case 4:
4072             op = aco_opcode::buffer_load_dword;
4073             break;
4074          case 8:
4075             op = aco_opcode::buffer_load_dwordx2;
4076             break;
4077          case 12:
4078             assert(ctx->options->chip_class > GFX6);
4079             op = aco_opcode::buffer_load_dwordx3;
4080             break;
4081          case 16:
4082             op = aco_opcode::buffer_load_dwordx4;
4083             break;
4084          default:
4085             unreachable("Load SSBO not implemented for this size.");
4086       }
4087       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4088       mubuf->operands[0] = Operand(rsrc);
4089       mubuf->operands[1] = vaddr;
4090       mubuf->operands[2] = soffset;
4091       mubuf->offen = (offset.type() == RegType::vgpr);
4092       mubuf->glc = glc;
4093       mubuf->dlc = dlc;
4094       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4095       mubuf->can_reorder = readonly;
4096       mubuf->offset = const_offset;
4097       aco_ptr<Instruction> instr = std::move(mubuf);
4098
4099       if (dst.size() > 4) {
4100          assert(lower != Temp());
4101          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
4102          instr->definitions[0] = Definition(upper);
4103          bld.insert(std::move(instr));
4104          if (dst.size() == 8)
4105             emit_split_vector(ctx, upper, 2);
4106          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
4107          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
4108          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
4109          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
4110          if (dst.size() == 8)
4111             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
4112       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
4113          Temp vec = bld.tmp(v4);
4114          instr->definitions[0] = Definition(vec);
4115          bld.insert(std::move(instr));
4116          emit_split_vector(ctx, vec, 4);
4117
4118          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4119          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4120          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4121          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4122       }
4123
4124       if (dst.type() == RegType::sgpr) {
4125          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4126          instr->definitions[0] = Definition(vec);
4127          bld.insert(std::move(instr));
4128          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4129       } else {
4130          instr->definitions[0] = Definition(dst);
4131          bld.insert(std::move(instr));
4132          emit_split_vector(ctx, dst, num_components);
4133       }
4134    } else {
4135       switch (num_bytes) {
4136          case 4:
4137             op = aco_opcode::s_buffer_load_dword;
4138             break;
4139          case 8:
4140             op = aco_opcode::s_buffer_load_dwordx2;
4141             break;
4142          case 12:
4143          case 16:
4144             op = aco_opcode::s_buffer_load_dwordx4;
4145             break;
4146          case 24:
4147          case 32:
4148             op = aco_opcode::s_buffer_load_dwordx8;
4149             break;
4150          default:
4151             unreachable("Load SSBO not implemented for this size.");
4152       }
4153       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4154       load->operands[0] = Operand(rsrc);
4155       load->operands[1] = Operand(bld.as_uniform(offset));
4156       assert(load->operands[1].getTemp().type() == RegType::sgpr);
4157       load->definitions[0] = Definition(dst);
4158       load->glc = glc;
4159       load->dlc = dlc;
4160       load->barrier = readonly ? barrier_none : barrier_buffer;
4161       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
4162       assert(ctx->options->chip_class >= GFX8 || !glc);
4163
4164       /* trim vector */
4165       if (dst.size() == 3) {
4166          Temp vec = bld.tmp(s4);
4167          load->definitions[0] = Definition(vec);
4168          bld.insert(std::move(load));
4169          emit_split_vector(ctx, vec, 4);
4170
4171          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4172                     emit_extract_vector(ctx, vec, 0, s1),
4173                     emit_extract_vector(ctx, vec, 1, s1),
4174                     emit_extract_vector(ctx, vec, 2, s1));
4175       } else if (dst.size() == 6) {
4176          Temp vec = bld.tmp(s8);
4177          load->definitions[0] = Definition(vec);
4178          bld.insert(std::move(load));
4179          emit_split_vector(ctx, vec, 4);
4180
4181          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4182                     emit_extract_vector(ctx, vec, 0, s2),
4183                     emit_extract_vector(ctx, vec, 1, s2),
4184                     emit_extract_vector(ctx, vec, 2, s2));
4185       } else {
4186          bld.insert(std::move(load));
4187       }
4188       emit_split_vector(ctx, dst, num_components);
4189    }
4190 }
4191
4192 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
4193 {
4194    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4195    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
4196
4197    Builder bld(ctx->program, ctx->block);
4198
4199    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
4200    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
4201    unsigned binding = nir_intrinsic_binding(idx_instr);
4202    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
4203
4204    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
4205       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4206                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4207                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4208                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4209       if (ctx->options->chip_class >= GFX10) {
4210          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4211                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4212                       S_008F0C_RESOURCE_LEVEL(1);
4213       } else {
4214          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4215                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4216       }
4217       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
4218                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
4219                                      Operand(0xFFFFFFFFu),
4220                                      Operand(desc_type));
4221       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4222                         rsrc, upper_dwords);
4223    } else {
4224       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
4225       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4226    }
4227
4228    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
4229 }
4230
4231 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4232 {
4233    Builder bld(ctx->program, ctx->block);
4234    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4235
4236    unsigned offset = nir_intrinsic_base(instr);
4237    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
4238    if (index_cv && instr->dest.ssa.bit_size == 32) {
4239
4240       unsigned count = instr->dest.ssa.num_components;
4241       unsigned start = (offset + index_cv->u32) / 4u;
4242       start -= ctx->args->ac.base_inline_push_consts;
4243       if (start + count <= ctx->args->ac.num_inline_push_consts) {
4244          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4245          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4246          for (unsigned i = 0; i < count; ++i) {
4247             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
4248             vec->operands[i] = Operand{elems[i]};
4249          }
4250          vec->definitions[0] = Definition(dst);
4251          ctx->block->instructions.emplace_back(std::move(vec));
4252          ctx->allocated_vec.emplace(dst.id(), elems);
4253          return;
4254       }
4255    }
4256
4257    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
4258    if (offset != 0) // TODO check if index != 0 as well
4259       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
4260    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
4261    Temp vec = dst;
4262    bool trim = false;
4263    aco_opcode op;
4264
4265    switch (dst.size()) {
4266    case 1:
4267       op = aco_opcode::s_load_dword;
4268       break;
4269    case 2:
4270       op = aco_opcode::s_load_dwordx2;
4271       break;
4272    case 3:
4273       vec = bld.tmp(s4);
4274       trim = true;
4275    case 4:
4276       op = aco_opcode::s_load_dwordx4;
4277       break;
4278    case 6:
4279       vec = bld.tmp(s8);
4280       trim = true;
4281    case 8:
4282       op = aco_opcode::s_load_dwordx8;
4283       break;
4284    default:
4285       unreachable("unimplemented or forbidden load_push_constant.");
4286    }
4287
4288    bld.smem(op, Definition(vec), ptr, index);
4289
4290    if (trim) {
4291       emit_split_vector(ctx, vec, 4);
4292       RegClass rc = dst.size() == 3 ? s1 : s2;
4293       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4294                  emit_extract_vector(ctx, vec, 0, rc),
4295                  emit_extract_vector(ctx, vec, 1, rc),
4296                  emit_extract_vector(ctx, vec, 2, rc));
4297
4298    }
4299    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4300 }
4301
4302 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4303 {
4304    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4305
4306    Builder bld(ctx->program, ctx->block);
4307
4308    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4309                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4310                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4311                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4312    if (ctx->options->chip_class >= GFX10) {
4313       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4314                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4315                    S_008F0C_RESOURCE_LEVEL(1);
4316    } else {
4317       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4318                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4319    }
4320
4321    unsigned base = nir_intrinsic_base(instr);
4322    unsigned range = nir_intrinsic_range(instr);
4323
4324    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
4325    if (base && offset.type() == RegType::sgpr)
4326       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
4327    else if (base && offset.type() == RegType::vgpr)
4328       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
4329
4330    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4331                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
4332                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
4333                           Operand(desc_type));
4334
4335    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
4336 }
4337
4338 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
4339 {
4340    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4341       ctx->cf_info.exec_potentially_empty_discard = true;
4342
4343    ctx->program->needs_exact = true;
4344
4345    // TODO: optimize uniform conditions
4346    Builder bld(ctx->program, ctx->block);
4347    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
4348    assert(src.regClass() == bld.lm);
4349    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
4350    bld.pseudo(aco_opcode::p_discard_if, src);
4351    ctx->block->kind |= block_kind_uses_discard_if;
4352    return;
4353 }
4354
4355 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
4356 {
4357    Builder bld(ctx->program, ctx->block);
4358
4359    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4360       ctx->cf_info.exec_potentially_empty_discard = true;
4361
4362    bool divergent = ctx->cf_info.parent_if.is_divergent ||
4363                     ctx->cf_info.parent_loop.has_divergent_continue;
4364
4365    if (ctx->block->loop_nest_depth &&
4366        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
4367       /* we handle discards the same way as jump instructions */
4368       append_logical_end(ctx->block);
4369
4370       /* in loops, discard behaves like break */
4371       Block *linear_target = ctx->cf_info.parent_loop.exit;
4372       ctx->block->kind |= block_kind_discard;
4373
4374       if (!divergent) {
4375          /* uniform discard - loop ends here */
4376          assert(nir_instr_is_last(&instr->instr));
4377          ctx->block->kind |= block_kind_uniform;
4378          ctx->cf_info.has_branch = true;
4379          bld.branch(aco_opcode::p_branch);
4380          add_linear_edge(ctx->block->index, linear_target);
4381          return;
4382       }
4383
4384       /* we add a break right behind the discard() instructions */
4385       ctx->block->kind |= block_kind_break;
4386       unsigned idx = ctx->block->index;
4387
4388       /* remove critical edges from linear CFG */
4389       bld.branch(aco_opcode::p_branch);
4390       Block* break_block = ctx->program->create_and_insert_block();
4391       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
4392       break_block->kind |= block_kind_uniform;
4393       add_linear_edge(idx, break_block);
4394       add_linear_edge(break_block->index, linear_target);
4395       bld.reset(break_block);
4396       bld.branch(aco_opcode::p_branch);
4397
4398       Block* continue_block = ctx->program->create_and_insert_block();
4399       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
4400       add_linear_edge(idx, continue_block);
4401       append_logical_start(continue_block);
4402       ctx->block = continue_block;
4403
4404       return;
4405    }
4406
4407    /* it can currently happen that NIR doesn't remove the unreachable code */
4408    if (!nir_instr_is_last(&instr->instr)) {
4409       ctx->program->needs_exact = true;
4410       /* save exec somewhere temporarily so that it doesn't get
4411        * overwritten before the discard from outer exec masks */
4412       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
4413       bld.pseudo(aco_opcode::p_discard_if, cond);
4414       ctx->block->kind |= block_kind_uses_discard_if;
4415       return;
4416    }
4417
4418    /* This condition is incorrect for uniformly branched discards in a loop
4419     * predicated by a divergent condition, but the above code catches that case
4420     * and the discard would end up turning into a discard_if.
4421     * For example:
4422     * if (divergent) {
4423     *    while (...) {
4424     *       if (uniform) {
4425     *          discard;
4426     *       }
4427     *    }
4428     * }
4429     */
4430    if (!ctx->cf_info.parent_if.is_divergent) {
4431       /* program just ends here */
4432       ctx->block->kind |= block_kind_uniform;
4433       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
4434               0 /* enabled mask */, 9 /* dest */,
4435               false /* compressed */, true/* done */, true /* valid mask */);
4436       bld.sopp(aco_opcode::s_endpgm);
4437       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
4438    } else {
4439       ctx->block->kind |= block_kind_discard;
4440       /* branch and linear edge is added by visit_if() */
4441    }
4442 }
4443
4444 enum aco_descriptor_type {
4445    ACO_DESC_IMAGE,
4446    ACO_DESC_FMASK,
4447    ACO_DESC_SAMPLER,
4448    ACO_DESC_BUFFER,
4449    ACO_DESC_PLANE_0,
4450    ACO_DESC_PLANE_1,
4451    ACO_DESC_PLANE_2,
4452 };
4453
4454 static bool
4455 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
4456    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
4457       return false;
4458    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
4459    return dim == ac_image_cube ||
4460           dim == ac_image_1darray ||
4461           dim == ac_image_2darray ||
4462           dim == ac_image_2darraymsaa;
4463 }
4464
4465 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4466                       enum aco_descriptor_type desc_type,
4467                       const nir_tex_instr *tex_instr, bool image, bool write)
4468 {
4469 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4470    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4471    if (it != ctx->tex_desc.end())
4472       return it->second;
4473 */
4474    Temp index = Temp();
4475    bool index_set = false;
4476    unsigned constant_index = 0;
4477    unsigned descriptor_set;
4478    unsigned base_index;
4479    Builder bld(ctx->program, ctx->block);
4480
4481    if (!deref_instr) {
4482       assert(tex_instr && !image);
4483       descriptor_set = 0;
4484       base_index = tex_instr->sampler_index;
4485    } else {
4486       while(deref_instr->deref_type != nir_deref_type_var) {
4487          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4488          if (!array_size)
4489             array_size = 1;
4490
4491          assert(deref_instr->deref_type == nir_deref_type_array);
4492          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4493          if (const_value) {
4494             constant_index += array_size * const_value->u32;
4495          } else {
4496             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4497             if (indirect.type() == RegType::vgpr)
4498                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4499
4500             if (array_size != 1)
4501                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4502
4503             if (!index_set) {
4504                index = indirect;
4505                index_set = true;
4506             } else {
4507                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4508             }
4509          }
4510
4511          deref_instr = nir_src_as_deref(deref_instr->parent);
4512       }
4513       descriptor_set = deref_instr->var->data.descriptor_set;
4514       base_index = deref_instr->var->data.binding;
4515    }
4516
4517    Temp list = load_desc_ptr(ctx, descriptor_set);
4518    list = convert_pointer_to_64_bit(ctx, list);
4519
4520    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4521    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4522    unsigned offset = binding->offset;
4523    unsigned stride = binding->size;
4524    aco_opcode opcode;
4525    RegClass type;
4526
4527    assert(base_index < layout->binding_count);
4528
4529    switch (desc_type) {
4530    case ACO_DESC_IMAGE:
4531       type = s8;
4532       opcode = aco_opcode::s_load_dwordx8;
4533       break;
4534    case ACO_DESC_FMASK:
4535       type = s8;
4536       opcode = aco_opcode::s_load_dwordx8;
4537       offset += 32;
4538       break;
4539    case ACO_DESC_SAMPLER:
4540       type = s4;
4541       opcode = aco_opcode::s_load_dwordx4;
4542       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4543          offset += radv_combined_image_descriptor_sampler_offset(binding);
4544       break;
4545    case ACO_DESC_BUFFER:
4546       type = s4;
4547       opcode = aco_opcode::s_load_dwordx4;
4548       break;
4549    case ACO_DESC_PLANE_0:
4550    case ACO_DESC_PLANE_1:
4551       type = s8;
4552       opcode = aco_opcode::s_load_dwordx8;
4553       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4554       break;
4555    case ACO_DESC_PLANE_2:
4556       type = s4;
4557       opcode = aco_opcode::s_load_dwordx4;
4558       offset += 64;
4559       break;
4560    default:
4561       unreachable("invalid desc_type\n");
4562    }
4563
4564    offset += constant_index * stride;
4565
4566    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4567       (!index_set || binding->immutable_samplers_equal)) {
4568       if (binding->immutable_samplers_equal)
4569          constant_index = 0;
4570
4571       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4572       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4573                         Operand(samplers[constant_index * 4 + 0]),
4574                         Operand(samplers[constant_index * 4 + 1]),
4575                         Operand(samplers[constant_index * 4 + 2]),
4576                         Operand(samplers[constant_index * 4 + 3]));
4577    }
4578
4579    Operand off;
4580    if (!index_set) {
4581       off = bld.copy(bld.def(s1), Operand(offset));
4582    } else {
4583       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4584                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4585    }
4586
4587    Temp res = bld.smem(opcode, bld.def(type), list, off);
4588
4589    if (desc_type == ACO_DESC_PLANE_2) {
4590       Temp components[8];
4591       for (unsigned i = 0; i < 8; i++)
4592          components[i] = bld.tmp(s1);
4593       bld.pseudo(aco_opcode::p_split_vector,
4594                  Definition(components[0]),
4595                  Definition(components[1]),
4596                  Definition(components[2]),
4597                  Definition(components[3]),
4598                  res);
4599
4600       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4601       bld.pseudo(aco_opcode::p_split_vector,
4602                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4603                  Definition(components[4]),
4604                  Definition(components[5]),
4605                  Definition(components[6]),
4606                  Definition(components[7]),
4607                  desc2);
4608
4609       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4610                        components[0], components[1], components[2], components[3],
4611                        components[4], components[5], components[6], components[7]);
4612    }
4613
4614    return res;
4615 }
4616
4617 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4618 {
4619    switch (dim) {
4620    case GLSL_SAMPLER_DIM_BUF:
4621       return 1;
4622    case GLSL_SAMPLER_DIM_1D:
4623       return array ? 2 : 1;
4624    case GLSL_SAMPLER_DIM_2D:
4625       return array ? 3 : 2;
4626    case GLSL_SAMPLER_DIM_MS:
4627       return array ? 4 : 3;
4628    case GLSL_SAMPLER_DIM_3D:
4629    case GLSL_SAMPLER_DIM_CUBE:
4630       return 3;
4631    case GLSL_SAMPLER_DIM_RECT:
4632    case GLSL_SAMPLER_DIM_SUBPASS:
4633       return 2;
4634    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4635       return 3;
4636    default:
4637       break;
4638    }
4639    return 0;
4640 }
4641
4642
4643 /* Adjust the sample index according to FMASK.
4644  *
4645  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4646  * which is the identity mapping. Each nibble says which physical sample
4647  * should be fetched to get that sample.
4648  *
4649  * For example, 0x11111100 means there are only 2 samples stored and
4650  * the second sample covers 3/4 of the pixel. When reading samples 0
4651  * and 1, return physical sample 0 (determined by the first two 0s
4652  * in FMASK), otherwise return physical sample 1.
4653  *
4654  * The sample index should be adjusted as follows:
4655  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4656  */
4657 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
4658 {
4659    Builder bld(ctx->program, ctx->block);
4660    Temp fmask = bld.tmp(v1);
4661    unsigned dim = ctx->options->chip_class >= GFX10
4662                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4663                   : 0;
4664
4665    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
4666                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
4667    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
4668    load->operands[0] = Operand(fmask_desc_ptr);
4669    load->operands[1] = Operand(s4); /* no sampler */
4670    load->operands[2] = Operand(coord);
4671    load->definitions[0] = Definition(fmask);
4672    load->glc = false;
4673    load->dlc = false;
4674    load->dmask = 0x1;
4675    load->unrm = true;
4676    load->da = da;
4677    load->dim = dim;
4678    load->can_reorder = true; /* fmask images shouldn't be modified */
4679    ctx->block->instructions.emplace_back(std::move(load));
4680
4681    Operand sample_index4;
4682    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4683       sample_index4 = Operand(sample_index.constantValue() << 2);
4684    } else if (sample_index.regClass() == s1) {
4685       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4686    } else {
4687       assert(sample_index.regClass() == v1);
4688       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4689    }
4690
4691    Temp final_sample;
4692    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4693       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4694    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4695       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4696    else
4697       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4698
4699    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4700     * resource descriptor is 0 (invalid),
4701     */
4702    Temp compare = bld.tmp(bld.lm);
4703    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4704                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4705
4706    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4707
4708    /* Replace the MSAA sample index. */
4709    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4710 }
4711
4712 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4713 {
4714
4715    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4716    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4717    bool is_array = glsl_sampler_type_is_array(type);
4718    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4719    assert(!add_frag_pos && "Input attachments should be lowered.");
4720    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4721    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4722    int count = image_type_to_components_count(dim, is_array);
4723    std::vector<Temp> coords(count);
4724    Builder bld(ctx->program, ctx->block);
4725
4726    if (is_ms) {
4727       count--;
4728       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
4729       /* get sample index */
4730       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4731          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4732          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
4733          std::vector<Temp> fmask_load_address;
4734          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
4735             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
4736
4737          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4738          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
4739       } else {
4740          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
4741       }
4742    }
4743
4744    if (gfx9_1d) {
4745       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
4746       coords.resize(coords.size() + 1);
4747       coords[1] = bld.copy(bld.def(v1), Operand(0u));
4748       if (is_array)
4749          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
4750    } else {
4751       for (int i = 0; i < count; i++)
4752          coords[i] = emit_extract_vector(ctx, src0, i, v1);
4753    }
4754
4755    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4756        instr->intrinsic == nir_intrinsic_image_deref_store) {
4757       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4758       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4759
4760       if (!level_zero)
4761          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
4762    }
4763
4764    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4765    for (unsigned i = 0; i < coords.size(); i++)
4766       vec->operands[i] = Operand(coords[i]);
4767    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4768    vec->definitions[0] = Definition(res);
4769    ctx->block->instructions.emplace_back(std::move(vec));
4770    return res;
4771 }
4772
4773
4774 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4775 {
4776    Builder bld(ctx->program, ctx->block);
4777    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4778    const struct glsl_type *type = glsl_without_array(var->type);
4779    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4780    bool is_array = glsl_sampler_type_is_array(type);
4781    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4782
4783    if (dim == GLSL_SAMPLER_DIM_BUF) {
4784       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4785       unsigned num_channels = util_last_bit(mask);
4786       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4787       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4788
4789       aco_opcode opcode;
4790       switch (num_channels) {
4791       case 1:
4792          opcode = aco_opcode::buffer_load_format_x;
4793          break;
4794       case 2:
4795          opcode = aco_opcode::buffer_load_format_xy;
4796          break;
4797       case 3:
4798          opcode = aco_opcode::buffer_load_format_xyz;
4799          break;
4800       case 4:
4801          opcode = aco_opcode::buffer_load_format_xyzw;
4802          break;
4803       default:
4804          unreachable(">4 channel buffer image load");
4805       }
4806       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4807       load->operands[0] = Operand(rsrc);
4808       load->operands[1] = Operand(vindex);
4809       load->operands[2] = Operand((uint32_t) 0);
4810       Temp tmp;
4811       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4812          tmp = dst;
4813       else
4814          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4815       load->definitions[0] = Definition(tmp);
4816       load->idxen = true;
4817       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4818       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4819       load->barrier = barrier_image;
4820       ctx->block->instructions.emplace_back(std::move(load));
4821
4822       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4823       return;
4824    }
4825
4826    Temp coords = get_image_coords(ctx, instr, type);
4827    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4828
4829    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4830    unsigned num_components = util_bitcount(dmask);
4831    Temp tmp;
4832    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4833       tmp = dst;
4834    else
4835       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4836
4837    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4838    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4839
4840    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
4841    load->operands[0] = Operand(resource);
4842    load->operands[1] = Operand(s4); /* no sampler */
4843    load->operands[2] = Operand(coords);
4844    load->definitions[0] = Definition(tmp);
4845    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4846    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4847    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4848    load->dmask = dmask;
4849    load->unrm = true;
4850    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4851    load->barrier = barrier_image;
4852    ctx->block->instructions.emplace_back(std::move(load));
4853
4854    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4855    return;
4856 }
4857
4858 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4859 {
4860    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4861    const struct glsl_type *type = glsl_without_array(var->type);
4862    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4863    bool is_array = glsl_sampler_type_is_array(type);
4864    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4865
4866    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4867
4868    if (dim == GLSL_SAMPLER_DIM_BUF) {
4869       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4870       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4871       aco_opcode opcode;
4872       switch (data.size()) {
4873       case 1:
4874          opcode = aco_opcode::buffer_store_format_x;
4875          break;
4876       case 2:
4877          opcode = aco_opcode::buffer_store_format_xy;
4878          break;
4879       case 3:
4880          opcode = aco_opcode::buffer_store_format_xyz;
4881          break;
4882       case 4:
4883          opcode = aco_opcode::buffer_store_format_xyzw;
4884          break;
4885       default:
4886          unreachable(">4 channel buffer image store");
4887       }
4888       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4889       store->operands[0] = Operand(rsrc);
4890       store->operands[1] = Operand(vindex);
4891       store->operands[2] = Operand((uint32_t) 0);
4892       store->operands[3] = Operand(data);
4893       store->idxen = true;
4894       store->glc = glc;
4895       store->dlc = false;
4896       store->disable_wqm = true;
4897       store->barrier = barrier_image;
4898       ctx->program->needs_exact = true;
4899       ctx->block->instructions.emplace_back(std::move(store));
4900       return;
4901    }
4902
4903    assert(data.type() == RegType::vgpr);
4904    Temp coords = get_image_coords(ctx, instr, type);
4905    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4906
4907    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4908    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4909
4910    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
4911    store->operands[0] = Operand(resource);
4912    store->operands[1] = Operand(data);
4913    store->operands[2] = Operand(coords);
4914    store->glc = glc;
4915    store->dlc = false;
4916    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4917    store->dmask = (1 << data.size()) - 1;
4918    store->unrm = true;
4919    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4920    store->disable_wqm = true;
4921    store->barrier = barrier_image;
4922    ctx->program->needs_exact = true;
4923    ctx->block->instructions.emplace_back(std::move(store));
4924    return;
4925 }
4926
4927 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4928 {
4929    /* return the previous value if dest is ever used */
4930    bool return_previous = false;
4931    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4932       return_previous = true;
4933       break;
4934    }
4935    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4936       return_previous = true;
4937       break;
4938    }
4939
4940    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4941    const struct glsl_type *type = glsl_without_array(var->type);
4942    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4943    bool is_array = glsl_sampler_type_is_array(type);
4944    Builder bld(ctx->program, ctx->block);
4945
4946    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4947    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4948
4949    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4950       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4951
4952    aco_opcode buf_op, image_op;
4953    switch (instr->intrinsic) {
4954       case nir_intrinsic_image_deref_atomic_add:
4955          buf_op = aco_opcode::buffer_atomic_add;
4956          image_op = aco_opcode::image_atomic_add;
4957          break;
4958       case nir_intrinsic_image_deref_atomic_umin:
4959          buf_op = aco_opcode::buffer_atomic_umin;
4960          image_op = aco_opcode::image_atomic_umin;
4961          break;
4962       case nir_intrinsic_image_deref_atomic_imin:
4963          buf_op = aco_opcode::buffer_atomic_smin;
4964          image_op = aco_opcode::image_atomic_smin;
4965          break;
4966       case nir_intrinsic_image_deref_atomic_umax:
4967          buf_op = aco_opcode::buffer_atomic_umax;
4968          image_op = aco_opcode::image_atomic_umax;
4969          break;
4970       case nir_intrinsic_image_deref_atomic_imax:
4971          buf_op = aco_opcode::buffer_atomic_smax;
4972          image_op = aco_opcode::image_atomic_smax;
4973          break;
4974       case nir_intrinsic_image_deref_atomic_and:
4975          buf_op = aco_opcode::buffer_atomic_and;
4976          image_op = aco_opcode::image_atomic_and;
4977          break;
4978       case nir_intrinsic_image_deref_atomic_or:
4979          buf_op = aco_opcode::buffer_atomic_or;
4980          image_op = aco_opcode::image_atomic_or;
4981          break;
4982       case nir_intrinsic_image_deref_atomic_xor:
4983          buf_op = aco_opcode::buffer_atomic_xor;
4984          image_op = aco_opcode::image_atomic_xor;
4985          break;
4986       case nir_intrinsic_image_deref_atomic_exchange:
4987          buf_op = aco_opcode::buffer_atomic_swap;
4988          image_op = aco_opcode::image_atomic_swap;
4989          break;
4990       case nir_intrinsic_image_deref_atomic_comp_swap:
4991          buf_op = aco_opcode::buffer_atomic_cmpswap;
4992          image_op = aco_opcode::image_atomic_cmpswap;
4993          break;
4994       default:
4995          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4996    }
4997
4998    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4999
5000    if (dim == GLSL_SAMPLER_DIM_BUF) {
5001       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5002       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5003       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
5004       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5005       mubuf->operands[0] = Operand(resource);
5006       mubuf->operands[1] = Operand(vindex);
5007       mubuf->operands[2] = Operand((uint32_t)0);
5008       mubuf->operands[3] = Operand(data);
5009       if (return_previous)
5010          mubuf->definitions[0] = Definition(dst);
5011       mubuf->offset = 0;
5012       mubuf->idxen = true;
5013       mubuf->glc = return_previous;
5014       mubuf->dlc = false; /* Not needed for atomics */
5015       mubuf->disable_wqm = true;
5016       mubuf->barrier = barrier_image;
5017       ctx->program->needs_exact = true;
5018       ctx->block->instructions.emplace_back(std::move(mubuf));
5019       return;
5020    }
5021
5022    Temp coords = get_image_coords(ctx, instr, type);
5023    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5024    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
5025    mimg->operands[0] = Operand(resource);
5026    mimg->operands[1] = Operand(data);
5027    mimg->operands[2] = Operand(coords);
5028    if (return_previous)
5029       mimg->definitions[0] = Definition(dst);
5030    mimg->glc = return_previous;
5031    mimg->dlc = false; /* Not needed for atomics */
5032    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5033    mimg->dmask = (1 << data.size()) - 1;
5034    mimg->unrm = true;
5035    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5036    mimg->disable_wqm = true;
5037    mimg->barrier = barrier_image;
5038    ctx->program->needs_exact = true;
5039    ctx->block->instructions.emplace_back(std::move(mimg));
5040    return;
5041 }
5042
5043 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
5044 {
5045    if (in_elements && ctx->options->chip_class == GFX8) {
5046       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
5047       Builder bld(ctx->program, ctx->block);
5048
5049       Temp size = emit_extract_vector(ctx, desc, 2, s1);
5050
5051       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
5052       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
5053
5054       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
5055       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
5056
5057       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
5058       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
5059
5060       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
5061       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
5062                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
5063       if (dst.type() == RegType::vgpr)
5064          bld.copy(Definition(dst), shr_dst);
5065
5066       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
5067    } else {
5068       emit_extract_vector(ctx, desc, 2, dst);
5069    }
5070 }
5071
5072 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
5073 {
5074    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5075    const struct glsl_type *type = glsl_without_array(var->type);
5076    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5077    bool is_array = glsl_sampler_type_is_array(type);
5078    Builder bld(ctx->program, ctx->block);
5079
5080    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
5081       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
5082       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
5083    }
5084
5085    /* LOD */
5086    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
5087
5088    /* Resource */
5089    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
5090
5091    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5092
5093    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
5094    mimg->operands[0] = Operand(resource);
5095    mimg->operands[1] = Operand(s4); /* no sampler */
5096    mimg->operands[2] = Operand(lod);
5097    uint8_t& dmask = mimg->dmask;
5098    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5099    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
5100    mimg->da = glsl_sampler_type_is_array(type);
5101    mimg->can_reorder = true;
5102    Definition& def = mimg->definitions[0];
5103    ctx->block->instructions.emplace_back(std::move(mimg));
5104
5105    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
5106        glsl_sampler_type_is_array(type)) {
5107
5108       assert(instr->dest.ssa.num_components == 3);
5109       Temp tmp = {ctx->program->allocateId(), v3};
5110       def = Definition(tmp);
5111       emit_split_vector(ctx, tmp, 3);
5112
5113       /* divide 3rd value by 6 by multiplying with magic number */
5114       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
5115       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
5116
5117       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5118                  emit_extract_vector(ctx, tmp, 0, v1),
5119                  emit_extract_vector(ctx, tmp, 1, v1),
5120                  by_6);
5121
5122    } else if (ctx->options->chip_class == GFX9 &&
5123               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
5124               glsl_sampler_type_is_array(type)) {
5125       assert(instr->dest.ssa.num_components == 2);
5126       def = Definition(dst);
5127       dmask = 0x5;
5128    } else {
5129       def = Definition(dst);
5130    }
5131
5132    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
5133 }
5134
5135 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5136 {
5137    Builder bld(ctx->program, ctx->block);
5138    unsigned num_components = instr->num_components;
5139
5140    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5141    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5142    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5143
5144    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5145    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
5146 }
5147
5148 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5149 {
5150    Builder bld(ctx->program, ctx->block);
5151    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5152    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5153    unsigned writemask = nir_intrinsic_write_mask(instr);
5154    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
5155
5156    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5157    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5158
5159    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
5160                ctx->options->chip_class >= GFX8;
5161    if (smem)
5162       offset = bld.as_uniform(offset);
5163    bool smem_nonfs = smem && ctx->stage != fragment_fs;
5164
5165    while (writemask) {
5166       int start, count;
5167       u_bit_scan_consecutive_range(&writemask, &start, &count);
5168       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
5169          /* GFX6 doesn't support storing vec3, split it. */
5170          writemask |= 1u << (start + 2);
5171          count = 2;
5172       }
5173       int num_bytes = count * elem_size_bytes;
5174
5175       if (num_bytes > 16) {
5176          assert(elem_size_bytes == 8);
5177          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5178          count = 2;
5179          num_bytes = 16;
5180       }
5181
5182       // TODO: check alignment of sub-dword stores
5183       // TODO: split 3 bytes. there is no store instruction for that
5184
5185       Temp write_data;
5186       if (count != instr->num_components) {
5187          emit_split_vector(ctx, data, instr->num_components);
5188          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5189          for (int i = 0; i < count; i++) {
5190             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
5191             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
5192          }
5193          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
5194          vec->definitions[0] = Definition(write_data);
5195          ctx->block->instructions.emplace_back(std::move(vec));
5196       } else if (!smem && data.type() != RegType::vgpr) {
5197          assert(num_bytes % 4 == 0);
5198          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
5199       } else if (smem_nonfs && data.type() == RegType::vgpr) {
5200          assert(num_bytes % 4 == 0);
5201          write_data = bld.as_uniform(data);
5202       } else {
5203          write_data = data;
5204       }
5205
5206       aco_opcode vmem_op, smem_op;
5207       switch (num_bytes) {
5208          case 4:
5209             vmem_op = aco_opcode::buffer_store_dword;
5210             smem_op = aco_opcode::s_buffer_store_dword;
5211             break;
5212          case 8:
5213             vmem_op = aco_opcode::buffer_store_dwordx2;
5214             smem_op = aco_opcode::s_buffer_store_dwordx2;
5215             break;
5216          case 12:
5217             vmem_op = aco_opcode::buffer_store_dwordx3;
5218             smem_op = aco_opcode::last_opcode;
5219             assert(!smem && ctx->options->chip_class > GFX6);
5220             break;
5221          case 16:
5222             vmem_op = aco_opcode::buffer_store_dwordx4;
5223             smem_op = aco_opcode::s_buffer_store_dwordx4;
5224             break;
5225          default:
5226             unreachable("Store SSBO not implemented for this size.");
5227       }
5228       if (ctx->stage == fragment_fs)
5229          smem_op = aco_opcode::p_fs_buffer_store_smem;
5230
5231       if (smem) {
5232          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
5233          store->operands[0] = Operand(rsrc);
5234          if (start) {
5235             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
5236                                 offset, Operand(start * elem_size_bytes));
5237             store->operands[1] = Operand(off);
5238          } else {
5239             store->operands[1] = Operand(offset);
5240          }
5241          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
5242             store->operands[1].setFixed(m0);
5243          store->operands[2] = Operand(write_data);
5244          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5245          store->dlc = false;
5246          store->disable_wqm = true;
5247          store->barrier = barrier_buffer;
5248          ctx->block->instructions.emplace_back(std::move(store));
5249          ctx->program->wb_smem_l1_on_end = true;
5250          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
5251             ctx->block->kind |= block_kind_needs_lowering;
5252             ctx->program->needs_exact = true;
5253          }
5254       } else {
5255          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
5256          store->operands[0] = Operand(rsrc);
5257          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5258          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5259          store->operands[3] = Operand(write_data);
5260          store->offset = start * elem_size_bytes;
5261          store->offen = (offset.type() == RegType::vgpr);
5262          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5263          store->dlc = false;
5264          store->disable_wqm = true;
5265          store->barrier = barrier_buffer;
5266          ctx->program->needs_exact = true;
5267          ctx->block->instructions.emplace_back(std::move(store));
5268       }
5269    }
5270 }
5271
5272 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5273 {
5274    /* return the previous value if dest is ever used */
5275    bool return_previous = false;
5276    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5277       return_previous = true;
5278       break;
5279    }
5280    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5281       return_previous = true;
5282       break;
5283    }
5284
5285    Builder bld(ctx->program, ctx->block);
5286    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
5287
5288    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
5289       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5290                         get_ssa_temp(ctx, instr->src[3].ssa), data);
5291
5292    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
5293    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5294    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5295
5296    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5297
5298    aco_opcode op32, op64;
5299    switch (instr->intrinsic) {
5300       case nir_intrinsic_ssbo_atomic_add:
5301          op32 = aco_opcode::buffer_atomic_add;
5302          op64 = aco_opcode::buffer_atomic_add_x2;
5303          break;
5304       case nir_intrinsic_ssbo_atomic_imin:
5305          op32 = aco_opcode::buffer_atomic_smin;
5306          op64 = aco_opcode::buffer_atomic_smin_x2;
5307          break;
5308       case nir_intrinsic_ssbo_atomic_umin:
5309          op32 = aco_opcode::buffer_atomic_umin;
5310          op64 = aco_opcode::buffer_atomic_umin_x2;
5311          break;
5312       case nir_intrinsic_ssbo_atomic_imax:
5313          op32 = aco_opcode::buffer_atomic_smax;
5314          op64 = aco_opcode::buffer_atomic_smax_x2;
5315          break;
5316       case nir_intrinsic_ssbo_atomic_umax:
5317          op32 = aco_opcode::buffer_atomic_umax;
5318          op64 = aco_opcode::buffer_atomic_umax_x2;
5319          break;
5320       case nir_intrinsic_ssbo_atomic_and:
5321          op32 = aco_opcode::buffer_atomic_and;
5322          op64 = aco_opcode::buffer_atomic_and_x2;
5323          break;
5324       case nir_intrinsic_ssbo_atomic_or:
5325          op32 = aco_opcode::buffer_atomic_or;
5326          op64 = aco_opcode::buffer_atomic_or_x2;
5327          break;
5328       case nir_intrinsic_ssbo_atomic_xor:
5329          op32 = aco_opcode::buffer_atomic_xor;
5330          op64 = aco_opcode::buffer_atomic_xor_x2;
5331          break;
5332       case nir_intrinsic_ssbo_atomic_exchange:
5333          op32 = aco_opcode::buffer_atomic_swap;
5334          op64 = aco_opcode::buffer_atomic_swap_x2;
5335          break;
5336       case nir_intrinsic_ssbo_atomic_comp_swap:
5337          op32 = aco_opcode::buffer_atomic_cmpswap;
5338          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5339          break;
5340       default:
5341          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
5342    }
5343    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5344    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5345    mubuf->operands[0] = Operand(rsrc);
5346    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5347    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5348    mubuf->operands[3] = Operand(data);
5349    if (return_previous)
5350       mubuf->definitions[0] = Definition(dst);
5351    mubuf->offset = 0;
5352    mubuf->offen = (offset.type() == RegType::vgpr);
5353    mubuf->glc = return_previous;
5354    mubuf->dlc = false; /* Not needed for atomics */
5355    mubuf->disable_wqm = true;
5356    mubuf->barrier = barrier_buffer;
5357    ctx->program->needs_exact = true;
5358    ctx->block->instructions.emplace_back(std::move(mubuf));
5359 }
5360
5361 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
5362
5363    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5364    Builder bld(ctx->program, ctx->block);
5365    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
5366    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
5367 }
5368
5369 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
5370 {
5371    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5372                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5373
5374    if (addr.type() == RegType::vgpr)
5375       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
5376    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
5377 }
5378
5379 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
5380 {
5381    Builder bld(ctx->program, ctx->block);
5382    unsigned num_components = instr->num_components;
5383    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
5384
5385    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5386    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5387
5388    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5389    bool dlc = glc && ctx->options->chip_class >= GFX10;
5390    aco_opcode op;
5391    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
5392       bool global = ctx->options->chip_class >= GFX9;
5393
5394       if (ctx->options->chip_class >= GFX7) {
5395          aco_opcode op;
5396          switch (num_bytes) {
5397          case 4:
5398             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
5399             break;
5400          case 8:
5401             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
5402             break;
5403          case 12:
5404             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
5405             break;
5406          case 16:
5407             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
5408             break;
5409          default:
5410             unreachable("load_global not implemented for this size.");
5411          }
5412
5413          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
5414          flat->operands[0] = Operand(addr);
5415          flat->operands[1] = Operand(s1);
5416          flat->glc = glc;
5417          flat->dlc = dlc;
5418          flat->barrier = barrier_buffer;
5419
5420          if (dst.type() == RegType::sgpr) {
5421             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5422             flat->definitions[0] = Definition(vec);
5423             ctx->block->instructions.emplace_back(std::move(flat));
5424             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5425          } else {
5426             flat->definitions[0] = Definition(dst);
5427             ctx->block->instructions.emplace_back(std::move(flat));
5428          }
5429          emit_split_vector(ctx, dst, num_components);
5430       } else {
5431          assert(ctx->options->chip_class == GFX6);
5432
5433          /* GFX6 doesn't support loading vec3, expand to vec4. */
5434          num_bytes = num_bytes == 12 ? 16 : num_bytes;
5435
5436          aco_opcode op;
5437          switch (num_bytes) {
5438          case 4:
5439             op = aco_opcode::buffer_load_dword;
5440             break;
5441          case 8:
5442             op = aco_opcode::buffer_load_dwordx2;
5443             break;
5444          case 16:
5445             op = aco_opcode::buffer_load_dwordx4;
5446             break;
5447          default:
5448             unreachable("load_global not implemented for this size.");
5449          }
5450
5451          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5452
5453          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
5454          mubuf->operands[0] = Operand(rsrc);
5455          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5456          mubuf->operands[2] = Operand(0u);
5457          mubuf->glc = glc;
5458          mubuf->dlc = false;
5459          mubuf->offset = 0;
5460          mubuf->addr64 = addr.type() == RegType::vgpr;
5461          mubuf->disable_wqm = false;
5462          mubuf->barrier = barrier_buffer;
5463          aco_ptr<Instruction> instr = std::move(mubuf);
5464
5465          /* expand vector */
5466          if (dst.size() == 3) {
5467             Temp vec = bld.tmp(v4);
5468             instr->definitions[0] = Definition(vec);
5469             bld.insert(std::move(instr));
5470             emit_split_vector(ctx, vec, 4);
5471
5472             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5473             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5474             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5475             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5476          }
5477
5478          if (dst.type() == RegType::sgpr) {
5479             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5480             instr->definitions[0] = Definition(vec);
5481             bld.insert(std::move(instr));
5482             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5483             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5484          } else {
5485             instr->definitions[0] = Definition(dst);
5486             bld.insert(std::move(instr));
5487             emit_split_vector(ctx, dst, num_components);
5488          }
5489       }
5490    } else {
5491       switch (num_bytes) {
5492          case 4:
5493             op = aco_opcode::s_load_dword;
5494             break;
5495          case 8:
5496             op = aco_opcode::s_load_dwordx2;
5497             break;
5498          case 12:
5499          case 16:
5500             op = aco_opcode::s_load_dwordx4;
5501             break;
5502          default:
5503             unreachable("load_global not implemented for this size.");
5504       }
5505       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5506       load->operands[0] = Operand(addr);
5507       load->operands[1] = Operand(0u);
5508       load->definitions[0] = Definition(dst);
5509       load->glc = glc;
5510       load->dlc = dlc;
5511       load->barrier = barrier_buffer;
5512       assert(ctx->options->chip_class >= GFX8 || !glc);
5513
5514       if (dst.size() == 3) {
5515          /* trim vector */
5516          Temp vec = bld.tmp(s4);
5517          load->definitions[0] = Definition(vec);
5518          ctx->block->instructions.emplace_back(std::move(load));
5519          emit_split_vector(ctx, vec, 4);
5520
5521          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5522                     emit_extract_vector(ctx, vec, 0, s1),
5523                     emit_extract_vector(ctx, vec, 1, s1),
5524                     emit_extract_vector(ctx, vec, 2, s1));
5525       } else {
5526          ctx->block->instructions.emplace_back(std::move(load));
5527       }
5528    }
5529 }
5530
5531 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5532 {
5533    Builder bld(ctx->program, ctx->block);
5534    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5535
5536    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5537    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5538
5539    if (ctx->options->chip_class >= GFX7)
5540       addr = as_vgpr(ctx, addr);
5541
5542    unsigned writemask = nir_intrinsic_write_mask(instr);
5543    while (writemask) {
5544       int start, count;
5545       u_bit_scan_consecutive_range(&writemask, &start, &count);
5546       if (count == 3 && ctx->options->chip_class == GFX6) {
5547          /* GFX6 doesn't support storing vec3, split it. */
5548          writemask |= 1u << (start + 2);
5549          count = 2;
5550       }
5551       unsigned num_bytes = count * elem_size_bytes;
5552
5553       Temp write_data = data;
5554       if (count != instr->num_components) {
5555          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5556          for (int i = 0; i < count; i++)
5557             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5558          write_data = bld.tmp(RegType::vgpr, count);
5559          vec->definitions[0] = Definition(write_data);
5560          ctx->block->instructions.emplace_back(std::move(vec));
5561       }
5562
5563       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5564       unsigned offset = start * elem_size_bytes;
5565
5566       if (ctx->options->chip_class >= GFX7) {
5567          if (offset > 0 && ctx->options->chip_class < GFX9) {
5568             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5569             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5570             Temp carry = bld.tmp(bld.lm);
5571             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5572
5573             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5574                      Operand(offset), addr0);
5575             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5576                      Operand(0u), addr1,
5577                      carry).def(1).setHint(vcc);
5578
5579             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5580
5581             offset = 0;
5582          }
5583
5584          bool global = ctx->options->chip_class >= GFX9;
5585          aco_opcode op;
5586          switch (num_bytes) {
5587          case 4:
5588             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5589             break;
5590          case 8:
5591             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5592             break;
5593          case 12:
5594             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5595             break;
5596          case 16:
5597             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5598             break;
5599          default:
5600             unreachable("store_global not implemented for this size.");
5601          }
5602
5603          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5604          flat->operands[0] = Operand(addr);
5605          flat->operands[1] = Operand(s1);
5606          flat->operands[2] = Operand(data);
5607          flat->glc = glc;
5608          flat->dlc = false;
5609          flat->offset = offset;
5610          flat->disable_wqm = true;
5611          flat->barrier = barrier_buffer;
5612          ctx->program->needs_exact = true;
5613          ctx->block->instructions.emplace_back(std::move(flat));
5614       } else {
5615          assert(ctx->options->chip_class == GFX6);
5616
5617          aco_opcode op;
5618          switch (num_bytes) {
5619          case 4:
5620             op = aco_opcode::buffer_store_dword;
5621             break;
5622          case 8:
5623             op = aco_opcode::buffer_store_dwordx2;
5624             break;
5625          case 16:
5626             op = aco_opcode::buffer_store_dwordx4;
5627             break;
5628          default:
5629             unreachable("store_global not implemented for this size.");
5630          }
5631
5632          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5633
5634          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5635          mubuf->operands[0] = Operand(rsrc);
5636          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5637          mubuf->operands[2] = Operand(0u);
5638          mubuf->operands[3] = Operand(write_data);
5639          mubuf->glc = glc;
5640          mubuf->dlc = false;
5641          mubuf->offset = offset;
5642          mubuf->addr64 = addr.type() == RegType::vgpr;
5643          mubuf->disable_wqm = true;
5644          mubuf->barrier = barrier_buffer;
5645          ctx->program->needs_exact = true;
5646          ctx->block->instructions.emplace_back(std::move(mubuf));
5647       }
5648    }
5649 }
5650
5651 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5652 {
5653    /* return the previous value if dest is ever used */
5654    bool return_previous = false;
5655    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5656       return_previous = true;
5657       break;
5658    }
5659    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5660       return_previous = true;
5661       break;
5662    }
5663
5664    Builder bld(ctx->program, ctx->block);
5665    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5666    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5667
5668    if (ctx->options->chip_class >= GFX7)
5669       addr = as_vgpr(ctx, addr);
5670
5671    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5672       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5673                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5674
5675    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5676
5677    aco_opcode op32, op64;
5678
5679    if (ctx->options->chip_class >= GFX7) {
5680       bool global = ctx->options->chip_class >= GFX9;
5681       switch (instr->intrinsic) {
5682          case nir_intrinsic_global_atomic_add:
5683             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5684             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5685             break;
5686          case nir_intrinsic_global_atomic_imin:
5687             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5688             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5689             break;
5690          case nir_intrinsic_global_atomic_umin:
5691             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5692             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5693             break;
5694          case nir_intrinsic_global_atomic_imax:
5695             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5696             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5697             break;
5698          case nir_intrinsic_global_atomic_umax:
5699             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5700             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5701             break;
5702          case nir_intrinsic_global_atomic_and:
5703             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5704             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5705             break;
5706          case nir_intrinsic_global_atomic_or:
5707             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5708             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5709             break;
5710          case nir_intrinsic_global_atomic_xor:
5711             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5712             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5713             break;
5714          case nir_intrinsic_global_atomic_exchange:
5715             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5716             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5717             break;
5718          case nir_intrinsic_global_atomic_comp_swap:
5719             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5720             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5721             break;
5722          default:
5723             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5724       }
5725
5726       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5727       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5728       flat->operands[0] = Operand(addr);
5729       flat->operands[1] = Operand(s1);
5730       flat->operands[2] = Operand(data);
5731       if (return_previous)
5732          flat->definitions[0] = Definition(dst);
5733       flat->glc = return_previous;
5734       flat->dlc = false; /* Not needed for atomics */
5735       flat->offset = 0;
5736       flat->disable_wqm = true;
5737       flat->barrier = barrier_buffer;
5738       ctx->program->needs_exact = true;
5739       ctx->block->instructions.emplace_back(std::move(flat));
5740    } else {
5741       assert(ctx->options->chip_class == GFX6);
5742
5743       switch (instr->intrinsic) {
5744          case nir_intrinsic_global_atomic_add:
5745             op32 = aco_opcode::buffer_atomic_add;
5746             op64 = aco_opcode::buffer_atomic_add_x2;
5747             break;
5748          case nir_intrinsic_global_atomic_imin:
5749             op32 = aco_opcode::buffer_atomic_smin;
5750             op64 = aco_opcode::buffer_atomic_smin_x2;
5751             break;
5752          case nir_intrinsic_global_atomic_umin:
5753             op32 = aco_opcode::buffer_atomic_umin;
5754             op64 = aco_opcode::buffer_atomic_umin_x2;
5755             break;
5756          case nir_intrinsic_global_atomic_imax:
5757             op32 = aco_opcode::buffer_atomic_smax;
5758             op64 = aco_opcode::buffer_atomic_smax_x2;
5759             break;
5760          case nir_intrinsic_global_atomic_umax:
5761             op32 = aco_opcode::buffer_atomic_umax;
5762             op64 = aco_opcode::buffer_atomic_umax_x2;
5763             break;
5764          case nir_intrinsic_global_atomic_and:
5765             op32 = aco_opcode::buffer_atomic_and;
5766             op64 = aco_opcode::buffer_atomic_and_x2;
5767             break;
5768          case nir_intrinsic_global_atomic_or:
5769             op32 = aco_opcode::buffer_atomic_or;
5770             op64 = aco_opcode::buffer_atomic_or_x2;
5771             break;
5772          case nir_intrinsic_global_atomic_xor:
5773             op32 = aco_opcode::buffer_atomic_xor;
5774             op64 = aco_opcode::buffer_atomic_xor_x2;
5775             break;
5776          case nir_intrinsic_global_atomic_exchange:
5777             op32 = aco_opcode::buffer_atomic_swap;
5778             op64 = aco_opcode::buffer_atomic_swap_x2;
5779             break;
5780          case nir_intrinsic_global_atomic_comp_swap:
5781             op32 = aco_opcode::buffer_atomic_cmpswap;
5782             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5783             break;
5784          default:
5785             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5786       }
5787
5788       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5789
5790       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5791
5792       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5793       mubuf->operands[0] = Operand(rsrc);
5794       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5795       mubuf->operands[2] = Operand(0u);
5796       mubuf->operands[3] = Operand(data);
5797       if (return_previous)
5798          mubuf->definitions[0] = Definition(dst);
5799       mubuf->glc = return_previous;
5800       mubuf->dlc = false;
5801       mubuf->offset = 0;
5802       mubuf->addr64 = addr.type() == RegType::vgpr;
5803       mubuf->disable_wqm = true;
5804       mubuf->barrier = barrier_buffer;
5805       ctx->program->needs_exact = true;
5806       ctx->block->instructions.emplace_back(std::move(mubuf));
5807    }
5808 }
5809
5810 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5811    Builder bld(ctx->program, ctx->block);
5812    switch(instr->intrinsic) {
5813       case nir_intrinsic_group_memory_barrier:
5814       case nir_intrinsic_memory_barrier:
5815          bld.barrier(aco_opcode::p_memory_barrier_common);
5816          break;
5817       case nir_intrinsic_memory_barrier_buffer:
5818          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5819          break;
5820       case nir_intrinsic_memory_barrier_image:
5821          bld.barrier(aco_opcode::p_memory_barrier_image);
5822          break;
5823       case nir_intrinsic_memory_barrier_tcs_patch:
5824       case nir_intrinsic_memory_barrier_shared:
5825          bld.barrier(aco_opcode::p_memory_barrier_shared);
5826          break;
5827       default:
5828          unreachable("Unimplemented memory barrier intrinsic");
5829          break;
5830    }
5831 }
5832
5833 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5834 {
5835    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5836    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5837    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5838    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5839    Builder bld(ctx->program, ctx->block);
5840
5841    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5842    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5843    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5844 }
5845
5846 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5847 {
5848    unsigned writemask = nir_intrinsic_write_mask(instr);
5849    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5850    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5851    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5852    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5853
5854    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5855    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5856 }
5857
5858 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5859 {
5860    unsigned offset = nir_intrinsic_base(instr);
5861    Operand m = load_lds_size_m0(ctx);
5862    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5863    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5864
5865    unsigned num_operands = 3;
5866    aco_opcode op32, op64, op32_rtn, op64_rtn;
5867    switch(instr->intrinsic) {
5868       case nir_intrinsic_shared_atomic_add:
5869          op32 = aco_opcode::ds_add_u32;
5870          op64 = aco_opcode::ds_add_u64;
5871          op32_rtn = aco_opcode::ds_add_rtn_u32;
5872          op64_rtn = aco_opcode::ds_add_rtn_u64;
5873          break;
5874       case nir_intrinsic_shared_atomic_imin:
5875          op32 = aco_opcode::ds_min_i32;
5876          op64 = aco_opcode::ds_min_i64;
5877          op32_rtn = aco_opcode::ds_min_rtn_i32;
5878          op64_rtn = aco_opcode::ds_min_rtn_i64;
5879          break;
5880       case nir_intrinsic_shared_atomic_umin:
5881          op32 = aco_opcode::ds_min_u32;
5882          op64 = aco_opcode::ds_min_u64;
5883          op32_rtn = aco_opcode::ds_min_rtn_u32;
5884          op64_rtn = aco_opcode::ds_min_rtn_u64;
5885          break;
5886       case nir_intrinsic_shared_atomic_imax:
5887          op32 = aco_opcode::ds_max_i32;
5888          op64 = aco_opcode::ds_max_i64;
5889          op32_rtn = aco_opcode::ds_max_rtn_i32;
5890          op64_rtn = aco_opcode::ds_max_rtn_i64;
5891          break;
5892       case nir_intrinsic_shared_atomic_umax:
5893          op32 = aco_opcode::ds_max_u32;
5894          op64 = aco_opcode::ds_max_u64;
5895          op32_rtn = aco_opcode::ds_max_rtn_u32;
5896          op64_rtn = aco_opcode::ds_max_rtn_u64;
5897          break;
5898       case nir_intrinsic_shared_atomic_and:
5899          op32 = aco_opcode::ds_and_b32;
5900          op64 = aco_opcode::ds_and_b64;
5901          op32_rtn = aco_opcode::ds_and_rtn_b32;
5902          op64_rtn = aco_opcode::ds_and_rtn_b64;
5903          break;
5904       case nir_intrinsic_shared_atomic_or:
5905          op32 = aco_opcode::ds_or_b32;
5906          op64 = aco_opcode::ds_or_b64;
5907          op32_rtn = aco_opcode::ds_or_rtn_b32;
5908          op64_rtn = aco_opcode::ds_or_rtn_b64;
5909          break;
5910       case nir_intrinsic_shared_atomic_xor:
5911          op32 = aco_opcode::ds_xor_b32;
5912          op64 = aco_opcode::ds_xor_b64;
5913          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5914          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5915          break;
5916       case nir_intrinsic_shared_atomic_exchange:
5917          op32 = aco_opcode::ds_write_b32;
5918          op64 = aco_opcode::ds_write_b64;
5919          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5920          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5921          break;
5922       case nir_intrinsic_shared_atomic_comp_swap:
5923          op32 = aco_opcode::ds_cmpst_b32;
5924          op64 = aco_opcode::ds_cmpst_b64;
5925          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5926          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5927          num_operands = 4;
5928          break;
5929       default:
5930          unreachable("Unhandled shared atomic intrinsic");
5931    }
5932
5933    /* return the previous value if dest is ever used */
5934    bool return_previous = false;
5935    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5936       return_previous = true;
5937       break;
5938    }
5939    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5940       return_previous = true;
5941       break;
5942    }
5943
5944    aco_opcode op;
5945    if (data.size() == 1) {
5946       assert(instr->dest.ssa.bit_size == 32);
5947       op = return_previous ? op32_rtn : op32;
5948    } else {
5949       assert(instr->dest.ssa.bit_size == 64);
5950       op = return_previous ? op64_rtn : op64;
5951    }
5952
5953    if (offset > 65535) {
5954       Builder bld(ctx->program, ctx->block);
5955       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5956       offset = 0;
5957    }
5958
5959    aco_ptr<DS_instruction> ds;
5960    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5961    ds->operands[0] = Operand(address);
5962    ds->operands[1] = Operand(data);
5963    if (num_operands == 4)
5964       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5965    ds->operands[num_operands - 1] = m;
5966    ds->offset0 = offset;
5967    if (return_previous)
5968       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5969    ctx->block->instructions.emplace_back(std::move(ds));
5970 }
5971
5972 Temp get_scratch_resource(isel_context *ctx)
5973 {
5974    Builder bld(ctx->program, ctx->block);
5975    Temp scratch_addr = ctx->program->private_segment_buffer;
5976    if (ctx->stage != compute_cs)
5977       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5978
5979    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5980                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5981
5982    if (ctx->program->chip_class >= GFX10) {
5983       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5984                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5985                    S_008F0C_RESOURCE_LEVEL(1);
5986    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5987       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5988                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5989    }
5990
5991    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5992    if (ctx->program->chip_class <= GFX8)
5993       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5994
5995    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5996 }
5997
5998 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5999    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
6000    Builder bld(ctx->program, ctx->block);
6001    Temp rsrc = get_scratch_resource(ctx);
6002    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6003    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6004
6005    aco_opcode op;
6006    switch (dst.size()) {
6007       case 1:
6008          op = aco_opcode::buffer_load_dword;
6009          break;
6010       case 2:
6011          op = aco_opcode::buffer_load_dwordx2;
6012          break;
6013       case 3:
6014          op = aco_opcode::buffer_load_dwordx3;
6015          break;
6016       case 4:
6017          op = aco_opcode::buffer_load_dwordx4;
6018          break;
6019       case 6:
6020       case 8: {
6021          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
6022          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
6023                                 bld.def(v4), rsrc, offset,
6024                                 ctx->program->scratch_offset, 0, true);
6025          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
6026                                                   aco_opcode::buffer_load_dwordx4,
6027                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
6028                                 rsrc, offset, ctx->program->scratch_offset, 16, true);
6029          emit_split_vector(ctx, lower, 2);
6030          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
6031          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
6032          if (dst.size() == 8) {
6033             emit_split_vector(ctx, upper, 2);
6034             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
6035             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
6036          } else {
6037             elems[2] = upper;
6038          }
6039
6040          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6041                                                                          Format::PSEUDO, dst.size() / 2, 1)};
6042          for (unsigned i = 0; i < dst.size() / 2; i++)
6043             vec->operands[i] = Operand(elems[i]);
6044          vec->definitions[0] = Definition(dst);
6045          bld.insert(std::move(vec));
6046          ctx->allocated_vec.emplace(dst.id(), elems);
6047          return;
6048       }
6049       default:
6050          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
6051    }
6052
6053    bld.mubuf(op, Definition(dst), rsrc, offset, ctx->program->scratch_offset, 0, true);
6054    emit_split_vector(ctx, dst, instr->num_components);
6055 }
6056
6057 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6058    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
6059    Builder bld(ctx->program, ctx->block);
6060    Temp rsrc = get_scratch_resource(ctx);
6061    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6062    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6063
6064    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6065    unsigned writemask = nir_intrinsic_write_mask(instr);
6066
6067    while (writemask) {
6068       int start, count;
6069       u_bit_scan_consecutive_range(&writemask, &start, &count);
6070       int num_bytes = count * elem_size_bytes;
6071
6072       if (num_bytes > 16) {
6073          assert(elem_size_bytes == 8);
6074          writemask |= (((count - 2) << 1) - 1) << (start + 2);
6075          count = 2;
6076          num_bytes = 16;
6077       }
6078
6079       // TODO: check alignment of sub-dword stores
6080       // TODO: split 3 bytes. there is no store instruction for that
6081
6082       Temp write_data;
6083       if (count != instr->num_components) {
6084          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6085          for (int i = 0; i < count; i++) {
6086             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
6087             vec->operands[i] = Operand(elem);
6088          }
6089          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
6090          vec->definitions[0] = Definition(write_data);
6091          ctx->block->instructions.emplace_back(std::move(vec));
6092       } else {
6093          write_data = data;
6094       }
6095
6096       aco_opcode op;
6097       switch (num_bytes) {
6098          case 4:
6099             op = aco_opcode::buffer_store_dword;
6100             break;
6101          case 8:
6102             op = aco_opcode::buffer_store_dwordx2;
6103             break;
6104          case 12:
6105             op = aco_opcode::buffer_store_dwordx3;
6106             break;
6107          case 16:
6108             op = aco_opcode::buffer_store_dwordx4;
6109             break;
6110          default:
6111             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
6112       }
6113
6114       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
6115    }
6116 }
6117
6118 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
6119    uint8_t log2_ps_iter_samples;
6120    if (ctx->program->info->ps.force_persample) {
6121       log2_ps_iter_samples =
6122          util_logbase2(ctx->options->key.fs.num_samples);
6123    } else {
6124       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
6125    }
6126
6127    /* The bit pattern matches that used by fixed function fragment
6128     * processing. */
6129    static const unsigned ps_iter_masks[] = {
6130       0xffff, /* not used */
6131       0x5555,
6132       0x1111,
6133       0x0101,
6134       0x0001,
6135    };
6136    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
6137
6138    Builder bld(ctx->program, ctx->block);
6139
6140    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
6141                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6142    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
6143    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
6144    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6145    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
6146 }
6147
6148 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
6149    Builder bld(ctx->program, ctx->block);
6150
6151    unsigned stream = nir_intrinsic_stream_id(instr);
6152    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6153    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
6154    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
6155
6156    /* get GSVS ring */
6157    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
6158
6159    unsigned num_components =
6160       ctx->program->info->gs.num_stream_output_components[stream];
6161    assert(num_components);
6162
6163    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
6164    unsigned stream_offset = 0;
6165    for (unsigned i = 0; i < stream; i++) {
6166       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
6167       stream_offset += prev_stride * ctx->program->wave_size;
6168    }
6169
6170    /* Limit on the stride field for <= GFX7. */
6171    assert(stride < (1 << 14));
6172
6173    Temp gsvs_dwords[4];
6174    for (unsigned i = 0; i < 4; i++)
6175       gsvs_dwords[i] = bld.tmp(s1);
6176    bld.pseudo(aco_opcode::p_split_vector,
6177               Definition(gsvs_dwords[0]),
6178               Definition(gsvs_dwords[1]),
6179               Definition(gsvs_dwords[2]),
6180               Definition(gsvs_dwords[3]),
6181               gsvs_ring);
6182
6183    if (stream_offset) {
6184       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
6185
6186       Temp carry = bld.tmp(s1);
6187       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
6188       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
6189    }
6190
6191    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
6192    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
6193
6194    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6195                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
6196
6197    unsigned offset = 0;
6198    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
6199       if (ctx->program->info->gs.output_streams[i] != stream)
6200          continue;
6201
6202       for (unsigned j = 0; j < 4; j++) {
6203          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
6204             continue;
6205
6206          if (ctx->outputs.mask[i] & (1 << j)) {
6207             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
6208             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
6209             if (const_offset >= 4096u) {
6210                if (vaddr_offset.isUndefined())
6211                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
6212                else
6213                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
6214                const_offset %= 4096u;
6215             }
6216
6217             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
6218             mtbuf->operands[0] = Operand(gsvs_ring);
6219             mtbuf->operands[1] = vaddr_offset;
6220             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
6221             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
6222             mtbuf->offen = !vaddr_offset.isUndefined();
6223             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
6224             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
6225             mtbuf->offset = const_offset;
6226             mtbuf->glc = true;
6227             mtbuf->slc = true;
6228             mtbuf->barrier = barrier_gs_data;
6229             mtbuf->can_reorder = true;
6230             bld.insert(std::move(mtbuf));
6231          }
6232
6233          offset += ctx->shader->info.gs.vertices_out;
6234       }
6235
6236       /* outputs for the next vertex are undefined and keeping them around can
6237        * create invalid IR with control flow */
6238       ctx->outputs.mask[i] = 0;
6239    }
6240
6241    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
6242 }
6243
6244 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
6245 {
6246    Builder bld(ctx->program, ctx->block);
6247
6248    if (cluster_size == 1) {
6249       return src;
6250    } if (op == nir_op_iand && cluster_size == 4) {
6251       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
6252       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6253       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
6254                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
6255    } else if (op == nir_op_ior && cluster_size == 4) {
6256       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
6257       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
6258                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
6259    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
6260       //subgroupAnd(val) -> (exec & ~val) == 0
6261       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6262       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6263       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
6264    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
6265       //subgroupOr(val) -> (val & exec) != 0
6266       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
6267       return bool_to_vector_condition(ctx, tmp);
6268    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
6269       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
6270       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6271       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
6272       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
6273       return bool_to_vector_condition(ctx, tmp);
6274    } else {
6275       //subgroupClustered{And,Or,Xor}(val, n) ->
6276       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
6277       //cluster_offset = ~(n - 1) & lane_id
6278       //cluster_mask = ((1 << n) - 1)
6279       //subgroupClusteredAnd():
6280       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
6281       //subgroupClusteredOr():
6282       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
6283       //subgroupClusteredXor():
6284       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
6285       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
6286       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
6287
6288       Temp tmp;
6289       if (op == nir_op_iand)
6290          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6291       else
6292          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6293
6294       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
6295
6296       if (ctx->program->chip_class <= GFX7)
6297          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
6298       else if (ctx->program->wave_size == 64)
6299          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
6300       else
6301          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
6302       tmp = emit_extract_vector(ctx, tmp, 0, v1);
6303       if (cluster_mask != 0xffffffff)
6304          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
6305
6306       Definition cmp_def = Definition();
6307       if (op == nir_op_iand) {
6308          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
6309       } else if (op == nir_op_ior) {
6310          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6311       } else if (op == nir_op_ixor) {
6312          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
6313                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
6314          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6315       }
6316       cmp_def.setHint(vcc);
6317       return cmp_def.getTemp();
6318    }
6319 }
6320
6321 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
6322 {
6323    Builder bld(ctx->program, ctx->block);
6324
6325    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
6326    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
6327    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
6328    Temp tmp;
6329    if (op == nir_op_iand)
6330       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6331    else
6332       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
6333
6334    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
6335    Temp lo = lohi.def(0).getTemp();
6336    Temp hi = lohi.def(1).getTemp();
6337    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
6338
6339    Definition cmp_def = Definition();
6340    if (op == nir_op_iand)
6341       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6342    else if (op == nir_op_ior)
6343       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6344    else if (op == nir_op_ixor)
6345       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
6346                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
6347    cmp_def.setHint(vcc);
6348    return cmp_def.getTemp();
6349 }
6350
6351 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
6352 {
6353    Builder bld(ctx->program, ctx->block);
6354
6355    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
6356    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
6357    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
6358    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
6359    if (op == nir_op_iand)
6360       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6361    else if (op == nir_op_ior)
6362       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6363    else if (op == nir_op_ixor)
6364       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6365
6366    assert(false);
6367    return Temp();
6368 }
6369
6370 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
6371 {
6372    Builder bld(ctx->program, ctx->block);
6373    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
6374    if (src.regClass().type() == RegType::vgpr) {
6375       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
6376    } else if (src.regClass() == s1) {
6377       bld.sop1(aco_opcode::s_mov_b32, dst, src);
6378    } else if (src.regClass() == s2) {
6379       bld.sop1(aco_opcode::s_mov_b64, dst, src);
6380    } else {
6381       fprintf(stderr, "Unimplemented NIR instr bit size: ");
6382       nir_print_instr(&instr->instr, stderr);
6383       fprintf(stderr, "\n");
6384    }
6385 }
6386
6387 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
6388 {
6389    Builder bld(ctx->program, ctx->block);
6390    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
6391    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
6392    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
6393
6394    Temp ddx_1, ddx_2, ddy_1, ddy_2;
6395    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
6396    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
6397    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
6398
6399    /* Build DD X/Y */
6400    if (ctx->program->chip_class >= GFX8) {
6401       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
6402       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
6403       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
6404       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
6405       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
6406       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
6407    } else {
6408       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
6409       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
6410       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
6411       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
6412       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
6413       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
6414       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
6415       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
6416       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
6417       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
6418    }
6419
6420    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
6421    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
6422    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
6423    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
6424    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
6425    Temp wqm1 = bld.tmp(v1);
6426    emit_wqm(ctx, tmp1, wqm1, true);
6427    Temp wqm2 = bld.tmp(v1);
6428    emit_wqm(ctx, tmp2, wqm2, true);
6429    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
6430    return;
6431 }
6432
6433 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
6434 {
6435    Builder bld(ctx->program, ctx->block);
6436    switch(instr->intrinsic) {
6437    case nir_intrinsic_load_barycentric_sample:
6438    case nir_intrinsic_load_barycentric_pixel:
6439    case nir_intrinsic_load_barycentric_centroid: {
6440       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
6441       Temp bary = Temp(0, s2);
6442       switch (mode) {
6443       case INTERP_MODE_SMOOTH:
6444       case INTERP_MODE_NONE:
6445          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6446             bary = get_arg(ctx, ctx->args->ac.persp_center);
6447          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6448             bary = ctx->persp_centroid;
6449          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6450             bary = get_arg(ctx, ctx->args->ac.persp_sample);
6451          break;
6452       case INTERP_MODE_NOPERSPECTIVE:
6453          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6454             bary = get_arg(ctx, ctx->args->ac.linear_center);
6455          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6456             bary = ctx->linear_centroid;
6457          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6458             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6459          break;
6460       default:
6461          break;
6462       }
6463       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6464       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6465       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6466       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6467                  Operand(p1), Operand(p2));
6468       emit_split_vector(ctx, dst, 2);
6469       break;
6470    }
6471    case nir_intrinsic_load_barycentric_model: {
6472       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
6473
6474       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6475       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
6476       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
6477       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
6478       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6479                  Operand(p1), Operand(p2), Operand(p3));
6480       emit_split_vector(ctx, dst, 3);
6481       break;
6482    }
6483    case nir_intrinsic_load_barycentric_at_sample: {
6484       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6485       switch (ctx->options->key.fs.num_samples) {
6486          case 2: sample_pos_offset += 1 << 3; break;
6487          case 4: sample_pos_offset += 3 << 3; break;
6488          case 8: sample_pos_offset += 7 << 3; break;
6489          default: break;
6490       }
6491       Temp sample_pos;
6492       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6493       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6494       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6495       if (addr.type() == RegType::sgpr) {
6496          Operand offset;
6497          if (const_addr) {
6498             sample_pos_offset += const_addr->u32 << 3;
6499             offset = Operand(sample_pos_offset);
6500          } else if (ctx->options->chip_class >= GFX9) {
6501             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6502          } else {
6503             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6504             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6505          }
6506
6507          Operand off = bld.copy(bld.def(s1), Operand(offset));
6508          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6509
6510       } else if (ctx->options->chip_class >= GFX9) {
6511          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6512          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6513       } else if (ctx->options->chip_class >= GFX7) {
6514          /* addr += private_segment_buffer + sample_pos_offset */
6515          Temp tmp0 = bld.tmp(s1);
6516          Temp tmp1 = bld.tmp(s1);
6517          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6518          Definition scc_tmp = bld.def(s1, scc);
6519          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6520          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6521          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6522          Temp pck0 = bld.tmp(v1);
6523          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6524          tmp1 = as_vgpr(ctx, tmp1);
6525          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6526          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6527
6528          /* sample_pos = flat_load_dwordx2 addr */
6529          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6530       } else {
6531          assert(ctx->options->chip_class == GFX6);
6532
6533          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6534                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6535          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6536
6537          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6538          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6539
6540          sample_pos = bld.tmp(v2);
6541
6542          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6543          load->definitions[0] = Definition(sample_pos);
6544          load->operands[0] = Operand(rsrc);
6545          load->operands[1] = Operand(addr);
6546          load->operands[2] = Operand(0u);
6547          load->offset = sample_pos_offset;
6548          load->offen = 0;
6549          load->addr64 = true;
6550          load->glc = false;
6551          load->dlc = false;
6552          load->disable_wqm = false;
6553          load->barrier = barrier_none;
6554          load->can_reorder = true;
6555          ctx->block->instructions.emplace_back(std::move(load));
6556       }
6557
6558       /* sample_pos -= 0.5 */
6559       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6560       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6561       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6562       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6563       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6564
6565       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6566       break;
6567    }
6568    case nir_intrinsic_load_barycentric_at_offset: {
6569       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6570       RegClass rc = RegClass(offset.type(), 1);
6571       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6572       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6573       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6574       break;
6575    }
6576    case nir_intrinsic_load_front_face: {
6577       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6578                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6579       break;
6580    }
6581    case nir_intrinsic_load_view_index: {
6582       if (ctx->stage & (sw_vs | sw_gs | sw_tcs | sw_tes)) {
6583          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6584          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6585          break;
6586       }
6587
6588       /* fallthrough */
6589    }
6590    case nir_intrinsic_load_layer_id: {
6591       unsigned idx = nir_intrinsic_base(instr);
6592       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6593                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6594       break;
6595    }
6596    case nir_intrinsic_load_frag_coord: {
6597       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6598       break;
6599    }
6600    case nir_intrinsic_load_sample_pos: {
6601       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6602       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6603       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6604                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6605                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6606       break;
6607    }
6608    case nir_intrinsic_load_tess_coord:
6609       visit_load_tess_coord(ctx, instr);
6610       break;
6611    case nir_intrinsic_load_interpolated_input:
6612       visit_load_interpolated_input(ctx, instr);
6613       break;
6614    case nir_intrinsic_store_output:
6615       visit_store_output(ctx, instr);
6616       break;
6617    case nir_intrinsic_load_input:
6618    case nir_intrinsic_load_input_vertex:
6619       visit_load_input(ctx, instr);
6620       break;
6621    case nir_intrinsic_load_output:
6622       visit_load_output(ctx, instr);
6623       break;
6624    case nir_intrinsic_load_per_vertex_input:
6625       visit_load_per_vertex_input(ctx, instr);
6626       break;
6627    case nir_intrinsic_load_per_vertex_output:
6628       visit_load_per_vertex_output(ctx, instr);
6629       break;
6630    case nir_intrinsic_store_per_vertex_output:
6631       visit_store_per_vertex_output(ctx, instr);
6632       break;
6633    case nir_intrinsic_load_ubo:
6634       visit_load_ubo(ctx, instr);
6635       break;
6636    case nir_intrinsic_load_push_constant:
6637       visit_load_push_constant(ctx, instr);
6638       break;
6639    case nir_intrinsic_load_constant:
6640       visit_load_constant(ctx, instr);
6641       break;
6642    case nir_intrinsic_vulkan_resource_index:
6643       visit_load_resource(ctx, instr);
6644       break;
6645    case nir_intrinsic_discard:
6646       visit_discard(ctx, instr);
6647       break;
6648    case nir_intrinsic_discard_if:
6649       visit_discard_if(ctx, instr);
6650       break;
6651    case nir_intrinsic_load_shared:
6652       visit_load_shared(ctx, instr);
6653       break;
6654    case nir_intrinsic_store_shared:
6655       visit_store_shared(ctx, instr);
6656       break;
6657    case nir_intrinsic_shared_atomic_add:
6658    case nir_intrinsic_shared_atomic_imin:
6659    case nir_intrinsic_shared_atomic_umin:
6660    case nir_intrinsic_shared_atomic_imax:
6661    case nir_intrinsic_shared_atomic_umax:
6662    case nir_intrinsic_shared_atomic_and:
6663    case nir_intrinsic_shared_atomic_or:
6664    case nir_intrinsic_shared_atomic_xor:
6665    case nir_intrinsic_shared_atomic_exchange:
6666    case nir_intrinsic_shared_atomic_comp_swap:
6667       visit_shared_atomic(ctx, instr);
6668       break;
6669    case nir_intrinsic_image_deref_load:
6670       visit_image_load(ctx, instr);
6671       break;
6672    case nir_intrinsic_image_deref_store:
6673       visit_image_store(ctx, instr);
6674       break;
6675    case nir_intrinsic_image_deref_atomic_add:
6676    case nir_intrinsic_image_deref_atomic_umin:
6677    case nir_intrinsic_image_deref_atomic_imin:
6678    case nir_intrinsic_image_deref_atomic_umax:
6679    case nir_intrinsic_image_deref_atomic_imax:
6680    case nir_intrinsic_image_deref_atomic_and:
6681    case nir_intrinsic_image_deref_atomic_or:
6682    case nir_intrinsic_image_deref_atomic_xor:
6683    case nir_intrinsic_image_deref_atomic_exchange:
6684    case nir_intrinsic_image_deref_atomic_comp_swap:
6685       visit_image_atomic(ctx, instr);
6686       break;
6687    case nir_intrinsic_image_deref_size:
6688       visit_image_size(ctx, instr);
6689       break;
6690    case nir_intrinsic_load_ssbo:
6691       visit_load_ssbo(ctx, instr);
6692       break;
6693    case nir_intrinsic_store_ssbo:
6694       visit_store_ssbo(ctx, instr);
6695       break;
6696    case nir_intrinsic_load_global:
6697       visit_load_global(ctx, instr);
6698       break;
6699    case nir_intrinsic_store_global:
6700       visit_store_global(ctx, instr);
6701       break;
6702    case nir_intrinsic_global_atomic_add:
6703    case nir_intrinsic_global_atomic_imin:
6704    case nir_intrinsic_global_atomic_umin:
6705    case nir_intrinsic_global_atomic_imax:
6706    case nir_intrinsic_global_atomic_umax:
6707    case nir_intrinsic_global_atomic_and:
6708    case nir_intrinsic_global_atomic_or:
6709    case nir_intrinsic_global_atomic_xor:
6710    case nir_intrinsic_global_atomic_exchange:
6711    case nir_intrinsic_global_atomic_comp_swap:
6712       visit_global_atomic(ctx, instr);
6713       break;
6714    case nir_intrinsic_ssbo_atomic_add:
6715    case nir_intrinsic_ssbo_atomic_imin:
6716    case nir_intrinsic_ssbo_atomic_umin:
6717    case nir_intrinsic_ssbo_atomic_imax:
6718    case nir_intrinsic_ssbo_atomic_umax:
6719    case nir_intrinsic_ssbo_atomic_and:
6720    case nir_intrinsic_ssbo_atomic_or:
6721    case nir_intrinsic_ssbo_atomic_xor:
6722    case nir_intrinsic_ssbo_atomic_exchange:
6723    case nir_intrinsic_ssbo_atomic_comp_swap:
6724       visit_atomic_ssbo(ctx, instr);
6725       break;
6726    case nir_intrinsic_load_scratch:
6727       visit_load_scratch(ctx, instr);
6728       break;
6729    case nir_intrinsic_store_scratch:
6730       visit_store_scratch(ctx, instr);
6731       break;
6732    case nir_intrinsic_get_buffer_size:
6733       visit_get_buffer_size(ctx, instr);
6734       break;
6735    case nir_intrinsic_control_barrier: {
6736       if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
6737          /* GFX6 only (thanks to a hw bug workaround):
6738           * The real barrier instruction isn’t needed, because an entire patch
6739           * always fits into a single wave.
6740           */
6741          break;
6742       }
6743
6744       if (ctx->shader->info.stage == MESA_SHADER_COMPUTE) {
6745          unsigned* bsize = ctx->program->info->cs.block_size;
6746          unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6747          if (workgroup_size > ctx->program->wave_size)
6748             bld.sopp(aco_opcode::s_barrier);
6749       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
6750          /* For each patch provided during rendering, n TCS shader invocations will be processed,
6751           * where n is the number of vertices in the output patch.
6752           */
6753          unsigned workgroup_size = ctx->tcs_num_patches * ctx->shader->info.tess.tcs_vertices_out;
6754          if (workgroup_size > ctx->program->wave_size)
6755             bld.sopp(aco_opcode::s_barrier);
6756       } else {
6757          /* We don't know the workgroup size, so always emit the s_barrier. */
6758          bld.sopp(aco_opcode::s_barrier);
6759       }
6760
6761       break;
6762    }
6763    case nir_intrinsic_memory_barrier_tcs_patch:
6764    case nir_intrinsic_group_memory_barrier:
6765    case nir_intrinsic_memory_barrier:
6766    case nir_intrinsic_memory_barrier_buffer:
6767    case nir_intrinsic_memory_barrier_image:
6768    case nir_intrinsic_memory_barrier_shared:
6769       emit_memory_barrier(ctx, instr);
6770       break;
6771    case nir_intrinsic_load_num_work_groups: {
6772       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6773       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6774       emit_split_vector(ctx, dst, 3);
6775       break;
6776    }
6777    case nir_intrinsic_load_local_invocation_id: {
6778       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6779       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6780       emit_split_vector(ctx, dst, 3);
6781       break;
6782    }
6783    case nir_intrinsic_load_work_group_id: {
6784       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6785       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6786       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6787                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6788                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6789                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6790       emit_split_vector(ctx, dst, 3);
6791       break;
6792    }
6793    case nir_intrinsic_load_local_invocation_index: {
6794       Temp id = emit_mbcnt(ctx, bld.def(v1));
6795
6796       /* The tg_size bits [6:11] contain the subgroup id,
6797        * we need this multiplied by the wave size, and then OR the thread id to it.
6798        */
6799       if (ctx->program->wave_size == 64) {
6800          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6801          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6802                                 get_arg(ctx, ctx->args->ac.tg_size));
6803          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6804       } else {
6805          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6806          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6807                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6808          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6809       }
6810       break;
6811    }
6812    case nir_intrinsic_load_subgroup_id: {
6813       if (ctx->stage == compute_cs) {
6814          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6815                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6816       } else {
6817          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6818       }
6819       break;
6820    }
6821    case nir_intrinsic_load_subgroup_invocation: {
6822       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6823       break;
6824    }
6825    case nir_intrinsic_load_num_subgroups: {
6826       if (ctx->stage == compute_cs)
6827          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6828                   get_arg(ctx, ctx->args->ac.tg_size));
6829       else
6830          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6831       break;
6832    }
6833    case nir_intrinsic_ballot: {
6834       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6835       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6836       Definition tmp = bld.def(dst.regClass());
6837       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6838       if (instr->src[0].ssa->bit_size == 1) {
6839          assert(src.regClass() == bld.lm);
6840          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6841       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6842          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6843       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6844          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6845       } else {
6846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6847          nir_print_instr(&instr->instr, stderr);
6848          fprintf(stderr, "\n");
6849       }
6850       if (dst.size() != bld.lm.size()) {
6851          /* Wave32 with ballot size set to 64 */
6852          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6853       }
6854       emit_wqm(ctx, tmp.getTemp(), dst);
6855       break;
6856    }
6857    case nir_intrinsic_shuffle:
6858    case nir_intrinsic_read_invocation: {
6859       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6860       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6861          emit_uniform_subgroup(ctx, instr, src);
6862       } else {
6863          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6864          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6865             tid = bld.as_uniform(tid);
6866          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6867          if (src.regClass() == v1) {
6868             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6869          } else if (src.regClass() == v2) {
6870             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6871             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6872             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6873             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6874             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6875             emit_split_vector(ctx, dst, 2);
6876          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6877             assert(src.regClass() == bld.lm);
6878             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6879             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6880          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6881             assert(src.regClass() == bld.lm);
6882             Temp tmp;
6883             if (ctx->program->chip_class <= GFX7)
6884                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6885             else if (ctx->program->wave_size == 64)
6886                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6887             else
6888                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6889             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6890             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6891             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6892          } else {
6893             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6894             nir_print_instr(&instr->instr, stderr);
6895             fprintf(stderr, "\n");
6896          }
6897       }
6898       break;
6899    }
6900    case nir_intrinsic_load_sample_id: {
6901       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6902                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6903       break;
6904    }
6905    case nir_intrinsic_load_sample_mask_in: {
6906       visit_load_sample_mask_in(ctx, instr);
6907       break;
6908    }
6909    case nir_intrinsic_read_first_invocation: {
6910       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6911       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6912       if (src.regClass() == v1) {
6913          emit_wqm(ctx,
6914                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6915                   dst);
6916       } else if (src.regClass() == v2) {
6917          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6918          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6919          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6920          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6921          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6922          emit_split_vector(ctx, dst, 2);
6923       } else if (instr->dest.ssa.bit_size == 1) {
6924          assert(src.regClass() == bld.lm);
6925          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6926                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6927          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6928       } else if (src.regClass() == s1) {
6929          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6930       } else if (src.regClass() == s2) {
6931          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6932       } else {
6933          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6934          nir_print_instr(&instr->instr, stderr);
6935          fprintf(stderr, "\n");
6936       }
6937       break;
6938    }
6939    case nir_intrinsic_vote_all: {
6940       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6941       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6942       assert(src.regClass() == bld.lm);
6943       assert(dst.regClass() == bld.lm);
6944
6945       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6946       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6947       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6948       break;
6949    }
6950    case nir_intrinsic_vote_any: {
6951       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6952       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6953       assert(src.regClass() == bld.lm);
6954       assert(dst.regClass() == bld.lm);
6955
6956       Temp tmp = bool_to_scalar_condition(ctx, src);
6957       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6958       break;
6959    }
6960    case nir_intrinsic_reduce:
6961    case nir_intrinsic_inclusive_scan:
6962    case nir_intrinsic_exclusive_scan: {
6963       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6964       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6965       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6966       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6967          nir_intrinsic_cluster_size(instr) : 0;
6968       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6969
6970       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6971          emit_uniform_subgroup(ctx, instr, src);
6972       } else if (instr->dest.ssa.bit_size == 1) {
6973          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6974             op = nir_op_iand;
6975          else if (op == nir_op_iadd)
6976             op = nir_op_ixor;
6977          else if (op == nir_op_umax || op == nir_op_imax)
6978             op = nir_op_ior;
6979          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6980
6981          switch (instr->intrinsic) {
6982          case nir_intrinsic_reduce:
6983             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6984             break;
6985          case nir_intrinsic_exclusive_scan:
6986             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6987             break;
6988          case nir_intrinsic_inclusive_scan:
6989             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6990             break;
6991          default:
6992             assert(false);
6993          }
6994       } else if (cluster_size == 1) {
6995          bld.copy(Definition(dst), src);
6996       } else {
6997          src = as_vgpr(ctx, src);
6998
6999          ReduceOp reduce_op;
7000          switch (op) {
7001          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
7002             CASE(iadd)
7003             CASE(imul)
7004             CASE(fadd)
7005             CASE(fmul)
7006             CASE(imin)
7007             CASE(umin)
7008             CASE(fmin)
7009             CASE(imax)
7010             CASE(umax)
7011             CASE(fmax)
7012             CASE(iand)
7013             CASE(ior)
7014             CASE(ixor)
7015             default:
7016                unreachable("unknown reduction op");
7017          #undef CASE
7018          }
7019
7020          aco_opcode aco_op;
7021          switch (instr->intrinsic) {
7022             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
7023             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
7024             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
7025             default:
7026                unreachable("unknown reduce intrinsic");
7027          }
7028
7029          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
7030          reduce->operands[0] = Operand(src);
7031          // filled in by aco_reduce_assign.cpp, used internally as part of the
7032          // reduce sequence
7033          assert(dst.size() == 1 || dst.size() == 2);
7034          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
7035          reduce->operands[2] = Operand(v1.as_linear());
7036
7037          Temp tmp_dst = bld.tmp(dst.regClass());
7038          reduce->definitions[0] = Definition(tmp_dst);
7039          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
7040          reduce->definitions[2] = Definition();
7041          reduce->definitions[3] = Definition(scc, s1);
7042          reduce->definitions[4] = Definition();
7043          reduce->reduce_op = reduce_op;
7044          reduce->cluster_size = cluster_size;
7045          ctx->block->instructions.emplace_back(std::move(reduce));
7046
7047          emit_wqm(ctx, tmp_dst, dst);
7048       }
7049       break;
7050    }
7051    case nir_intrinsic_quad_broadcast: {
7052       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7053       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7054          emit_uniform_subgroup(ctx, instr, src);
7055       } else {
7056          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7057          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
7058          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
7059
7060          if (instr->dest.ssa.bit_size == 1) {
7061             assert(src.regClass() == bld.lm);
7062             assert(dst.regClass() == bld.lm);
7063             uint32_t half_mask = 0x11111111u << lane;
7064             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
7065             Temp tmp = bld.tmp(bld.lm);
7066             bld.sop1(Builder::s_wqm, Definition(tmp),
7067                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
7068                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
7069             emit_wqm(ctx, tmp, dst);
7070          } else if (instr->dest.ssa.bit_size == 32) {
7071             if (ctx->program->chip_class >= GFX8)
7072                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
7073             else
7074                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
7075          } else if (instr->dest.ssa.bit_size == 64) {
7076             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7077             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7078             if (ctx->program->chip_class >= GFX8) {
7079                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7080                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7081             } else {
7082                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
7083                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
7084             }
7085             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7086             emit_split_vector(ctx, dst, 2);
7087          } else {
7088             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7089             nir_print_instr(&instr->instr, stderr);
7090             fprintf(stderr, "\n");
7091          }
7092       }
7093       break;
7094    }
7095    case nir_intrinsic_quad_swap_horizontal:
7096    case nir_intrinsic_quad_swap_vertical:
7097    case nir_intrinsic_quad_swap_diagonal:
7098    case nir_intrinsic_quad_swizzle_amd: {
7099       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7100       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7101          emit_uniform_subgroup(ctx, instr, src);
7102          break;
7103       }
7104       uint16_t dpp_ctrl = 0;
7105       switch (instr->intrinsic) {
7106       case nir_intrinsic_quad_swap_horizontal:
7107          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
7108          break;
7109       case nir_intrinsic_quad_swap_vertical:
7110          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
7111          break;
7112       case nir_intrinsic_quad_swap_diagonal:
7113          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
7114          break;
7115       case nir_intrinsic_quad_swizzle_amd:
7116          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
7117          break;
7118       default:
7119          break;
7120       }
7121       if (ctx->program->chip_class < GFX8)
7122          dpp_ctrl |= (1 << 15);
7123
7124       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7125       if (instr->dest.ssa.bit_size == 1) {
7126          assert(src.regClass() == bld.lm);
7127          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
7128          if (ctx->program->chip_class >= GFX8)
7129             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7130          else
7131             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7132          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
7133          emit_wqm(ctx, tmp, dst);
7134       } else if (instr->dest.ssa.bit_size == 32) {
7135          Temp tmp;
7136          if (ctx->program->chip_class >= GFX8)
7137             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7138          else
7139             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7140          emit_wqm(ctx, tmp, dst);
7141       } else if (instr->dest.ssa.bit_size == 64) {
7142          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7143          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7144          if (ctx->program->chip_class >= GFX8) {
7145             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7146             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7147          } else {
7148             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
7149             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
7150          }
7151          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7152          emit_split_vector(ctx, dst, 2);
7153       } else {
7154          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7155          nir_print_instr(&instr->instr, stderr);
7156          fprintf(stderr, "\n");
7157       }
7158       break;
7159    }
7160    case nir_intrinsic_masked_swizzle_amd: {
7161       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7162       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7163          emit_uniform_subgroup(ctx, instr, src);
7164          break;
7165       }
7166       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7167       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
7168       if (dst.regClass() == v1) {
7169          emit_wqm(ctx,
7170                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
7171                   dst);
7172       } else if (dst.regClass() == v2) {
7173          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7174          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7175          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
7176          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
7177          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7178          emit_split_vector(ctx, dst, 2);
7179       } else {
7180          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7181          nir_print_instr(&instr->instr, stderr);
7182          fprintf(stderr, "\n");
7183       }
7184       break;
7185    }
7186    case nir_intrinsic_write_invocation_amd: {
7187       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
7188       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
7189       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
7190       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7191       if (dst.regClass() == v1) {
7192          /* src2 is ignored for writelane. RA assigns the same reg for dst */
7193          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
7194       } else if (dst.regClass() == v2) {
7195          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
7196          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
7197          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
7198          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
7199          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
7200          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
7201          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7202          emit_split_vector(ctx, dst, 2);
7203       } else {
7204          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7205          nir_print_instr(&instr->instr, stderr);
7206          fprintf(stderr, "\n");
7207       }
7208       break;
7209    }
7210    case nir_intrinsic_mbcnt_amd: {
7211       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7212       RegClass rc = RegClass(src.type(), 1);
7213       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
7214       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
7215       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7216       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
7217       emit_wqm(ctx, wqm_tmp, dst);
7218       break;
7219    }
7220    case nir_intrinsic_load_helper_invocation: {
7221       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7222       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
7223       ctx->block->kind |= block_kind_needs_lowering;
7224       ctx->program->needs_exact = true;
7225       break;
7226    }
7227    case nir_intrinsic_is_helper_invocation: {
7228       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7229       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
7230       ctx->block->kind |= block_kind_needs_lowering;
7231       ctx->program->needs_exact = true;
7232       break;
7233    }
7234    case nir_intrinsic_demote:
7235       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
7236
7237       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7238          ctx->cf_info.exec_potentially_empty_discard = true;
7239       ctx->block->kind |= block_kind_uses_demote;
7240       ctx->program->needs_exact = true;
7241       break;
7242    case nir_intrinsic_demote_if: {
7243       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7244       assert(src.regClass() == bld.lm);
7245       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
7246       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
7247
7248       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7249          ctx->cf_info.exec_potentially_empty_discard = true;
7250       ctx->block->kind |= block_kind_uses_demote;
7251       ctx->program->needs_exact = true;
7252       break;
7253    }
7254    case nir_intrinsic_first_invocation: {
7255       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
7256                get_ssa_temp(ctx, &instr->dest.ssa));
7257       break;
7258    }
7259    case nir_intrinsic_shader_clock:
7260       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
7261       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
7262       break;
7263    case nir_intrinsic_load_vertex_id_zero_base: {
7264       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7265       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
7266       break;
7267    }
7268    case nir_intrinsic_load_first_vertex: {
7269       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7270       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
7271       break;
7272    }
7273    case nir_intrinsic_load_base_instance: {
7274       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7275       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
7276       break;
7277    }
7278    case nir_intrinsic_load_instance_id: {
7279       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7280       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
7281       break;
7282    }
7283    case nir_intrinsic_load_draw_id: {
7284       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7285       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
7286       break;
7287    }
7288    case nir_intrinsic_load_invocation_id: {
7289       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7290
7291       if (ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
7292          if (ctx->options->chip_class >= GFX10)
7293             bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7294          else
7295             bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7296       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7297          bld.vop3(aco_opcode::v_bfe_u32, Definition(dst),
7298                   get_arg(ctx, ctx->args->ac.tcs_rel_ids), Operand(8u), Operand(5u));
7299       } else {
7300          unreachable("Unsupported stage for load_invocation_id");
7301       }
7302
7303       break;
7304    }
7305    case nir_intrinsic_load_primitive_id: {
7306       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7307
7308       switch (ctx->shader->info.stage) {
7309       case MESA_SHADER_GEOMETRY:
7310          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
7311          break;
7312       case MESA_SHADER_TESS_CTRL:
7313          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tcs_patch_id));
7314          break;
7315       case MESA_SHADER_TESS_EVAL:
7316          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tes_patch_id));
7317          break;
7318       default:
7319          unreachable("Unimplemented shader stage for nir_intrinsic_load_primitive_id");
7320       }
7321
7322       break;
7323    }
7324    case nir_intrinsic_load_patch_vertices_in: {
7325       assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL ||
7326              ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
7327
7328       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7329       bld.copy(Definition(dst), Operand(ctx->args->options->key.tcs.input_vertices));
7330       break;
7331    }
7332    case nir_intrinsic_emit_vertex_with_counter: {
7333       visit_emit_vertex_with_counter(ctx, instr);
7334       break;
7335    }
7336    case nir_intrinsic_end_primitive_with_counter: {
7337       unsigned stream = nir_intrinsic_stream_id(instr);
7338       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
7339       break;
7340    }
7341    case nir_intrinsic_set_vertex_count: {
7342       /* unused, the HW keeps track of this for us */
7343       break;
7344    }
7345    default:
7346       fprintf(stderr, "Unimplemented intrinsic instr: ");
7347       nir_print_instr(&instr->instr, stderr);
7348       fprintf(stderr, "\n");
7349       abort();
7350
7351       break;
7352    }
7353 }
7354
7355
7356 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
7357                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
7358                     enum glsl_base_type *stype)
7359 {
7360    nir_deref_instr *texture_deref_instr = NULL;
7361    nir_deref_instr *sampler_deref_instr = NULL;
7362    int plane = -1;
7363
7364    for (unsigned i = 0; i < instr->num_srcs; i++) {
7365       switch (instr->src[i].src_type) {
7366       case nir_tex_src_texture_deref:
7367          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
7368          break;
7369       case nir_tex_src_sampler_deref:
7370          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
7371          break;
7372       case nir_tex_src_plane:
7373          plane = nir_src_as_int(instr->src[i].src);
7374          break;
7375       default:
7376          break;
7377       }
7378    }
7379
7380    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
7381
7382    if (!sampler_deref_instr)
7383       sampler_deref_instr = texture_deref_instr;
7384
7385    if (plane >= 0) {
7386       assert(instr->op != nir_texop_txf_ms &&
7387              instr->op != nir_texop_samples_identical);
7388       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
7389       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
7390    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
7391       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
7392    } else if (instr->op == nir_texop_fragment_mask_fetch) {
7393       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
7394    } else {
7395       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
7396    }
7397    if (samp_ptr) {
7398       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
7399
7400       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
7401          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
7402          Builder bld(ctx->program, ctx->block);
7403
7404          /* to avoid unnecessary moves, we split and recombine sampler and image */
7405          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
7406                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
7407          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
7408          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
7409                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
7410                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
7411          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
7412                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
7413
7414          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
7415          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
7416                                img[0], img[1], img[2], img[3],
7417                                img[4], img[5], img[6], img[7]);
7418          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
7419                                 samp[0], samp[1], samp[2], samp[3]);
7420       }
7421    }
7422    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
7423                      instr->op == nir_texop_samples_identical))
7424       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
7425 }
7426
7427 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
7428                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
7429 {
7430    Builder bld(ctx->program, ctx->block);
7431
7432    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
7433    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
7434    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
7435
7436    Operand neg_one(0xbf800000u);
7437    Operand one(0x3f800000u);
7438    Operand two(0x40000000u);
7439    Operand four(0x40800000u);
7440
7441    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
7442    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
7443    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
7444
7445    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
7446    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
7447    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
7448    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
7449
7450    // select sc
7451    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
7452    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
7453                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
7454                        one, is_ma_y);
7455    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
7456
7457    // select tc
7458    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
7459    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
7460    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
7461
7462    // select ma
7463    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7464                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
7465                   deriv_z, is_ma_z);
7466    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
7467    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
7468 }
7469
7470 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
7471 {
7472    Builder bld(ctx->program, ctx->block);
7473    Temp ma, tc, sc, id;
7474
7475    if (is_array) {
7476       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
7477
7478       // see comment in ac_prepare_cube_coords()
7479       if (ctx->options->chip_class <= GFX8)
7480          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
7481    }
7482
7483    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7484
7485    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
7486    vop3a->operands[0] = Operand(ma);
7487    vop3a->abs[0] = true;
7488    Temp invma = bld.tmp(v1);
7489    vop3a->definitions[0] = Definition(invma);
7490    ctx->block->instructions.emplace_back(std::move(vop3a));
7491
7492    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7493    if (!is_deriv)
7494       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
7495
7496    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7497    if (!is_deriv)
7498       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
7499
7500    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7501
7502    if (is_deriv) {
7503       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
7504       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
7505
7506       for (unsigned i = 0; i < 2; i++) {
7507          // see comment in ac_prepare_cube_coords()
7508          Temp deriv_ma;
7509          Temp deriv_sc, deriv_tc;
7510          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
7511                            &deriv_ma, &deriv_sc, &deriv_tc);
7512
7513          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
7514
7515          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7516                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
7517                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
7518          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7519                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
7520                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
7521          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
7522       }
7523
7524       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
7525       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
7526    }
7527
7528    if (is_array)
7529       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
7530    coords.resize(3);
7531    coords[0] = sc;
7532    coords[1] = tc;
7533    coords[2] = id;
7534 }
7535
7536 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7537 {
7538    if (vec->parent_instr->type != nir_instr_type_alu)
7539       return;
7540    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7541    if (vec_instr->op != nir_op_vec(vec->num_components))
7542       return;
7543
7544    for (unsigned i = 0; i < vec->num_components; i++) {
7545       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7546               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7547    }
7548 }
7549
7550 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7551 {
7552    Builder bld(ctx->program, ctx->block);
7553    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7554         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7555    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
7556         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp();
7557    std::vector<Temp> coords;
7558    std::vector<Temp> derivs;
7559    nir_const_value *sample_index_cv = NULL;
7560    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7561    enum glsl_base_type stype;
7562    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7563
7564    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7565                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7566    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7567                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7568
7569    for (unsigned i = 0; i < instr->num_srcs; i++) {
7570       switch (instr->src[i].src_type) {
7571       case nir_tex_src_coord: {
7572          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
7573          for (unsigned i = 0; i < coord.size(); i++)
7574             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
7575          break;
7576       }
7577       case nir_tex_src_bias:
7578          if (instr->op == nir_texop_txb) {
7579             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7580             has_bias = true;
7581          }
7582          break;
7583       case nir_tex_src_lod: {
7584          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7585
7586          if (val && val->f32 <= 0.0) {
7587             level_zero = true;
7588          } else {
7589             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7590             has_lod = true;
7591          }
7592          break;
7593       }
7594       case nir_tex_src_comparator:
7595          if (instr->is_shadow) {
7596             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7597             has_compare = true;
7598          }
7599          break;
7600       case nir_tex_src_offset:
7601          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7602          get_const_vec(instr->src[i].src.ssa, const_offset);
7603          has_offset = true;
7604          break;
7605       case nir_tex_src_ddx:
7606          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7607          has_ddx = true;
7608          break;
7609       case nir_tex_src_ddy:
7610          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7611          has_ddy = true;
7612          break;
7613       case nir_tex_src_ms_index:
7614          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7615          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7616          has_sample_index = true;
7617          break;
7618       case nir_tex_src_texture_offset:
7619       case nir_tex_src_sampler_offset:
7620       default:
7621          break;
7622       }
7623    }
7624
7625    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7626       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7627
7628    if (instr->op == nir_texop_texture_samples) {
7629       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7630
7631       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7632       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7633       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7634       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7635
7636       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7637                samples, Operand(1u), bld.scc(is_msaa));
7638       return;
7639    }
7640
7641    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7642       aco_ptr<Instruction> tmp_instr;
7643       Temp acc, pack = Temp();
7644
7645       uint32_t pack_const = 0;
7646       for (unsigned i = 0; i < offset.size(); i++) {
7647          if (!const_offset[i])
7648             continue;
7649          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7650       }
7651
7652       if (offset.type() == RegType::sgpr) {
7653          for (unsigned i = 0; i < offset.size(); i++) {
7654             if (const_offset[i])
7655                continue;
7656
7657             acc = emit_extract_vector(ctx, offset, i, s1);
7658             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7659
7660             if (i) {
7661                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7662             }
7663
7664             if (pack == Temp()) {
7665                pack = acc;
7666             } else {
7667                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7668             }
7669          }
7670
7671          if (pack_const && pack != Temp())
7672             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7673       } else {
7674          for (unsigned i = 0; i < offset.size(); i++) {
7675             if (const_offset[i])
7676                continue;
7677
7678             acc = emit_extract_vector(ctx, offset, i, v1);
7679             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7680
7681             if (i) {
7682                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7683             }
7684
7685             if (pack == Temp()) {
7686                pack = acc;
7687             } else {
7688                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7689             }
7690          }
7691
7692          if (pack_const && pack != Temp())
7693             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7694       }
7695       if (pack_const && pack == Temp())
7696          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7697       else if (pack == Temp())
7698          has_offset = false;
7699       else
7700          offset = pack;
7701    }
7702
7703    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7704       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7705
7706    /* pack derivatives */
7707    if (has_ddx || has_ddy) {
7708       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7709          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
7710          Temp zero = bld.copy(bld.def(v1), Operand(0u));
7711          derivs = {ddy, zero, ddy, zero};
7712       } else {
7713          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
7714             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
7715          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
7716             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
7717       }
7718       has_derivs = true;
7719    }
7720
7721    if (instr->coord_components > 1 &&
7722        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7723        instr->is_array &&
7724        instr->op != nir_texop_txf)
7725       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
7726
7727    if (instr->coord_components > 2 &&
7728       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7729        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7730        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7731        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7732        instr->is_array &&
7733        instr->op != nir_texop_txf &&
7734        instr->op != nir_texop_txf_ms &&
7735        instr->op != nir_texop_fragment_fetch &&
7736        instr->op != nir_texop_fragment_mask_fetch)
7737       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
7738
7739    if (ctx->options->chip_class == GFX9 &&
7740        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7741        instr->op != nir_texop_lod && instr->coord_components) {
7742       assert(coords.size() > 0 && coords.size() < 3);
7743
7744       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
7745                                                                      Operand((uint32_t) 0) :
7746                                                                      Operand((uint32_t) 0x3f000000)));
7747    }
7748
7749    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7750
7751    if (instr->op == nir_texop_samples_identical)
7752       resource = fmask_ptr;
7753
7754    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7755              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7756             instr->op != nir_texop_txs &&
7757             instr->op != nir_texop_fragment_fetch &&
7758             instr->op != nir_texop_fragment_mask_fetch) {
7759       assert(has_sample_index);
7760       Operand op(sample_index);
7761       if (sample_index_cv)
7762          op = Operand(sample_index_cv->u32);
7763       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7764    }
7765
7766    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7767       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
7768          Temp off = emit_extract_vector(ctx, offset, i, v1);
7769          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
7770       }
7771       has_offset = false;
7772    }
7773
7774    /* Build tex instruction */
7775    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7776    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7777                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7778                   : 0;
7779    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7780    Temp tmp_dst = dst;
7781
7782    /* gather4 selects the component by dmask and always returns vec4 */
7783    if (instr->op == nir_texop_tg4) {
7784       assert(instr->dest.ssa.num_components == 4);
7785       if (instr->is_shadow)
7786          dmask = 1;
7787       else
7788          dmask = 1 << instr->component;
7789       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7790          tmp_dst = bld.tmp(v4);
7791    } else if (instr->op == nir_texop_samples_identical) {
7792       tmp_dst = bld.tmp(v1);
7793    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7794       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7795    }
7796
7797    aco_ptr<MIMG_instruction> tex;
7798    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7799       if (!has_lod)
7800          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7801
7802       bool div_by_6 = instr->op == nir_texop_txs &&
7803                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7804                       instr->is_array &&
7805                       (dmask & (1 << 2));
7806       if (tmp_dst.id() == dst.id() && div_by_6)
7807          tmp_dst = bld.tmp(tmp_dst.regClass());
7808
7809       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7810       tex->operands[0] = Operand(resource);
7811       tex->operands[1] = Operand(s4); /* no sampler */
7812       tex->operands[2] = Operand(as_vgpr(ctx,lod));
7813       if (ctx->options->chip_class == GFX9 &&
7814           instr->op == nir_texop_txs &&
7815           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7816           instr->is_array) {
7817          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7818       } else if (instr->op == nir_texop_query_levels) {
7819          tex->dmask = 1 << 3;
7820       } else {
7821          tex->dmask = dmask;
7822       }
7823       tex->da = da;
7824       tex->definitions[0] = Definition(tmp_dst);
7825       tex->dim = dim;
7826       tex->can_reorder = true;
7827       ctx->block->instructions.emplace_back(std::move(tex));
7828
7829       if (div_by_6) {
7830          /* divide 3rd value by 6 by multiplying with magic number */
7831          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7832          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7833          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7834          assert(instr->dest.ssa.num_components == 3);
7835          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7836          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7837                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7838                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7839                               by_6);
7840
7841       }
7842
7843       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7844       return;
7845    }
7846
7847    Temp tg4_compare_cube_wa64 = Temp();
7848
7849    if (tg4_integer_workarounds) {
7850       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7851       tex->operands[0] = Operand(resource);
7852       tex->operands[1] = Operand(s4); /* no sampler */
7853       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7854       tex->dim = dim;
7855       tex->dmask = 0x3;
7856       tex->da = da;
7857       Temp size = bld.tmp(v2);
7858       tex->definitions[0] = Definition(size);
7859       tex->can_reorder = true;
7860       ctx->block->instructions.emplace_back(std::move(tex));
7861       emit_split_vector(ctx, size, size.size());
7862
7863       Temp half_texel[2];
7864       for (unsigned i = 0; i < 2; i++) {
7865          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7866          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7867          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7868          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7869       }
7870
7871       Temp new_coords[2] = {
7872          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
7873          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
7874       };
7875
7876       if (tg4_integer_cube_workaround) {
7877          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7878          Temp desc[resource.size()];
7879          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7880                                                                            Format::PSEUDO, 1, resource.size())};
7881          split->operands[0] = Operand(resource);
7882          for (unsigned i = 0; i < resource.size(); i++) {
7883             desc[i] = bld.tmp(s1);
7884             split->definitions[i] = Definition(desc[i]);
7885          }
7886          ctx->block->instructions.emplace_back(std::move(split));
7887
7888          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7889          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7890                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7891
7892          Temp nfmt;
7893          if (stype == GLSL_TYPE_UINT) {
7894             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7895                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7896                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7897                             bld.scc(compare_cube_wa));
7898          } else {
7899             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7900                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7901                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7902                             bld.scc(compare_cube_wa));
7903          }
7904          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7905          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7906
7907          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7908
7909          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7910                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7911          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7912
7913          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7914                                                                          Format::PSEUDO, resource.size(), 1)};
7915          for (unsigned i = 0; i < resource.size(); i++)
7916             vec->operands[i] = Operand(desc[i]);
7917          resource = bld.tmp(resource.regClass());
7918          vec->definitions[0] = Definition(resource);
7919          ctx->block->instructions.emplace_back(std::move(vec));
7920
7921          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7922                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
7923          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7924                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
7925       }
7926       coords[0] = new_coords[0];
7927       coords[1] = new_coords[1];
7928    }
7929
7930    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7931       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7932
7933       assert(coords.size() == 1);
7934       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7935       aco_opcode op;
7936       switch (last_bit) {
7937       case 1:
7938          op = aco_opcode::buffer_load_format_x; break;
7939       case 2:
7940          op = aco_opcode::buffer_load_format_xy; break;
7941       case 3:
7942          op = aco_opcode::buffer_load_format_xyz; break;
7943       case 4:
7944          op = aco_opcode::buffer_load_format_xyzw; break;
7945       default:
7946          unreachable("Tex instruction loads more than 4 components.");
7947       }
7948
7949       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7950       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7951          tmp_dst = dst;
7952       else
7953          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7954
7955       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7956       mubuf->operands[0] = Operand(resource);
7957       mubuf->operands[1] = Operand(coords[0]);
7958       mubuf->operands[2] = Operand((uint32_t) 0);
7959       mubuf->definitions[0] = Definition(tmp_dst);
7960       mubuf->idxen = true;
7961       mubuf->can_reorder = true;
7962       ctx->block->instructions.emplace_back(std::move(mubuf));
7963
7964       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7965       return;
7966    }
7967
7968    /* gather MIMG address components */
7969    std::vector<Temp> args;
7970    if (has_offset)
7971       args.emplace_back(offset);
7972    if (has_bias)
7973       args.emplace_back(bias);
7974    if (has_compare)
7975       args.emplace_back(compare);
7976    if (has_derivs)
7977       args.insert(args.end(), derivs.begin(), derivs.end());
7978
7979    args.insert(args.end(), coords.begin(), coords.end());
7980    if (has_sample_index)
7981       args.emplace_back(sample_index);
7982    if (has_lod)
7983       args.emplace_back(lod);
7984
7985    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
7986    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7987    vec->definitions[0] = Definition(arg);
7988    for (unsigned i = 0; i < args.size(); i++)
7989       vec->operands[i] = Operand(args[i]);
7990    ctx->block->instructions.emplace_back(std::move(vec));
7991
7992
7993    if (instr->op == nir_texop_txf ||
7994        instr->op == nir_texop_txf_ms ||
7995        instr->op == nir_texop_samples_identical ||
7996        instr->op == nir_texop_fragment_fetch ||
7997        instr->op == nir_texop_fragment_mask_fetch) {
7998       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7999       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
8000       tex->operands[0] = Operand(resource);
8001       tex->operands[1] = Operand(s4); /* no sampler */
8002       tex->operands[2] = Operand(arg);
8003       tex->dim = dim;
8004       tex->dmask = dmask;
8005       tex->unrm = true;
8006       tex->da = da;
8007       tex->definitions[0] = Definition(tmp_dst);
8008       tex->can_reorder = true;
8009       ctx->block->instructions.emplace_back(std::move(tex));
8010
8011       if (instr->op == nir_texop_samples_identical) {
8012          assert(dmask == 1 && dst.regClass() == v1);
8013          assert(dst.id() != tmp_dst.id());
8014
8015          Temp tmp = bld.tmp(bld.lm);
8016          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
8017          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
8018
8019       } else {
8020          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8021       }
8022       return;
8023    }
8024
8025    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
8026    aco_opcode opcode = aco_opcode::image_sample;
8027    if (has_offset) { /* image_sample_*_o */
8028       if (has_compare) {
8029          opcode = aco_opcode::image_sample_c_o;
8030          if (has_derivs)
8031             opcode = aco_opcode::image_sample_c_d_o;
8032          if (has_bias)
8033             opcode = aco_opcode::image_sample_c_b_o;
8034          if (level_zero)
8035             opcode = aco_opcode::image_sample_c_lz_o;
8036          if (has_lod)
8037             opcode = aco_opcode::image_sample_c_l_o;
8038       } else {
8039          opcode = aco_opcode::image_sample_o;
8040          if (has_derivs)
8041             opcode = aco_opcode::image_sample_d_o;
8042          if (has_bias)
8043             opcode = aco_opcode::image_sample_b_o;
8044          if (level_zero)
8045             opcode = aco_opcode::image_sample_lz_o;
8046          if (has_lod)
8047             opcode = aco_opcode::image_sample_l_o;
8048       }
8049    } else { /* no offset */
8050       if (has_compare) {
8051          opcode = aco_opcode::image_sample_c;
8052          if (has_derivs)
8053             opcode = aco_opcode::image_sample_c_d;
8054          if (has_bias)
8055             opcode = aco_opcode::image_sample_c_b;
8056          if (level_zero)
8057             opcode = aco_opcode::image_sample_c_lz;
8058          if (has_lod)
8059             opcode = aco_opcode::image_sample_c_l;
8060       } else {
8061          opcode = aco_opcode::image_sample;
8062          if (has_derivs)
8063             opcode = aco_opcode::image_sample_d;
8064          if (has_bias)
8065             opcode = aco_opcode::image_sample_b;
8066          if (level_zero)
8067             opcode = aco_opcode::image_sample_lz;
8068          if (has_lod)
8069             opcode = aco_opcode::image_sample_l;
8070       }
8071    }
8072
8073    if (instr->op == nir_texop_tg4) {
8074       if (has_offset) {
8075          opcode = aco_opcode::image_gather4_lz_o;
8076          if (has_compare)
8077             opcode = aco_opcode::image_gather4_c_lz_o;
8078       } else {
8079          opcode = aco_opcode::image_gather4_lz;
8080          if (has_compare)
8081             opcode = aco_opcode::image_gather4_c_lz;
8082       }
8083    } else if (instr->op == nir_texop_lod) {
8084       opcode = aco_opcode::image_get_lod;
8085    }
8086
8087    /* we don't need the bias, sample index, compare value or offset to be
8088     * computed in WQM but if the p_create_vector copies the coordinates, then it
8089     * needs to be in WQM */
8090    if (ctx->stage == fragment_fs &&
8091        !has_derivs && !has_lod && !level_zero &&
8092        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
8093        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
8094       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
8095
8096    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
8097    tex->operands[0] = Operand(resource);
8098    tex->operands[1] = Operand(sampler);
8099    tex->operands[2] = Operand(arg);
8100    tex->dim = dim;
8101    tex->dmask = dmask;
8102    tex->da = da;
8103    tex->definitions[0] = Definition(tmp_dst);
8104    tex->can_reorder = true;
8105    ctx->block->instructions.emplace_back(std::move(tex));
8106
8107    if (tg4_integer_cube_workaround) {
8108       assert(tmp_dst.id() != dst.id());
8109       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
8110
8111       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8112       Temp val[4];
8113       for (unsigned i = 0; i < dst.size(); i++) {
8114          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
8115          Temp cvt_val;
8116          if (stype == GLSL_TYPE_UINT)
8117             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
8118          else
8119             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
8120          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
8121       }
8122       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
8123       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8124                            val[0], val[1], val[2], val[3]);
8125    }
8126    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
8127    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
8128
8129 }
8130
8131
8132 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
8133 {
8134    Temp tmp = get_ssa_temp(ctx, ssa);
8135    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
8136       return Operand(tmp.regClass());
8137    else
8138       return Operand(tmp);
8139 }
8140
8141 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
8142 {
8143    aco_ptr<Pseudo_instruction> phi;
8144    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8145    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
8146
8147    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
8148    logical |= ctx->block->kind & block_kind_merge;
8149    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
8150
8151    /* we want a sorted list of sources, since the predecessor list is also sorted */
8152    std::map<unsigned, nir_ssa_def*> phi_src;
8153    nir_foreach_phi_src(src, instr)
8154       phi_src[src->pred->index] = src->src.ssa;
8155
8156    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
8157    unsigned num_operands = 0;
8158    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
8159    unsigned num_defined = 0;
8160    unsigned cur_pred_idx = 0;
8161    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
8162       if (cur_pred_idx < preds.size()) {
8163          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
8164          unsigned block = ctx->cf_info.nir_to_aco[src.first];
8165          unsigned skipped = 0;
8166          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
8167             skipped++;
8168          if (cur_pred_idx + skipped < preds.size()) {
8169             for (unsigned i = 0; i < skipped; i++)
8170                operands[num_operands++] = Operand(dst.regClass());
8171             cur_pred_idx += skipped;
8172          } else {
8173             continue;
8174          }
8175       }
8176       cur_pred_idx++;
8177       Operand op = get_phi_operand(ctx, src.second);
8178       operands[num_operands++] = op;
8179       num_defined += !op.isUndefined();
8180    }
8181    /* handle block_kind_continue_or_break at loop exit blocks */
8182    while (cur_pred_idx++ < preds.size())
8183       operands[num_operands++] = Operand(dst.regClass());
8184
8185    if (num_defined == 0) {
8186       Builder bld(ctx->program, ctx->block);
8187       if (dst.regClass() == s1) {
8188          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
8189       } else if (dst.regClass() == v1) {
8190          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
8191       } else {
8192          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8193          for (unsigned i = 0; i < dst.size(); i++)
8194             vec->operands[i] = Operand(0u);
8195          vec->definitions[0] = Definition(dst);
8196          ctx->block->instructions.emplace_back(std::move(vec));
8197       }
8198       return;
8199    }
8200
8201    /* we can use a linear phi in some cases if one src is undef */
8202    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
8203       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
8204
8205       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
8206       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
8207       assert(invert->kind & block_kind_invert);
8208
8209       unsigned then_block = invert->linear_preds[0];
8210
8211       Block* insert_block = NULL;
8212       for (unsigned i = 0; i < num_operands; i++) {
8213          Operand op = operands[i];
8214          if (op.isUndefined())
8215             continue;
8216          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
8217          phi->operands[0] = op;
8218          break;
8219       }
8220       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
8221       phi->operands[1] = Operand(dst.regClass());
8222       phi->definitions[0] = Definition(dst);
8223       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
8224       return;
8225    }
8226
8227    /* try to scalarize vector phis */
8228    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
8229       // TODO: scalarize linear phis on divergent ifs
8230       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
8231       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
8232       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
8233          Operand src = operands[i];
8234          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
8235             can_scalarize = false;
8236       }
8237       if (can_scalarize) {
8238          unsigned num_components = instr->dest.ssa.num_components;
8239          assert(dst.size() % num_components == 0);
8240          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
8241
8242          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
8243          for (unsigned k = 0; k < num_components; k++) {
8244             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8245             for (unsigned i = 0; i < num_operands; i++) {
8246                Operand src = operands[i];
8247                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
8248             }
8249             Temp phi_dst = {ctx->program->allocateId(), rc};
8250             phi->definitions[0] = Definition(phi_dst);
8251             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8252             new_vec[k] = phi_dst;
8253             vec->operands[k] = Operand(phi_dst);
8254          }
8255          vec->definitions[0] = Definition(dst);
8256          ctx->block->instructions.emplace_back(std::move(vec));
8257          ctx->allocated_vec.emplace(dst.id(), new_vec);
8258          return;
8259       }
8260    }
8261
8262    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8263    for (unsigned i = 0; i < num_operands; i++)
8264       phi->operands[i] = operands[i];
8265    phi->definitions[0] = Definition(dst);
8266    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8267 }
8268
8269
8270 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
8271 {
8272    Temp dst = get_ssa_temp(ctx, &instr->def);
8273
8274    assert(dst.type() == RegType::sgpr);
8275
8276    if (dst.size() == 1) {
8277       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
8278    } else {
8279       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8280       for (unsigned i = 0; i < dst.size(); i++)
8281          vec->operands[i] = Operand(0u);
8282       vec->definitions[0] = Definition(dst);
8283       ctx->block->instructions.emplace_back(std::move(vec));
8284    }
8285 }
8286
8287 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
8288 {
8289    Builder bld(ctx->program, ctx->block);
8290    Block *logical_target;
8291    append_logical_end(ctx->block);
8292    unsigned idx = ctx->block->index;
8293
8294    switch (instr->type) {
8295    case nir_jump_break:
8296       logical_target = ctx->cf_info.parent_loop.exit;
8297       add_logical_edge(idx, logical_target);
8298       ctx->block->kind |= block_kind_break;
8299
8300       if (!ctx->cf_info.parent_if.is_divergent &&
8301           !ctx->cf_info.parent_loop.has_divergent_continue) {
8302          /* uniform break - directly jump out of the loop */
8303          ctx->block->kind |= block_kind_uniform;
8304          ctx->cf_info.has_branch = true;
8305          bld.branch(aco_opcode::p_branch);
8306          add_linear_edge(idx, logical_target);
8307          return;
8308       }
8309       ctx->cf_info.parent_loop.has_divergent_branch = true;
8310       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8311       break;
8312    case nir_jump_continue:
8313       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8314       add_logical_edge(idx, logical_target);
8315       ctx->block->kind |= block_kind_continue;
8316
8317       if (ctx->cf_info.parent_if.is_divergent) {
8318          /* for potential uniform breaks after this continue,
8319             we must ensure that they are handled correctly */
8320          ctx->cf_info.parent_loop.has_divergent_continue = true;
8321          ctx->cf_info.parent_loop.has_divergent_branch = true;
8322          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8323       } else {
8324          /* uniform continue - directly jump to the loop header */
8325          ctx->block->kind |= block_kind_uniform;
8326          ctx->cf_info.has_branch = true;
8327          bld.branch(aco_opcode::p_branch);
8328          add_linear_edge(idx, logical_target);
8329          return;
8330       }
8331       break;
8332    default:
8333       fprintf(stderr, "Unknown NIR jump instr: ");
8334       nir_print_instr(&instr->instr, stderr);
8335       fprintf(stderr, "\n");
8336       abort();
8337    }
8338
8339    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
8340       ctx->cf_info.exec_potentially_empty_break = true;
8341       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
8342    }
8343
8344    /* remove critical edges from linear CFG */
8345    bld.branch(aco_opcode::p_branch);
8346    Block* break_block = ctx->program->create_and_insert_block();
8347    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8348    break_block->kind |= block_kind_uniform;
8349    add_linear_edge(idx, break_block);
8350    /* the loop_header pointer might be invalidated by this point */
8351    if (instr->type == nir_jump_continue)
8352       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8353    add_linear_edge(break_block->index, logical_target);
8354    bld.reset(break_block);
8355    bld.branch(aco_opcode::p_branch);
8356
8357    Block* continue_block = ctx->program->create_and_insert_block();
8358    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8359    add_linear_edge(idx, continue_block);
8360    append_logical_start(continue_block);
8361    ctx->block = continue_block;
8362    return;
8363 }
8364
8365 void visit_block(isel_context *ctx, nir_block *block)
8366 {
8367    nir_foreach_instr(instr, block) {
8368       switch (instr->type) {
8369       case nir_instr_type_alu:
8370          visit_alu_instr(ctx, nir_instr_as_alu(instr));
8371          break;
8372       case nir_instr_type_load_const:
8373          visit_load_const(ctx, nir_instr_as_load_const(instr));
8374          break;
8375       case nir_instr_type_intrinsic:
8376          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
8377          break;
8378       case nir_instr_type_tex:
8379          visit_tex(ctx, nir_instr_as_tex(instr));
8380          break;
8381       case nir_instr_type_phi:
8382          visit_phi(ctx, nir_instr_as_phi(instr));
8383          break;
8384       case nir_instr_type_ssa_undef:
8385          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
8386          break;
8387       case nir_instr_type_deref:
8388          break;
8389       case nir_instr_type_jump:
8390          visit_jump(ctx, nir_instr_as_jump(instr));
8391          break;
8392       default:
8393          fprintf(stderr, "Unknown NIR instr type: ");
8394          nir_print_instr(instr, stderr);
8395          fprintf(stderr, "\n");
8396          //abort();
8397       }
8398    }
8399
8400    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8401       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
8402 }
8403
8404
8405
8406 static void visit_loop(isel_context *ctx, nir_loop *loop)
8407 {
8408    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
8409    append_logical_end(ctx->block);
8410    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
8411    Builder bld(ctx->program, ctx->block);
8412    bld.branch(aco_opcode::p_branch);
8413    unsigned loop_preheader_idx = ctx->block->index;
8414
8415    Block loop_exit = Block();
8416    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8417    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
8418
8419    Block* loop_header = ctx->program->create_and_insert_block();
8420    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
8421    loop_header->kind |= block_kind_loop_header;
8422    add_edge(loop_preheader_idx, loop_header);
8423    ctx->block = loop_header;
8424
8425    /* emit loop body */
8426    unsigned loop_header_idx = loop_header->index;
8427    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
8428    append_logical_start(ctx->block);
8429    visit_cf_list(ctx, &loop->body);
8430
8431    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
8432    if (!ctx->cf_info.has_branch) {
8433       append_logical_end(ctx->block);
8434       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
8435          /* Discards can result in code running with an empty exec mask.
8436           * This would result in divergent breaks not ever being taken. As a
8437           * workaround, break the loop when the loop mask is empty instead of
8438           * always continuing. */
8439          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
8440          unsigned block_idx = ctx->block->index;
8441
8442          /* create helper blocks to avoid critical edges */
8443          Block *break_block = ctx->program->create_and_insert_block();
8444          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8445          break_block->kind = block_kind_uniform;
8446          bld.reset(break_block);
8447          bld.branch(aco_opcode::p_branch);
8448          add_linear_edge(block_idx, break_block);
8449          add_linear_edge(break_block->index, &loop_exit);
8450
8451          Block *continue_block = ctx->program->create_and_insert_block();
8452          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8453          continue_block->kind = block_kind_uniform;
8454          bld.reset(continue_block);
8455          bld.branch(aco_opcode::p_branch);
8456          add_linear_edge(block_idx, continue_block);
8457          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
8458
8459          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8460             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
8461          ctx->block = &ctx->program->blocks[block_idx];
8462       } else {
8463          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
8464          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8465             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8466          else
8467             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8468       }
8469
8470       bld.reset(ctx->block);
8471       bld.branch(aco_opcode::p_branch);
8472    }
8473
8474    /* fixup phis in loop header from unreachable blocks */
8475    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
8476       bool linear = ctx->cf_info.has_branch;
8477       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
8478       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
8479          if ((logical && instr->opcode == aco_opcode::p_phi) ||
8480              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
8481             /* the last operand should be the one that needs to be removed */
8482             instr->operands.pop_back();
8483          } else if (!is_phi(instr)) {
8484             break;
8485          }
8486       }
8487    }
8488
8489    ctx->cf_info.has_branch = false;
8490
8491    // TODO: if the loop has not a single exit, we must add one °°
8492    /* emit loop successor block */
8493    ctx->block = ctx->program->insert_block(std::move(loop_exit));
8494    append_logical_start(ctx->block);
8495
8496    #if 0
8497    // TODO: check if it is beneficial to not branch on continues
8498    /* trim linear phis in loop header */
8499    for (auto&& instr : loop_entry->instructions) {
8500       if (instr->opcode == aco_opcode::p_linear_phi) {
8501          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8502          new_phi->definitions[0] = instr->definitions[0];
8503          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8504             new_phi->operands[i] = instr->operands[i];
8505          /* check that the remaining operands are all the same */
8506          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8507             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8508          instr.swap(new_phi);
8509       } else if (instr->opcode == aco_opcode::p_phi) {
8510          continue;
8511       } else {
8512          break;
8513       }
8514    }
8515    #endif
8516 }
8517
8518 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8519 {
8520    ic->cond = cond;
8521
8522    append_logical_end(ctx->block);
8523    ctx->block->kind |= block_kind_branch;
8524
8525    /* branch to linear then block */
8526    assert(cond.regClass() == ctx->program->lane_mask);
8527    aco_ptr<Pseudo_branch_instruction> branch;
8528    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8529    branch->operands[0] = Operand(cond);
8530    ctx->block->instructions.push_back(std::move(branch));
8531
8532    ic->BB_if_idx = ctx->block->index;
8533    ic->BB_invert = Block();
8534    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8535    /* Invert blocks are intentionally not marked as top level because they
8536     * are not part of the logical cfg. */
8537    ic->BB_invert.kind |= block_kind_invert;
8538    ic->BB_endif = Block();
8539    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8540    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8541
8542    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
8543    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
8544    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
8545    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8546    ctx->cf_info.parent_if.is_divergent = true;
8547
8548    /* divergent branches use cbranch_execz */
8549    ctx->cf_info.exec_potentially_empty_discard = false;
8550    ctx->cf_info.exec_potentially_empty_break = false;
8551    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8552
8553    /** emit logical then block */
8554    Block* BB_then_logical = ctx->program->create_and_insert_block();
8555    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8556    add_edge(ic->BB_if_idx, BB_then_logical);
8557    ctx->block = BB_then_logical;
8558    append_logical_start(BB_then_logical);
8559 }
8560
8561 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8562 {
8563    Block *BB_then_logical = ctx->block;
8564    append_logical_end(BB_then_logical);
8565     /* branch from logical then block to invert block */
8566    aco_ptr<Pseudo_branch_instruction> branch;
8567    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8568    BB_then_logical->instructions.emplace_back(std::move(branch));
8569    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8570    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8571       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8572    BB_then_logical->kind |= block_kind_uniform;
8573    assert(!ctx->cf_info.has_branch);
8574    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8575    ctx->cf_info.parent_loop.has_divergent_branch = false;
8576
8577    /** emit linear then block */
8578    Block* BB_then_linear = ctx->program->create_and_insert_block();
8579    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8580    BB_then_linear->kind |= block_kind_uniform;
8581    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8582    /* branch from linear then block to invert block */
8583    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8584    BB_then_linear->instructions.emplace_back(std::move(branch));
8585    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8586
8587    /** emit invert merge block */
8588    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8589    ic->invert_idx = ctx->block->index;
8590
8591    /* branch to linear else block (skip else) */
8592    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8593    branch->operands[0] = Operand(ic->cond);
8594    ctx->block->instructions.push_back(std::move(branch));
8595
8596    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
8597    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
8598    ic->exec_potentially_empty_break_depth_old =
8599       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8600    /* divergent branches use cbranch_execz */
8601    ctx->cf_info.exec_potentially_empty_discard = false;
8602    ctx->cf_info.exec_potentially_empty_break = false;
8603    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8604
8605    /** emit logical else block */
8606    Block* BB_else_logical = ctx->program->create_and_insert_block();
8607    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8608    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8609    add_linear_edge(ic->invert_idx, BB_else_logical);
8610    ctx->block = BB_else_logical;
8611    append_logical_start(BB_else_logical);
8612 }
8613
8614 static void end_divergent_if(isel_context *ctx, if_context *ic)
8615 {
8616    Block *BB_else_logical = ctx->block;
8617    append_logical_end(BB_else_logical);
8618
8619    /* branch from logical else block to endif block */
8620    aco_ptr<Pseudo_branch_instruction> branch;
8621    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8622    BB_else_logical->instructions.emplace_back(std::move(branch));
8623    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8624    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8625       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8626    BB_else_logical->kind |= block_kind_uniform;
8627
8628    assert(!ctx->cf_info.has_branch);
8629    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8630
8631
8632    /** emit linear else block */
8633    Block* BB_else_linear = ctx->program->create_and_insert_block();
8634    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8635    BB_else_linear->kind |= block_kind_uniform;
8636    add_linear_edge(ic->invert_idx, BB_else_linear);
8637
8638    /* branch from linear else block to endif block */
8639    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8640    BB_else_linear->instructions.emplace_back(std::move(branch));
8641    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8642
8643
8644    /** emit endif merge block */
8645    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8646    append_logical_start(ctx->block);
8647
8648
8649    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8650    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
8651    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
8652    ctx->cf_info.exec_potentially_empty_break_depth =
8653       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8654    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
8655        !ctx->cf_info.parent_if.is_divergent) {
8656       ctx->cf_info.exec_potentially_empty_break = false;
8657       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8658    }
8659    /* uniform control flow never has an empty exec-mask */
8660    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
8661       ctx->cf_info.exec_potentially_empty_discard = false;
8662       ctx->cf_info.exec_potentially_empty_break = false;
8663       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8664    }
8665 }
8666
8667 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8668 {
8669    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8670    Builder bld(ctx->program, ctx->block);
8671    aco_ptr<Pseudo_branch_instruction> branch;
8672
8673    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8674       /**
8675        * Uniform conditionals are represented in the following way*) :
8676        *
8677        * The linear and logical CFG:
8678        *                        BB_IF
8679        *                        /    \
8680        *       BB_THEN (logical)      BB_ELSE (logical)
8681        *                        \    /
8682        *                        BB_ENDIF
8683        *
8684        * *) Exceptions may be due to break and continue statements within loops
8685        *    If a break/continue happens within uniform control flow, it branches
8686        *    to the loop exit/entry block. Otherwise, it branches to the next
8687        *    merge block.
8688        **/
8689       append_logical_end(ctx->block);
8690       ctx->block->kind |= block_kind_uniform;
8691
8692       /* emit branch */
8693       assert(cond.regClass() == bld.lm);
8694       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8695       cond = bool_to_scalar_condition(ctx, cond);
8696
8697       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8698       branch->operands[0] = Operand(cond);
8699       branch->operands[0].setFixed(scc);
8700       ctx->block->instructions.emplace_back(std::move(branch));
8701
8702       unsigned BB_if_idx = ctx->block->index;
8703       Block BB_endif = Block();
8704       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8705       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8706
8707       /** emit then block */
8708       Block* BB_then = ctx->program->create_and_insert_block();
8709       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8710       add_edge(BB_if_idx, BB_then);
8711       append_logical_start(BB_then);
8712       ctx->block = BB_then;
8713       visit_cf_list(ctx, &if_stmt->then_list);
8714       BB_then = ctx->block;
8715       bool then_branch = ctx->cf_info.has_branch;
8716       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8717
8718       if (!then_branch) {
8719          append_logical_end(BB_then);
8720          /* branch from then block to endif block */
8721          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8722          BB_then->instructions.emplace_back(std::move(branch));
8723          add_linear_edge(BB_then->index, &BB_endif);
8724          if (!then_branch_divergent)
8725             add_logical_edge(BB_then->index, &BB_endif);
8726          BB_then->kind |= block_kind_uniform;
8727       }
8728
8729       ctx->cf_info.has_branch = false;
8730       ctx->cf_info.parent_loop.has_divergent_branch = false;
8731
8732       /** emit else block */
8733       Block* BB_else = ctx->program->create_and_insert_block();
8734       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8735       add_edge(BB_if_idx, BB_else);
8736       append_logical_start(BB_else);
8737       ctx->block = BB_else;
8738       visit_cf_list(ctx, &if_stmt->else_list);
8739       BB_else = ctx->block;
8740
8741       if (!ctx->cf_info.has_branch) {
8742          append_logical_end(BB_else);
8743          /* branch from then block to endif block */
8744          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8745          BB_else->instructions.emplace_back(std::move(branch));
8746          add_linear_edge(BB_else->index, &BB_endif);
8747          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8748             add_logical_edge(BB_else->index, &BB_endif);
8749          BB_else->kind |= block_kind_uniform;
8750       }
8751
8752       ctx->cf_info.has_branch &= then_branch;
8753       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8754
8755       /** emit endif merge block */
8756       if (!ctx->cf_info.has_branch) {
8757          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8758          append_logical_start(ctx->block);
8759       }
8760    } else { /* non-uniform condition */
8761       /**
8762        * To maintain a logical and linear CFG without critical edges,
8763        * non-uniform conditionals are represented in the following way*) :
8764        *
8765        * The linear CFG:
8766        *                        BB_IF
8767        *                        /    \
8768        *       BB_THEN (logical)      BB_THEN (linear)
8769        *                        \    /
8770        *                        BB_INVERT (linear)
8771        *                        /    \
8772        *       BB_ELSE (logical)      BB_ELSE (linear)
8773        *                        \    /
8774        *                        BB_ENDIF
8775        *
8776        * The logical CFG:
8777        *                        BB_IF
8778        *                        /    \
8779        *       BB_THEN (logical)      BB_ELSE (logical)
8780        *                        \    /
8781        *                        BB_ENDIF
8782        *
8783        * *) Exceptions may be due to break and continue statements within loops
8784        **/
8785
8786       if_context ic;
8787
8788       begin_divergent_if_then(ctx, &ic, cond);
8789       visit_cf_list(ctx, &if_stmt->then_list);
8790
8791       begin_divergent_if_else(ctx, &ic);
8792       visit_cf_list(ctx, &if_stmt->else_list);
8793
8794       end_divergent_if(ctx, &ic);
8795    }
8796 }
8797
8798 static void visit_cf_list(isel_context *ctx,
8799                           struct exec_list *list)
8800 {
8801    foreach_list_typed(nir_cf_node, node, node, list) {
8802       switch (node->type) {
8803       case nir_cf_node_block:
8804          visit_block(ctx, nir_cf_node_as_block(node));
8805          break;
8806       case nir_cf_node_if:
8807          visit_if(ctx, nir_cf_node_as_if(node));
8808          break;
8809       case nir_cf_node_loop:
8810          visit_loop(ctx, nir_cf_node_as_loop(node));
8811          break;
8812       default:
8813          unreachable("unimplemented cf list type");
8814       }
8815    }
8816 }
8817
8818 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8819 {
8820    assert(ctx->stage == vertex_vs ||
8821           ctx->stage == tess_eval_vs ||
8822           ctx->stage == gs_copy_vs);
8823
8824    int offset = ctx->stage == tess_eval_vs
8825                 ? ctx->program->info->tes.outinfo.vs_output_param_offset[slot]
8826                 : ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8827    uint64_t mask = ctx->outputs.mask[slot];
8828    if (!is_pos && !mask)
8829       return;
8830    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8831       return;
8832    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8833    exp->enabled_mask = mask;
8834    for (unsigned i = 0; i < 4; ++i) {
8835       if (mask & (1 << i))
8836          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8837       else
8838          exp->operands[i] = Operand(v1);
8839    }
8840    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8841     * Setting valid_mask=1 prevents it and has no other effect.
8842     */
8843    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8844    exp->done = false;
8845    exp->compressed = false;
8846    if (is_pos)
8847       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8848    else
8849       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8850    ctx->block->instructions.emplace_back(std::move(exp));
8851 }
8852
8853 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8854 {
8855    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8856    exp->enabled_mask = 0;
8857    for (unsigned i = 0; i < 4; ++i)
8858       exp->operands[i] = Operand(v1);
8859    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8860       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8861       exp->enabled_mask |= 0x1;
8862    }
8863    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8864       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8865       exp->enabled_mask |= 0x4;
8866    }
8867    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8868       if (ctx->options->chip_class < GFX9) {
8869          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8870          exp->enabled_mask |= 0x8;
8871       } else {
8872          Builder bld(ctx->program, ctx->block);
8873
8874          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8875                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8876          if (exp->operands[2].isTemp())
8877             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8878
8879          exp->operands[2] = Operand(out);
8880          exp->enabled_mask |= 0x4;
8881       }
8882    }
8883    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8884    exp->done = false;
8885    exp->compressed = false;
8886    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8887    ctx->block->instructions.emplace_back(std::move(exp));
8888 }
8889
8890 static void create_vs_exports(isel_context *ctx)
8891 {
8892    assert(ctx->stage == vertex_vs ||
8893           ctx->stage == tess_eval_vs ||
8894           ctx->stage == gs_copy_vs);
8895
8896    radv_vs_output_info *outinfo = ctx->stage == tess_eval_vs
8897                                   ? &ctx->program->info->tes.outinfo
8898                                   : &ctx->program->info->vs.outinfo;
8899
8900    if (outinfo->export_prim_id) {
8901       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8902       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8903    }
8904
8905    if (ctx->options->key.has_multiview_view_index) {
8906       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8907       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8908    }
8909
8910    /* the order these position exports are created is important */
8911    int next_pos = 0;
8912    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8913    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8914       export_vs_psiz_layer_viewport(ctx, &next_pos);
8915    }
8916    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8917       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8918    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8919       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8920
8921    if (ctx->export_clip_dists) {
8922       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8923          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8924       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8925          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8926    }
8927
8928    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8929       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8930           i != VARYING_SLOT_PRIMITIVE_ID)
8931          continue;
8932
8933       export_vs_varying(ctx, i, false, NULL);
8934    }
8935 }
8936
8937 static void export_fs_mrt_z(isel_context *ctx)
8938 {
8939    Builder bld(ctx->program, ctx->block);
8940    unsigned enabled_channels = 0;
8941    bool compr = false;
8942    Operand values[4];
8943
8944    for (unsigned i = 0; i < 4; ++i) {
8945       values[i] = Operand(v1);
8946    }
8947
8948    /* Both stencil and sample mask only need 16-bits. */
8949    if (!ctx->program->info->ps.writes_z &&
8950        (ctx->program->info->ps.writes_stencil ||
8951         ctx->program->info->ps.writes_sample_mask)) {
8952       compr = true; /* COMPR flag */
8953
8954       if (ctx->program->info->ps.writes_stencil) {
8955          /* Stencil should be in X[23:16]. */
8956          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8957          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8958          enabled_channels |= 0x3;
8959       }
8960
8961       if (ctx->program->info->ps.writes_sample_mask) {
8962          /* SampleMask should be in Y[15:0]. */
8963          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8964          enabled_channels |= 0xc;
8965      }
8966    } else {
8967       if (ctx->program->info->ps.writes_z) {
8968          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8969          enabled_channels |= 0x1;
8970       }
8971
8972       if (ctx->program->info->ps.writes_stencil) {
8973          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8974          enabled_channels |= 0x2;
8975       }
8976
8977       if (ctx->program->info->ps.writes_sample_mask) {
8978          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8979          enabled_channels |= 0x4;
8980       }
8981    }
8982
8983    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8984     * writemask component.
8985     */
8986    if (ctx->options->chip_class == GFX6 &&
8987        ctx->options->family != CHIP_OLAND &&
8988        ctx->options->family != CHIP_HAINAN) {
8989             enabled_channels |= 0x1;
8990    }
8991
8992    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8993            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8994 }
8995
8996 static void export_fs_mrt_color(isel_context *ctx, int slot)
8997 {
8998    Builder bld(ctx->program, ctx->block);
8999    unsigned write_mask = ctx->outputs.mask[slot];
9000    Operand values[4];
9001
9002    for (unsigned i = 0; i < 4; ++i) {
9003       if (write_mask & (1 << i)) {
9004          values[i] = Operand(ctx->outputs.outputs[slot][i]);
9005       } else {
9006          values[i] = Operand(v1);
9007       }
9008    }
9009
9010    unsigned target, col_format;
9011    unsigned enabled_channels = 0;
9012    aco_opcode compr_op = (aco_opcode)0;
9013
9014    slot -= FRAG_RESULT_DATA0;
9015    target = V_008DFC_SQ_EXP_MRT + slot;
9016    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
9017
9018    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
9019    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
9020
9021    switch (col_format)
9022    {
9023    case V_028714_SPI_SHADER_ZERO:
9024       enabled_channels = 0; /* writemask */
9025       target = V_008DFC_SQ_EXP_NULL;
9026       break;
9027
9028    case V_028714_SPI_SHADER_32_R:
9029       enabled_channels = 1;
9030       break;
9031
9032    case V_028714_SPI_SHADER_32_GR:
9033       enabled_channels = 0x3;
9034       break;
9035
9036    case V_028714_SPI_SHADER_32_AR:
9037       if (ctx->options->chip_class >= GFX10) {
9038          /* Special case: on GFX10, the outputs are different for 32_AR */
9039          enabled_channels = 0x3;
9040          values[1] = values[3];
9041          values[3] = Operand(v1);
9042       } else {
9043          enabled_channels = 0x9;
9044       }
9045       break;
9046
9047    case V_028714_SPI_SHADER_FP16_ABGR:
9048       enabled_channels = 0x5;
9049       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
9050       break;
9051
9052    case V_028714_SPI_SHADER_UNORM16_ABGR:
9053       enabled_channels = 0x5;
9054       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
9055       break;
9056
9057    case V_028714_SPI_SHADER_SNORM16_ABGR:
9058       enabled_channels = 0x5;
9059       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
9060       break;
9061
9062    case V_028714_SPI_SHADER_UINT16_ABGR: {
9063       enabled_channels = 0x5;
9064       compr_op = aco_opcode::v_cvt_pk_u16_u32;
9065       if (is_int8 || is_int10) {
9066          /* clamp */
9067          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
9068          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9069
9070          for (unsigned i = 0; i < 4; i++) {
9071             if ((write_mask >> i) & 1) {
9072                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
9073                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
9074                                     values[i]);
9075             }
9076          }
9077       }
9078       break;
9079    }
9080
9081    case V_028714_SPI_SHADER_SINT16_ABGR:
9082       enabled_channels = 0x5;
9083       compr_op = aco_opcode::v_cvt_pk_i16_i32;
9084       if (is_int8 || is_int10) {
9085          /* clamp */
9086          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
9087          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
9088          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9089          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
9090
9091          for (unsigned i = 0; i < 4; i++) {
9092             if ((write_mask >> i) & 1) {
9093                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
9094                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
9095                                     values[i]);
9096                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
9097                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
9098                                     values[i]);
9099             }
9100          }
9101       }
9102       break;
9103
9104    case V_028714_SPI_SHADER_32_ABGR:
9105       enabled_channels = 0xF;
9106       break;
9107
9108    default:
9109       break;
9110    }
9111
9112    if (target == V_008DFC_SQ_EXP_NULL)
9113       return;
9114
9115    if ((bool) compr_op) {
9116       for (int i = 0; i < 2; i++) {
9117          /* check if at least one of the values to be compressed is enabled */
9118          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
9119          if (enabled) {
9120             enabled_channels |= enabled << (i*2);
9121             values[i] = bld.vop3(compr_op, bld.def(v1),
9122                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
9123                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
9124          } else {
9125             values[i] = Operand(v1);
9126          }
9127       }
9128       values[2] = Operand(v1);
9129       values[3] = Operand(v1);
9130    } else {
9131       for (int i = 0; i < 4; i++)
9132          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
9133    }
9134
9135    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9136            enabled_channels, target, (bool) compr_op);
9137 }
9138
9139 static void create_fs_exports(isel_context *ctx)
9140 {
9141    /* Export depth, stencil and sample mask. */
9142    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
9143        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
9144        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
9145       export_fs_mrt_z(ctx);
9146    }
9147
9148    /* Export all color render targets. */
9149    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
9150       if (ctx->outputs.mask[i])
9151          export_fs_mrt_color(ctx, i);
9152    }
9153 }
9154
9155 static void write_tcs_tess_factors(isel_context *ctx)
9156 {
9157    unsigned outer_comps;
9158    unsigned inner_comps;
9159
9160    switch (ctx->args->options->key.tcs.primitive_mode) {
9161    case GL_ISOLINES:
9162       outer_comps = 2;
9163       inner_comps = 0;
9164       break;
9165    case GL_TRIANGLES:
9166       outer_comps = 3;
9167       inner_comps = 1;
9168       break;
9169    case GL_QUADS:
9170       outer_comps = 4;
9171       inner_comps = 2;
9172       break;
9173    default:
9174       return;
9175    }
9176
9177    const unsigned tess_index_inner = shader_io_get_unique_index(VARYING_SLOT_TESS_LEVEL_INNER);
9178    const unsigned tess_index_outer = shader_io_get_unique_index(VARYING_SLOT_TESS_LEVEL_OUTER);
9179
9180    Builder bld(ctx->program, ctx->block);
9181
9182    bld.barrier(aco_opcode::p_memory_barrier_shared);
9183    unsigned workgroup_size = ctx->tcs_num_patches * ctx->shader->info.tess.tcs_vertices_out;
9184    if (unlikely(ctx->program->chip_class != GFX6 && workgroup_size > ctx->program->wave_size))
9185       bld.sopp(aco_opcode::s_barrier);
9186
9187    Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids);
9188    Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u));
9189
9190    Temp invocation_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), invocation_id);
9191    if_context ic_invocation_id_is_zero;
9192    begin_divergent_if_then(ctx, &ic_invocation_id_is_zero, invocation_id_is_zero);
9193    bld.reset(ctx->block);
9194
9195    Temp hs_ring_tess_factor = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_FACTOR * 16u));
9196
9197    std::pair<Temp, unsigned> lds_base = get_tcs_output_lds_offset(ctx);
9198    unsigned stride = inner_comps + outer_comps;
9199    Temp inner[4];
9200    Temp outer[4];
9201    Temp out[6];
9202    assert(inner_comps <= (sizeof(inner) / sizeof(Temp)));
9203    assert(outer_comps <= (sizeof(outer) / sizeof(Temp)));
9204    assert(stride <= (sizeof(out) / sizeof(Temp)));
9205
9206    if (ctx->args->options->key.tcs.primitive_mode == GL_ISOLINES) {
9207       // LINES reversal
9208       outer[0] = out[1] = load_lds(ctx, 4, bld.tmp(v1), lds_base.first, lds_base.second + tess_index_outer * 16 + 0 * 4, 4);
9209       outer[1] = out[0] = load_lds(ctx, 4, bld.tmp(v1), lds_base.first, lds_base.second + tess_index_outer * 16 + 1 * 4, 4);
9210    } else {
9211       for (unsigned i = 0; i < outer_comps; ++i)
9212          outer[i] = out[i] = load_lds(ctx, 4, bld.tmp(v1), lds_base.first, lds_base.second + tess_index_outer * 16 + i * 4, 4);
9213
9214       for (unsigned i = 0; i < inner_comps; ++i)
9215          inner[i] = out[outer_comps + i] = load_lds(ctx, 4, bld.tmp(v1), lds_base.first, lds_base.second + tess_index_inner * 16 + i * 4, 4);
9216    }
9217
9218    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
9219    Temp tf_base = get_arg(ctx, ctx->args->tess_factor_offset);
9220    Temp byte_offset = bld.v_mul_imm(bld.def(v1), rel_patch_id, stride * 4u);
9221    unsigned tf_const_offset = 0;
9222
9223    if (ctx->program->chip_class <= GFX8) {
9224       Temp rel_patch_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), rel_patch_id);
9225       if_context ic_rel_patch_id_is_zero;
9226       begin_divergent_if_then(ctx, &ic_rel_patch_id_is_zero, rel_patch_id_is_zero);
9227       bld.reset(ctx->block);
9228
9229       /* Store the dynamic HS control word. */
9230       Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u));
9231       bld.mubuf(aco_opcode::buffer_store_dword,
9232                 /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word,
9233                 /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false,
9234                 /* disable_wqm */ false, /* glc */ true);
9235       tf_const_offset += 4;
9236
9237       begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero);
9238       end_divergent_if(ctx, &ic_rel_patch_id_is_zero);
9239       bld.reset(ctx->block);
9240    }
9241
9242    assert(stride == 2 || stride == 4 || stride == 6);
9243    Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr);
9244    store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false);
9245
9246    begin_divergent_if_else(ctx, &ic_invocation_id_is_zero);
9247    end_divergent_if(ctx, &ic_invocation_id_is_zero);
9248 }
9249
9250 static void emit_stream_output(isel_context *ctx,
9251                                Temp const *so_buffers,
9252                                Temp const *so_write_offset,
9253                                const struct radv_stream_output *output)
9254 {
9255    unsigned num_comps = util_bitcount(output->component_mask);
9256    unsigned writemask = (1 << num_comps) - 1;
9257    unsigned loc = output->location;
9258    unsigned buf = output->buffer;
9259
9260    assert(num_comps && num_comps <= 4);
9261    if (!num_comps || num_comps > 4)
9262       return;
9263
9264    unsigned start = ffs(output->component_mask) - 1;
9265
9266    Temp out[4];
9267    bool all_undef = true;
9268    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
9269    for (unsigned i = 0; i < num_comps; i++) {
9270       out[i] = ctx->outputs.outputs[loc][start + i];
9271       all_undef = all_undef && !out[i].id();
9272    }
9273    if (all_undef)
9274       return;
9275
9276    while (writemask) {
9277       int start, count;
9278       u_bit_scan_consecutive_range(&writemask, &start, &count);
9279       if (count == 3 && ctx->options->chip_class == GFX6) {
9280          /* GFX6 doesn't support storing vec3, split it. */
9281          writemask |= 1u << (start + 2);
9282          count = 2;
9283       }
9284
9285       unsigned offset = output->offset + start * 4;
9286
9287       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
9288       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
9289       for (int i = 0; i < count; ++i)
9290          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
9291       vec->definitions[0] = Definition(write_data);
9292       ctx->block->instructions.emplace_back(std::move(vec));
9293
9294       aco_opcode opcode;
9295       switch (count) {
9296       case 1:
9297          opcode = aco_opcode::buffer_store_dword;
9298          break;
9299       case 2:
9300          opcode = aco_opcode::buffer_store_dwordx2;
9301          break;
9302       case 3:
9303          opcode = aco_opcode::buffer_store_dwordx3;
9304          break;
9305       case 4:
9306          opcode = aco_opcode::buffer_store_dwordx4;
9307          break;
9308       default:
9309          unreachable("Unsupported dword count.");
9310       }
9311
9312       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
9313       store->operands[0] = Operand(so_buffers[buf]);
9314       store->operands[1] = Operand(so_write_offset[buf]);
9315       store->operands[2] = Operand((uint32_t) 0);
9316       store->operands[3] = Operand(write_data);
9317       if (offset > 4095) {
9318          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
9319          Builder bld(ctx->program, ctx->block);
9320          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
9321       } else {
9322          store->offset = offset;
9323       }
9324       store->offen = true;
9325       store->glc = true;
9326       store->dlc = false;
9327       store->slc = true;
9328       store->can_reorder = true;
9329       ctx->block->instructions.emplace_back(std::move(store));
9330    }
9331 }
9332
9333 static void emit_streamout(isel_context *ctx, unsigned stream)
9334 {
9335    Builder bld(ctx->program, ctx->block);
9336
9337    Temp so_buffers[4];
9338    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
9339    for (unsigned i = 0; i < 4; i++) {
9340       unsigned stride = ctx->program->info->so.strides[i];
9341       if (!stride)
9342          continue;
9343
9344       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
9345       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
9346    }
9347
9348    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9349                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
9350
9351    Temp tid = emit_mbcnt(ctx, bld.def(v1));
9352
9353    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
9354
9355    if_context ic;
9356    begin_divergent_if_then(ctx, &ic, can_emit);
9357
9358    bld.reset(ctx->block);
9359
9360    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
9361
9362    Temp so_write_offset[4];
9363
9364    for (unsigned i = 0; i < 4; i++) {
9365       unsigned stride = ctx->program->info->so.strides[i];
9366       if (!stride)
9367          continue;
9368
9369       if (stride == 1) {
9370          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
9371                                 get_arg(ctx, ctx->args->streamout_write_idx),
9372                                 get_arg(ctx, ctx->args->streamout_offset[i]));
9373          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
9374
9375          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
9376       } else {
9377          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
9378          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
9379                                  get_arg(ctx, ctx->args->streamout_offset[i]));
9380          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
9381       }
9382    }
9383
9384    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
9385       struct radv_stream_output *output =
9386          &ctx->program->info->so.outputs[i];
9387       if (stream != output->stream)
9388          continue;
9389
9390       emit_stream_output(ctx, so_buffers, so_write_offset, output);
9391    }
9392
9393    begin_divergent_if_else(ctx, &ic);
9394    end_divergent_if(ctx, &ic);
9395 }
9396
9397 } /* end namespace */
9398
9399 void fix_ls_vgpr_init_bug(isel_context *ctx, Pseudo_instruction *startpgm)
9400 {
9401    assert(ctx->shader->info.stage == MESA_SHADER_VERTEX);
9402    Builder bld(ctx->program, ctx->block);
9403    constexpr unsigned hs_idx = 1u;
9404    Builder::Result hs_thread_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9405                                               get_arg(ctx, ctx->args->merged_wave_info),
9406                                               Operand((8u << 16) | (hs_idx * 8u)));
9407    Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp());
9408
9409    /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */
9410
9411    Temp instance_id = bld.sop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9412                                get_arg(ctx, ctx->args->ac.instance_id),
9413                                get_arg(ctx, ctx->args->rel_auto_id),
9414                                ls_has_nonzero_hs_threads);
9415    Temp rel_auto_id = bld.sop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9416                                get_arg(ctx, ctx->args->rel_auto_id),
9417                                get_arg(ctx, ctx->args->ac.tcs_rel_ids),
9418                                ls_has_nonzero_hs_threads);
9419    Temp vertex_id = bld.sop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9420                              get_arg(ctx, ctx->args->ac.vertex_id),
9421                              get_arg(ctx, ctx->args->ac.tcs_patch_id),
9422                              ls_has_nonzero_hs_threads);
9423
9424    ctx->arg_temps[ctx->args->ac.instance_id.arg_index] = instance_id;
9425    ctx->arg_temps[ctx->args->rel_auto_id.arg_index] = rel_auto_id;
9426    ctx->arg_temps[ctx->args->ac.vertex_id.arg_index] = vertex_id;
9427 }
9428
9429 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
9430 {
9431    /* Split all arguments except for the first (ring_offsets) and the last
9432     * (exec) so that the dead channels don't stay live throughout the program.
9433     */
9434    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
9435       if (startpgm->definitions[i].regClass().size() > 1) {
9436          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
9437                            startpgm->definitions[i].regClass().size());
9438       }
9439    }
9440 }
9441
9442 void handle_bc_optimize(isel_context *ctx)
9443 {
9444    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
9445    Builder bld(ctx->program, ctx->block);
9446    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
9447    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
9448    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
9449    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
9450    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
9451    if (uses_center && uses_centroid) {
9452       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
9453                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
9454
9455       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
9456          Temp new_coord[2];
9457          for (unsigned i = 0; i < 2; i++) {
9458             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
9459             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
9460             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9461                                     persp_centroid, persp_center, sel);
9462          }
9463          ctx->persp_centroid = bld.tmp(v2);
9464          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
9465                     Operand(new_coord[0]), Operand(new_coord[1]));
9466          emit_split_vector(ctx, ctx->persp_centroid, 2);
9467       }
9468
9469       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
9470          Temp new_coord[2];
9471          for (unsigned i = 0; i < 2; i++) {
9472             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
9473             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
9474             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9475                                     linear_centroid, linear_center, sel);
9476          }
9477          ctx->linear_centroid = bld.tmp(v2);
9478          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
9479                     Operand(new_coord[0]), Operand(new_coord[1]));
9480          emit_split_vector(ctx, ctx->linear_centroid, 2);
9481       }
9482    }
9483 }
9484
9485 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
9486 {
9487    Program *program = ctx->program;
9488
9489    unsigned float_controls = shader->info.float_controls_execution_mode;
9490
9491    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
9492       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
9493    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
9494       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
9495                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
9496
9497    program->next_fp_mode.must_flush_denorms32 =
9498       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
9499    program->next_fp_mode.must_flush_denorms16_64 =
9500       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
9501                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
9502
9503    program->next_fp_mode.care_about_round32 =
9504       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
9505
9506    program->next_fp_mode.care_about_round16_64 =
9507       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
9508                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
9509
9510    /* default to preserving fp16 and fp64 denorms, since it's free */
9511    if (program->next_fp_mode.must_flush_denorms16_64)
9512       program->next_fp_mode.denorm16_64 = 0;
9513    else
9514       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9515
9516    /* preserving fp32 denorms is expensive, so only do it if asked */
9517    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
9518       program->next_fp_mode.denorm32 = fp_denorm_keep;
9519    else
9520       program->next_fp_mode.denorm32 = 0;
9521
9522    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
9523       program->next_fp_mode.round32 = fp_round_tz;
9524    else
9525       program->next_fp_mode.round32 = fp_round_ne;
9526
9527    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
9528       program->next_fp_mode.round16_64 = fp_round_tz;
9529    else
9530       program->next_fp_mode.round16_64 = fp_round_ne;
9531
9532    ctx->block->fp_mode = program->next_fp_mode;
9533 }
9534
9535 void cleanup_cfg(Program *program)
9536 {
9537    /* create linear_succs/logical_succs */
9538    for (Block& BB : program->blocks) {
9539       for (unsigned idx : BB.linear_preds)
9540          program->blocks[idx].linear_succs.emplace_back(BB.index);
9541       for (unsigned idx : BB.logical_preds)
9542          program->blocks[idx].logical_succs.emplace_back(BB.index);
9543    }
9544 }
9545
9546 void select_program(Program *program,
9547                     unsigned shader_count,
9548                     struct nir_shader *const *shaders,
9549                     ac_shader_config* config,
9550                     struct radv_shader_args *args)
9551 {
9552    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
9553
9554    for (unsigned i = 0; i < shader_count; i++) {
9555       nir_shader *nir = shaders[i];
9556       init_context(&ctx, nir);
9557
9558       setup_fp_mode(&ctx, nir);
9559
9560       if (!i) {
9561          /* needs to be after init_context() for FS */
9562          Pseudo_instruction *startpgm = add_startpgm(&ctx);
9563          append_logical_start(ctx.block);
9564
9565          if (unlikely(args->options->has_ls_vgpr_init_bug && ctx.stage == vertex_tess_control_hs))
9566             fix_ls_vgpr_init_bug(&ctx, startpgm);
9567
9568          split_arguments(&ctx, startpgm);
9569       }
9570
9571       if_context ic;
9572       if (shader_count >= 2) {
9573          Builder bld(ctx.program, ctx.block);
9574          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
9575          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
9576          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
9577
9578          begin_divergent_if_then(&ctx, &ic, cond);
9579       }
9580
9581       if (i) {
9582          Builder bld(ctx.program, ctx.block);
9583
9584          bld.barrier(aco_opcode::p_memory_barrier_shared);
9585          bld.sopp(aco_opcode::s_barrier);
9586
9587          if (ctx.stage == vertex_geometry_gs) {
9588             ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
9589          }
9590       } else if (ctx.stage == geometry_gs)
9591          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
9592
9593       if (ctx.stage == fragment_fs)
9594          handle_bc_optimize(&ctx);
9595
9596       nir_function_impl *func = nir_shader_get_entrypoint(nir);
9597       visit_cf_list(&ctx, &func->body);
9598
9599       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
9600          emit_streamout(&ctx, 0);
9601
9602       if (ctx.stage == vertex_vs || ctx.stage == tess_eval_vs) {
9603          create_vs_exports(&ctx);
9604       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
9605          Builder bld(ctx.program, ctx.block);
9606          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
9607          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
9608       } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
9609          write_tcs_tess_factors(&ctx);
9610       }
9611
9612       if (ctx.stage == fragment_fs)
9613          create_fs_exports(&ctx);
9614
9615       if (shader_count >= 2) {
9616          begin_divergent_if_else(&ctx, &ic);
9617          end_divergent_if(&ctx, &ic);
9618       }
9619
9620       ralloc_free(ctx.divergent_vals);
9621    }
9622
9623    program->config->float_mode = program->blocks[0].fp_mode.val;
9624
9625    append_logical_end(ctx.block);
9626    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
9627    Builder bld(ctx.program, ctx.block);
9628    if (ctx.program->wb_smem_l1_on_end)
9629       bld.smem(aco_opcode::s_dcache_wb, false);
9630    bld.sopp(aco_opcode::s_endpgm);
9631
9632    cleanup_cfg(program);
9633 }
9634
9635 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
9636                            ac_shader_config* config,
9637                            struct radv_shader_args *args)
9638 {
9639    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
9640
9641    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
9642    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
9643    program->next_fp_mode.must_flush_denorms32 = false;
9644    program->next_fp_mode.must_flush_denorms16_64 = false;
9645    program->next_fp_mode.care_about_round32 = false;
9646    program->next_fp_mode.care_about_round16_64 = false;
9647    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9648    program->next_fp_mode.denorm32 = 0;
9649    program->next_fp_mode.round32 = fp_round_ne;
9650    program->next_fp_mode.round16_64 = fp_round_ne;
9651    ctx.block->fp_mode = program->next_fp_mode;
9652
9653    add_startpgm(&ctx);
9654    append_logical_start(ctx.block);
9655
9656    Builder bld(ctx.program, ctx.block);
9657
9658    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
9659
9660    Operand stream_id(0u);
9661    if (args->shader_info->so.num_outputs)
9662       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9663                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
9664
9665    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9666
9667    std::stack<Block> endif_blocks;
9668
9669    for (unsigned stream = 0; stream < 4; stream++) {
9670       if (stream_id.isConstant() && stream != stream_id.constantValue())
9671          continue;
9672
9673       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9674       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9675          continue;
9676
9677       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9678
9679       unsigned BB_if_idx = ctx.block->index;
9680       Block BB_endif = Block();
9681       if (!stream_id.isConstant()) {
9682          /* begin IF */
9683          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9684          append_logical_end(ctx.block);
9685          ctx.block->kind |= block_kind_uniform;
9686          bld.branch(aco_opcode::p_cbranch_z, cond);
9687
9688          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9689
9690          ctx.block = ctx.program->create_and_insert_block();
9691          add_edge(BB_if_idx, ctx.block);
9692          bld.reset(ctx.block);
9693          append_logical_start(ctx.block);
9694       }
9695
9696       unsigned offset = 0;
9697       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9698          if (args->shader_info->gs.output_streams[i] != stream)
9699             continue;
9700
9701          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9702          unsigned length = util_last_bit(output_usage_mask);
9703          for (unsigned j = 0; j < length; ++j) {
9704             if (!(output_usage_mask & (1 << j)))
9705                continue;
9706
9707             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9708             Temp voffset = vtx_offset;
9709             if (const_offset >= 4096u) {
9710                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9711                const_offset %= 4096u;
9712             }
9713
9714             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9715             mubuf->definitions[0] = bld.def(v1);
9716             mubuf->operands[0] = Operand(gsvs_ring);
9717             mubuf->operands[1] = Operand(voffset);
9718             mubuf->operands[2] = Operand(0u);
9719             mubuf->offen = true;
9720             mubuf->offset = const_offset;
9721             mubuf->glc = true;
9722             mubuf->slc = true;
9723             mubuf->dlc = args->options->chip_class >= GFX10;
9724             mubuf->barrier = barrier_none;
9725             mubuf->can_reorder = true;
9726
9727             ctx.outputs.mask[i] |= 1 << j;
9728             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
9729
9730             bld.insert(std::move(mubuf));
9731
9732             offset++;
9733          }
9734       }
9735
9736       if (args->shader_info->so.num_outputs) {
9737          emit_streamout(&ctx, stream);
9738          bld.reset(ctx.block);
9739       }
9740
9741       if (stream == 0) {
9742          create_vs_exports(&ctx);
9743          ctx.block->kind |= block_kind_export_end;
9744       }
9745
9746       if (!stream_id.isConstant()) {
9747          append_logical_end(ctx.block);
9748
9749          /* branch from then block to endif block */
9750          bld.branch(aco_opcode::p_branch);
9751          add_edge(ctx.block->index, &BB_endif);
9752          ctx.block->kind |= block_kind_uniform;
9753
9754          /* emit else block */
9755          ctx.block = ctx.program->create_and_insert_block();
9756          add_edge(BB_if_idx, ctx.block);
9757          bld.reset(ctx.block);
9758          append_logical_start(ctx.block);
9759
9760          endif_blocks.push(std::move(BB_endif));
9761       }
9762    }
9763
9764    while (!endif_blocks.empty()) {
9765       Block BB_endif = std::move(endif_blocks.top());
9766       endif_blocks.pop();
9767
9768       Block *BB_else = ctx.block;
9769
9770       append_logical_end(BB_else);
9771       /* branch from else block to endif block */
9772       bld.branch(aco_opcode::p_branch);
9773       add_edge(BB_else->index, &BB_endif);
9774       BB_else->kind |= block_kind_uniform;
9775
9776       /** emit endif merge block */
9777       ctx.block = program->insert_block(std::move(BB_endif));
9778       bld.reset(ctx.block);
9779       append_logical_start(ctx.block);
9780    }
9781
9782    program->config->float_mode = program->blocks[0].fp_mode.val;
9783
9784    append_logical_end(ctx.block);
9785    ctx.block->kind |= block_kind_uniform;
9786    bld.sopp(aco_opcode::s_endpgm);
9787
9788    cleanup_cfg(program);
9789 }
9790 }