src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty_discard = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_discard_old;
  83    bool exec_potentially_empty_break_old;
  84    uint16_t exec_potentially_empty_break_depth_old;
  85
  86    unsigned BB_if_idx;
  87    unsigned invert_idx;
  88    bool then_branch_divergent;
  89    Block BB_invert;
  90    Block BB_endif;
  91 };
  92
  93 static bool visit_cf_list(struct isel_context *ctx,
  94                           struct exec_list *list);
  95
  96 static void add_logical_edge(unsigned pred_idx, Block *succ)
  97 {
  98    succ->logical_preds.emplace_back(pred_idx);
  99 }
 100
 101
 102 static void add_linear_edge(unsigned pred_idx, Block *succ)
 103 {
 104    succ->linear_preds.emplace_back(pred_idx);
 105 }
 106
 107 static void add_edge(unsigned pred_idx, Block *succ)
 108 {
 109    add_logical_edge(pred_idx, succ);
 110    add_linear_edge(pred_idx, succ);
 111 }
 112
 113 static void append_logical_start(Block *b)
 114 {
 115    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 116 }
 117
 118 static void append_logical_end(Block *b)
 119 {
 120    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 121 }
 122
 123 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 124 {
 125    assert(ctx->allocated[def->index].id());
 126    return ctx->allocated[def->index];
 127 }
 128
 129 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 130                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 131 {
 132    Builder bld(ctx->program, ctx->block);
 133    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 134    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 135
 136    if (ctx->program->wave_size == 32) {
 137       return thread_id_lo;
 138    } else {
 139       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 140       return thread_id_hi;
 141    }
 142 }
 143
 144 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 145 {
 146    Builder bld(ctx->program, ctx->block);
 147
 148    if (!dst.id())
 149       dst = bld.tmp(src.regClass());
 150
 151    assert(src.size() == dst.size());
 152
 153    if (ctx->stage != fragment_fs) {
 154       if (!dst.id())
 155          return src;
 156
 157       bld.copy(Definition(dst), src);
 158       return dst;
 159    }
 160
 161    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 162    ctx->program->needs_wqm |= program_needs_wqm;
 163    return dst;
 164 }
 165
 166 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 167 {
 168    if (index.regClass() == s1)
 169       return bld.readlane(bld.def(s1), data, index);
 170
 171    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 172
 173    /* Currently not implemented on GFX6-7 */
 174    assert(ctx->options->chip_class >= GFX8);
 175
 176    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 177       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 178    }
 179
 180    /* GFX10, wave64 mode:
 181     * The bpermute instruction is limited to half-wave operation, which means that it can't
 182     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 183     * emulate it here.
 184     */
 185    if (!ctx->has_gfx10_wave64_bpermute) {
 186       ctx->has_gfx10_wave64_bpermute = true;
 187       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 188       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 189    }
 190
 191    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 192    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 193    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 194    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 195
 196    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 197                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 198 }
 199
 200 Temp as_vgpr(isel_context *ctx, Temp val)
 201 {
 202    if (val.type() == RegType::sgpr) {
 203       Builder bld(ctx->program, ctx->block);
 204       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 205    }
 206    assert(val.type() == RegType::vgpr);
 207    return val;
 208 }
 209
 210 //assumes a != 0xffffffff
 211 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 212 {
 213    assert(b != 0);
 214    Builder bld(ctx->program, ctx->block);
 215
 216    if (util_is_power_of_two_or_zero(b)) {
 217       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 218       return;
 219    }
 220
 221    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 222
 223    assert(info.multiplier <= 0xffffffff);
 224
 225    bool pre_shift = info.pre_shift != 0;
 226    bool increment = info.increment != 0;
 227    bool multiply = true;
 228    bool post_shift = info.post_shift != 0;
 229
 230    if (!pre_shift && !increment && !multiply && !post_shift) {
 231       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 232       return;
 233    }
 234
 235    Temp pre_shift_dst = a;
 236    if (pre_shift) {
 237       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 238       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 239    }
 240
 241    Temp increment_dst = pre_shift_dst;
 242    if (increment) {
 243       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 244       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 245    }
 246
 247    Temp multiply_dst = increment_dst;
 248    if (multiply) {
 249       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 250       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 251                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 252    }
 253
 254    if (post_shift) {
 255       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 256    }
 257 }
 258
 259 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 260 {
 261    Builder bld(ctx->program, ctx->block);
 262    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 263 }
 264
 265
 266 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 267 {
 268    /* no need to extract the whole vector */
 269    if (src.regClass() == dst_rc) {
 270       assert(idx == 0);
 271       return src;
 272    }
 273    assert(src.size() > idx);
 274    Builder bld(ctx->program, ctx->block);
 275    auto it = ctx->allocated_vec.find(src.id());
 276    /* the size check needs to be early because elements other than 0 may be garbage */
 277    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 278       if (it->second[idx].regClass() == dst_rc) {
 279          return it->second[idx];
 280       } else {
 281          assert(dst_rc.size() == it->second[idx].regClass().size());
 282          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 283          return bld.copy(bld.def(dst_rc), it->second[idx]);
 284       }
 285    }
 286
 287    if (src.size() == dst_rc.size()) {
 288       assert(idx == 0);
 289       return bld.copy(bld.def(dst_rc), src);
 290    } else {
 291       Temp dst = bld.tmp(dst_rc);
 292       emit_extract_vector(ctx, src, idx, dst);
 293       return dst;
 294    }
 295 }
 296
 297 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 298 {
 299    if (num_components == 1)
 300       return;
 301    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 302       return;
 303    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 304    split->operands[0] = Operand(vec_src);
 305    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 306    for (unsigned i = 0; i < num_components; i++) {
 307       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 308       split->definitions[i] = Definition(elems[i]);
 309    }
 310    ctx->block->instructions.emplace_back(std::move(split));
 311    ctx->allocated_vec.emplace(vec_src.id(), elems);
 312 }
 313
 314 /* This vector expansion uses a mask to determine which elements in the new vector
 315  * come from the original vector. The other elements are undefined. */
 316 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 317 {
 318    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 319
 320    if (vec_src == dst)
 321       return;
 322
 323    Builder bld(ctx->program, ctx->block);
 324    if (num_components == 1) {
 325       if (dst.type() == RegType::sgpr)
 326          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 327       else
 328          bld.copy(Definition(dst), vec_src);
 329       return;
 330    }
 331
 332    unsigned component_size = dst.size() / num_components;
 333    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 334
 335    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 336    vec->definitions[0] = Definition(dst);
 337    unsigned k = 0;
 338    for (unsigned i = 0; i < num_components; i++) {
 339       if (mask & (1 << i)) {
 340          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 341          if (dst.type() == RegType::sgpr)
 342             src = bld.as_uniform(src);
 343          vec->operands[i] = Operand(src);
 344       } else {
 345          vec->operands[i] = Operand(0u);
 346       }
 347       elems[i] = vec->operands[i].getTemp();
 348    }
 349    ctx->block->instructions.emplace_back(std::move(vec));
 350    ctx->allocated_vec.emplace(dst.id(), elems);
 351 }
 352
 353 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 354 {
 355    Builder bld(ctx->program, ctx->block);
 356    if (!dst.id())
 357       dst = bld.tmp(bld.lm);
 358
 359    assert(val.regClass() == s1);
 360    assert(dst.regClass() == bld.lm);
 361
 362    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 363 }
 364
 365 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 366 {
 367    Builder bld(ctx->program, ctx->block);
 368    if (!dst.id())
 369       dst = bld.tmp(s1);
 370
 371    assert(val.regClass() == bld.lm);
 372    assert(dst.regClass() == s1);
 373
 374    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 375    Temp tmp = bld.tmp(s1);
 376    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 377    return emit_wqm(ctx, tmp, dst);
 378 }
 379
 380 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 381 {
 382    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 383       return get_ssa_temp(ctx, src.src.ssa);
 384
 385    if (src.src.ssa->num_components == size) {
 386       bool identity_swizzle = true;
 387       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 388          if (src.swizzle[i] != i)
 389             identity_swizzle = false;
 390       }
 391       if (identity_swizzle)
 392          return get_ssa_temp(ctx, src.src.ssa);
 393    }
 394
 395    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 396    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 397    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 398    assert(vec.size() % elem_size == 0);
 399
 400    RegClass elem_rc = RegClass(vec.type(), elem_size);
 401    if (size == 1) {
 402       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 403    } else {
 404       assert(size <= 4);
 405       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 406       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 407       for (unsigned i = 0; i < size; ++i) {
 408          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 409          vec_instr->operands[i] = Operand{elems[i]};
 410       }
 411       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 412       vec_instr->definitions[0] = Definition(dst);
 413       ctx->block->instructions.emplace_back(std::move(vec_instr));
 414       ctx->allocated_vec.emplace(dst.id(), elems);
 415       return dst;
 416    }
 417 }
 418
 419 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 420 {
 421    if (ptr.size() == 2)
 422       return ptr;
 423    Builder bld(ctx->program, ctx->block);
 424    if (ptr.type() == RegType::vgpr)
 425       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 426    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 427                      ptr, Operand((unsigned)ctx->options->address32_hi));
 428 }
 429
 430 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 431 {
 432    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 433    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 434    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 435    sop2->definitions[0] = Definition(dst);
 436    if (writes_scc)
 437       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 438    ctx->block->instructions.emplace_back(std::move(sop2));
 439 }
 440
 441 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 442                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 443 {
 444    Builder bld(ctx->program, ctx->block);
 445    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 446    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 447    if (src1.type() == RegType::sgpr) {
 448       if (commutative && src0.type() == RegType::vgpr) {
 449          Temp t = src0;
 450          src0 = src1;
 451          src1 = t;
 452       } else if (src0.type() == RegType::vgpr &&
 453                  op != aco_opcode::v_madmk_f32 &&
 454                  op != aco_opcode::v_madak_f32 &&
 455                  op != aco_opcode::v_madmk_f16 &&
 456                  op != aco_opcode::v_madak_f16) {
 457          /* If the instruction is not commutative, we emit a VOP3A instruction */
 458          bld.vop2_e64(op, Definition(dst), src0, src1);
 459          return;
 460       } else {
 461          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 462       }
 463    }
 464
 465    if (flush_denorms && ctx->program->chip_class < GFX9) {
 466       assert(dst.size() == 1);
 467       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 468       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 469    } else {
 470       bld.vop2(op, Definition(dst), src0, src1);
 471    }
 472 }
 473
 474 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 475                             bool flush_denorms = false)
 476 {
 477    Temp src0 = get_alu_src(ctx, instr->src[0]);
 478    Temp src1 = get_alu_src(ctx, instr->src[1]);
 479    Temp src2 = get_alu_src(ctx, instr->src[2]);
 480
 481    /* ensure that the instruction has at most 1 sgpr operand
 482     * The optimizer will inline constants for us */
 483    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 484       src0 = as_vgpr(ctx, src0);
 485    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 486       src1 = as_vgpr(ctx, src1);
 487    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 488       src2 = as_vgpr(ctx, src2);
 489
 490    Builder bld(ctx->program, ctx->block);
 491    if (flush_denorms && ctx->program->chip_class < GFX9) {
 492       assert(dst.size() == 1);
 493       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 494       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 495    } else {
 496       bld.vop3(op, Definition(dst), src0, src1, src2);
 497    }
 498 }
 499
 500 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 501 {
 502    Builder bld(ctx->program, ctx->block);
 503    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 504 }
 505
 506 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 507 {
 508    Temp src0 = get_alu_src(ctx, instr->src[0]);
 509    Temp src1 = get_alu_src(ctx, instr->src[1]);
 510    assert(src0.size() == src1.size());
 511
 512    aco_ptr<Instruction> vopc;
 513    if (src1.type() == RegType::sgpr) {
 514       if (src0.type() == RegType::vgpr) {
 515          /* to swap the operands, we might also have to change the opcode */
 516          switch (op) {
 517             case aco_opcode::v_cmp_lt_f32:
 518                op = aco_opcode::v_cmp_gt_f32;
 519                break;
 520             case aco_opcode::v_cmp_ge_f32:
 521                op = aco_opcode::v_cmp_le_f32;
 522                break;
 523             case aco_opcode::v_cmp_lt_i32:
 524                op = aco_opcode::v_cmp_gt_i32;
 525                break;
 526             case aco_opcode::v_cmp_ge_i32:
 527                op = aco_opcode::v_cmp_le_i32;
 528                break;
 529             case aco_opcode::v_cmp_lt_u32:
 530                op = aco_opcode::v_cmp_gt_u32;
 531                break;
 532             case aco_opcode::v_cmp_ge_u32:
 533                op = aco_opcode::v_cmp_le_u32;
 534                break;
 535             case aco_opcode::v_cmp_lt_f64:
 536                op = aco_opcode::v_cmp_gt_f64;
 537                break;
 538             case aco_opcode::v_cmp_ge_f64:
 539                op = aco_opcode::v_cmp_le_f64;
 540                break;
 541             case aco_opcode::v_cmp_lt_i64:
 542                op = aco_opcode::v_cmp_gt_i64;
 543                break;
 544             case aco_opcode::v_cmp_ge_i64:
 545                op = aco_opcode::v_cmp_le_i64;
 546                break;
 547             case aco_opcode::v_cmp_lt_u64:
 548                op = aco_opcode::v_cmp_gt_u64;
 549                break;
 550             case aco_opcode::v_cmp_ge_u64:
 551                op = aco_opcode::v_cmp_le_u64;
 552                break;
 553             default: /* eq and ne are commutative */
 554                break;
 555          }
 556          Temp t = src0;
 557          src0 = src1;
 558          src1 = t;
 559       } else {
 560          src1 = as_vgpr(ctx, src1);
 561       }
 562    }
 563
 564    Builder bld(ctx->program, ctx->block);
 565    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 566 }
 567
 568 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 569 {
 570    Temp src0 = get_alu_src(ctx, instr->src[0]);
 571    Temp src1 = get_alu_src(ctx, instr->src[1]);
 572    Builder bld(ctx->program, ctx->block);
 573
 574    assert(dst.regClass() == bld.lm);
 575    assert(src0.type() == RegType::sgpr);
 576    assert(src1.type() == RegType::sgpr);
 577    assert(src0.regClass() == src1.regClass());
 578
 579    /* Emit the SALU comparison instruction */
 580    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 581    /* Turn the result into a per-lane bool */
 582    bool_to_vector_condition(ctx, cmp, dst);
 583 }
 584
 585 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 586                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 587 {
 588    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 589    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 590    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 591    bool use_valu = s_op == aco_opcode::num_opcodes ||
 592                    divergent_vals ||
 593                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 594                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 595    aco_opcode op = use_valu ? v_op : s_op;
 596    assert(op != aco_opcode::num_opcodes);
 597    assert(dst.regClass() == ctx->program->lane_mask);
 598
 599    if (use_valu)
 600       emit_vopc_instruction(ctx, instr, op, dst);
 601    else
 602       emit_sopc_instruction(ctx, instr, op, dst);
 603 }
 604
 605 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 606 {
 607    Builder bld(ctx->program, ctx->block);
 608    Temp src0 = get_alu_src(ctx, instr->src[0]);
 609    Temp src1 = get_alu_src(ctx, instr->src[1]);
 610
 611    assert(dst.regClass() == bld.lm);
 612    assert(src0.regClass() == bld.lm);
 613    assert(src1.regClass() == bld.lm);
 614
 615    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 616 }
 617
 618 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 619 {
 620    Builder bld(ctx->program, ctx->block);
 621    Temp cond = get_alu_src(ctx, instr->src[0]);
 622    Temp then = get_alu_src(ctx, instr->src[1]);
 623    Temp els = get_alu_src(ctx, instr->src[2]);
 624
 625    assert(cond.regClass() == bld.lm);
 626
 627    if (dst.type() == RegType::vgpr) {
 628       aco_ptr<Instruction> bcsel;
 629       if (dst.size() == 1) {
 630          then = as_vgpr(ctx, then);
 631          els = as_vgpr(ctx, els);
 632
 633          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 634       } else if (dst.size() == 2) {
 635          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 637          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 638          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 639
 640          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 641          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 642
 643          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 644       } else {
 645          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 646          nir_print_instr(&instr->instr, stderr);
 647          fprintf(stderr, "\n");
 648       }
 649       return;
 650    }
 651
 652    if (instr->dest.dest.ssa.bit_size == 1) {
 653       assert(dst.regClass() == bld.lm);
 654       assert(then.regClass() == bld.lm);
 655       assert(els.regClass() == bld.lm);
 656    }
 657
 658    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 659       if (dst.regClass() == s1 || dst.regClass() == s2) {
 660          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 661          assert(dst.size() == then.size());
 662          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 663          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 664       } else {
 665          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 666          nir_print_instr(&instr->instr, stderr);
 667          fprintf(stderr, "\n");
 668       }
 669       return;
 670    }
 671
 672    /* divergent boolean bcsel
 673     * this implements bcsel on bools: dst = s0 ? s1 : s2
 674     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 675    assert(instr->dest.dest.ssa.bit_size == 1);
 676
 677    if (cond.id() != then.id())
 678       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 679
 680    if (cond.id() == els.id())
 681       bld.sop1(Builder::s_mov, Definition(dst), then);
 682    else
 683       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 684                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 685 }
 686
 687 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 688                     aco_opcode op, uint32_t undo)
 689 {
 690    /* multiply by 16777216 to handle denormals */
 691    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 692                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 693    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 694    scaled = bld.vop1(op, bld.def(v1), scaled);
 695    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 696
 697    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 698
 699    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 700 }
 701
 702 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 703 {
 704    if (ctx->block->fp_mode.denorm32 == 0) {
 705       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 706       return;
 707    }
 708
 709    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 710 }
 711
 712 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 713 {
 714    if (ctx->block->fp_mode.denorm32 == 0) {
 715       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 716       return;
 717    }
 718
 719    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 720 }
 721
 722 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 723 {
 724    if (ctx->block->fp_mode.denorm32 == 0) {
 725       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 726       return;
 727    }
 728
 729    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 730 }
 731
 732 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 733 {
 734    if (ctx->block->fp_mode.denorm32 == 0) {
 735       bld.vop1(aco_opcode::v_log_f32, dst, val);
 736       return;
 737    }
 738
 739    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 740 }
 741
 742 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 743 {
 744    if (ctx->options->chip_class >= GFX7)
 745       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 746
 747    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 748    /* TODO: create more efficient code! */
 749    if (val.type() == RegType::sgpr)
 750       val = as_vgpr(ctx, val);
 751
 752    /* Split the input value. */
 753    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 754    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 755
 756    /* Extract the exponent and compute the unbiased value. */
 757    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 758
 759    /* Extract the fractional part. */
 760    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 761    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 762
 763    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 764    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 765
 766    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 767    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 768    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 769    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 770    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 771
 772    /* Get the sign bit. */
 773    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 774
 775    /* Decide the operation to apply depending on the unbiased exponent. */
 776    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 777    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 778    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 779    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 780    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 781    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 782
 783    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 784 }
 785
 786 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 787 {
 788    if (ctx->options->chip_class >= GFX7)
 789       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 790
 791    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 792    Temp src0 = as_vgpr(ctx, val);
 793
 794    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 795    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 796
 797    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 798    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 799    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 800
 801    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 803    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 804    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 805
 806    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 807    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 808
 809    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 810
 811    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 812    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 813
 814    return add->definitions[0].getTemp();
 815 }
 816
 817 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 818 {
 819    if (!instr->dest.dest.is_ssa) {
 820       fprintf(stderr, "nir alu dst not in ssa: ");
 821       nir_print_instr(&instr->instr, stderr);
 822       fprintf(stderr, "\n");
 823       abort();
 824    }
 825    Builder bld(ctx->program, ctx->block);
 826    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 827    switch(instr->op) {
 828    case nir_op_vec2:
 829    case nir_op_vec3:
 830    case nir_op_vec4: {
 831       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 832       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 833       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 834          elems[i] = get_alu_src(ctx, instr->src[i]);
 835          vec->operands[i] = Operand{elems[i]};
 836       }
 837       vec->definitions[0] = Definition(dst);
 838       ctx->block->instructions.emplace_back(std::move(vec));
 839       ctx->allocated_vec.emplace(dst.id(), elems);
 840       break;
 841    }
 842    case nir_op_mov: {
 843       Temp src = get_alu_src(ctx, instr->src[0]);
 844       aco_ptr<Instruction> mov;
 845       if (dst.type() == RegType::sgpr) {
 846          if (src.type() == RegType::vgpr)
 847             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 848          else if (src.regClass() == s1)
 849             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 850          else if (src.regClass() == s2)
 851             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 852          else
 853             unreachable("wrong src register class for nir_op_imov");
 854       } else if (dst.regClass() == v1) {
 855          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 856       } else if (dst.regClass() == v2) {
 857          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 858       } else {
 859          nir_print_instr(&instr->instr, stderr);
 860          unreachable("Should have been lowered to scalar.");
 861       }
 862       break;
 863    }
 864    case nir_op_inot: {
 865       Temp src = get_alu_src(ctx, instr->src[0]);
 866       if (instr->dest.dest.ssa.bit_size == 1) {
 867          assert(src.regClass() == bld.lm);
 868          assert(dst.regClass() == bld.lm);
 869          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 870          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 871          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 872       } else if (dst.regClass() == v1) {
 873          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 874       } else if (dst.type() == RegType::sgpr) {
 875          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 876          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 877       } else {
 878          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 879          nir_print_instr(&instr->instr, stderr);
 880          fprintf(stderr, "\n");
 881       }
 882       break;
 883    }
 884    case nir_op_ineg: {
 885       Temp src = get_alu_src(ctx, instr->src[0]);
 886       if (dst.regClass() == v1) {
 887          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 888       } else if (dst.regClass() == s1) {
 889          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 890       } else if (dst.size() == 2) {
 891          Temp src0 = bld.tmp(dst.type(), 1);
 892          Temp src1 = bld.tmp(dst.type(), 1);
 893          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 894
 895          if (dst.regClass() == s2) {
 896             Temp carry = bld.tmp(s1);
 897             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 898             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 899             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 900          } else {
 901             Temp lower = bld.tmp(v1);
 902             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 903             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 904             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 905          }
 906       } else {
 907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 908          nir_print_instr(&instr->instr, stderr);
 909          fprintf(stderr, "\n");
 910       }
 911       break;
 912    }
 913    case nir_op_iabs: {
 914       if (dst.regClass() == s1) {
 915          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 916       } else if (dst.regClass() == v1) {
 917          Temp src = get_alu_src(ctx, instr->src[0]);
 918          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 919       } else {
 920          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 921          nir_print_instr(&instr->instr, stderr);
 922          fprintf(stderr, "\n");
 923       }
 924       break;
 925    }
 926    case nir_op_isign: {
 927       Temp src = get_alu_src(ctx, instr->src[0]);
 928       if (dst.regClass() == s1) {
 929          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 930          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 931          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 932       } else if (dst.regClass() == s2) {
 933          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 934          Temp neqz;
 935          if (ctx->program->chip_class >= GFX8)
 936             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 937          else
 938             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 939          /* SCC gets zero-extended to 64 bit */
 940          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 941       } else if (dst.regClass() == v1) {
 942          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 943          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 944          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 945       } else if (dst.regClass() == v2) {
 946          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 947          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 948          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 949          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 950          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 951          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_imax: {
 960       if (dst.regClass() == v1) {
 961          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 962       } else if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 964       } else {
 965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 966          nir_print_instr(&instr->instr, stderr);
 967          fprintf(stderr, "\n");
 968       }
 969       break;
 970    }
 971    case nir_op_umax: {
 972       if (dst.regClass() == v1) {
 973          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 974       } else if (dst.regClass() == s1) {
 975          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 976       } else {
 977          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 978          nir_print_instr(&instr->instr, stderr);
 979          fprintf(stderr, "\n");
 980       }
 981       break;
 982    }
 983    case nir_op_imin: {
 984       if (dst.regClass() == v1) {
 985          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 986       } else if (dst.regClass() == s1) {
 987          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 988       } else {
 989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 990          nir_print_instr(&instr->instr, stderr);
 991          fprintf(stderr, "\n");
 992       }
 993       break;
 994    }
 995    case nir_op_umin: {
 996       if (dst.regClass() == v1) {
 997          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 998       } else if (dst.regClass() == s1) {
 999          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
1000       } else {
1001          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1002          nir_print_instr(&instr->instr, stderr);
1003          fprintf(stderr, "\n");
1004       }
1005       break;
1006    }
1007    case nir_op_ior: {
1008       if (instr->dest.dest.ssa.bit_size == 1) {
1009          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1010       } else if (dst.regClass() == v1) {
1011          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1012       } else if (dst.regClass() == s1) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1014       } else if (dst.regClass() == s2) {
1015          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1016       } else {
1017          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1018          nir_print_instr(&instr->instr, stderr);
1019          fprintf(stderr, "\n");
1020       }
1021       break;
1022    }
1023    case nir_op_iand: {
1024       if (instr->dest.dest.ssa.bit_size == 1) {
1025          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1026       } else if (dst.regClass() == v1) {
1027          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1028       } else if (dst.regClass() == s1) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1030       } else if (dst.regClass() == s2) {
1031          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1032       } else {
1033          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1034          nir_print_instr(&instr->instr, stderr);
1035          fprintf(stderr, "\n");
1036       }
1037       break;
1038    }
1039    case nir_op_ixor: {
1040       if (instr->dest.dest.ssa.bit_size == 1) {
1041          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1042       } else if (dst.regClass() == v1) {
1043          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1044       } else if (dst.regClass() == s1) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1046       } else if (dst.regClass() == s2) {
1047          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1048       } else {
1049          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1050          nir_print_instr(&instr->instr, stderr);
1051          fprintf(stderr, "\n");
1052       }
1053       break;
1054    }
1055    case nir_op_ushr: {
1056       if (dst.regClass() == v1) {
1057          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1058       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1059          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1060                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1061       } else if (dst.regClass() == v2) {
1062          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1063                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1064       } else if (dst.regClass() == s2) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1066       } else if (dst.regClass() == s1) {
1067          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1068       } else {
1069          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1070          nir_print_instr(&instr->instr, stderr);
1071          fprintf(stderr, "\n");
1072       }
1073       break;
1074    }
1075    case nir_op_ishl: {
1076       if (dst.regClass() == v1) {
1077          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1078       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1079          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1080                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1081       } else if (dst.regClass() == v2) {
1082          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1083                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1084       } else if (dst.regClass() == s1) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1086       } else if (dst.regClass() == s2) {
1087          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1088       } else {
1089          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1090          nir_print_instr(&instr->instr, stderr);
1091          fprintf(stderr, "\n");
1092       }
1093       break;
1094    }
1095    case nir_op_ishr: {
1096       if (dst.regClass() == v1) {
1097          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1098       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1099          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1100                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1101       } else if (dst.regClass() == v2) {
1102          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1103                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1104       } else if (dst.regClass() == s1) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1106       } else if (dst.regClass() == s2) {
1107          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1108       } else {
1109          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1110          nir_print_instr(&instr->instr, stderr);
1111          fprintf(stderr, "\n");
1112       }
1113       break;
1114    }
1115    case nir_op_find_lsb: {
1116       Temp src = get_alu_src(ctx, instr->src[0]);
1117       if (src.regClass() == s1) {
1118          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1119       } else if (src.regClass() == v1) {
1120          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1121       } else if (src.regClass() == s2) {
1122          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_ufind_msb:
1131    case nir_op_ifind_msb: {
1132       Temp src = get_alu_src(ctx, instr->src[0]);
1133       if (src.regClass() == s1 || src.regClass() == s2) {
1134          aco_opcode op = src.regClass() == s2 ?
1135                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1136                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1137          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1138
1139          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1140                                         Operand(src.size() * 32u - 1u), msb_rev);
1141          Temp msb = sub.def(0).getTemp();
1142          Temp carry = sub.def(1).getTemp();
1143
1144          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1145       } else if (src.regClass() == v1) {
1146          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1147          Temp msb_rev = bld.tmp(v1);
1148          emit_vop1_instruction(ctx, instr, op, msb_rev);
1149          Temp msb = bld.tmp(v1);
1150          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1151          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1152       } else {
1153          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1154          nir_print_instr(&instr->instr, stderr);
1155          fprintf(stderr, "\n");
1156       }
1157       break;
1158    }
1159    case nir_op_bitfield_reverse: {
1160       if (dst.regClass() == s1) {
1161          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else if (dst.regClass() == v1) {
1163          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1164       } else {
1165          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1166          nir_print_instr(&instr->instr, stderr);
1167          fprintf(stderr, "\n");
1168       }
1169       break;
1170    }
1171    case nir_op_iadd: {
1172       if (dst.regClass() == s1) {
1173          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1174          break;
1175       }
1176
1177       Temp src0 = get_alu_src(ctx, instr->src[0]);
1178       Temp src1 = get_alu_src(ctx, instr->src[1]);
1179       if (dst.regClass() == v1) {
1180          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1181          break;
1182       }
1183
1184       assert(src0.size() == 2 && src1.size() == 2);
1185       Temp src00 = bld.tmp(src0.type(), 1);
1186       Temp src01 = bld.tmp(dst.type(), 1);
1187       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1188       Temp src10 = bld.tmp(src1.type(), 1);
1189       Temp src11 = bld.tmp(dst.type(), 1);
1190       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1191
1192       if (dst.regClass() == s2) {
1193          Temp carry = bld.tmp(s1);
1194          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1195          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1196          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1197       } else if (dst.regClass() == v2) {
1198          Temp dst0 = bld.tmp(v1);
1199          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1200          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1201          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1202       } else {
1203          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1204          nir_print_instr(&instr->instr, stderr);
1205          fprintf(stderr, "\n");
1206       }
1207       break;
1208    }
1209    case nir_op_uadd_sat: {
1210       Temp src0 = get_alu_src(ctx, instr->src[0]);
1211       Temp src1 = get_alu_src(ctx, instr->src[1]);
1212       if (dst.regClass() == s1) {
1213          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1214          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1215                   src0, src1);
1216          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1217       } else if (dst.regClass() == v1) {
1218          if (ctx->options->chip_class >= GFX9) {
1219             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1220             add->operands[0] = Operand(src0);
1221             add->operands[1] = Operand(src1);
1222             add->definitions[0] = Definition(dst);
1223             add->clamp = 1;
1224             ctx->block->instructions.emplace_back(std::move(add));
1225          } else {
1226             if (src1.regClass() != v1)
1227                std::swap(src0, src1);
1228             assert(src1.regClass() == v1);
1229             Temp tmp = bld.tmp(v1);
1230             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1231             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1232          }
1233       } else {
1234          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1235          nir_print_instr(&instr->instr, stderr);
1236          fprintf(stderr, "\n");
1237       }
1238       break;
1239    }
1240    case nir_op_uadd_carry: {
1241       Temp src0 = get_alu_src(ctx, instr->src[0]);
1242       Temp src1 = get_alu_src(ctx, instr->src[1]);
1243       if (dst.regClass() == s1) {
1244          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1245          break;
1246       }
1247       if (dst.regClass() == v1) {
1248          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1249          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1250          break;
1251       }
1252
1253       Temp src00 = bld.tmp(src0.type(), 1);
1254       Temp src01 = bld.tmp(dst.type(), 1);
1255       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1256       Temp src10 = bld.tmp(src1.type(), 1);
1257       Temp src11 = bld.tmp(dst.type(), 1);
1258       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1259       if (dst.regClass() == s2) {
1260          Temp carry = bld.tmp(s1);
1261          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1262          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1263          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1264       } else if (dst.regClass() == v2) {
1265          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1266          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1267          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1268          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1269       } else {
1270          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1271          nir_print_instr(&instr->instr, stderr);
1272          fprintf(stderr, "\n");
1273       }
1274       break;
1275    }
1276    case nir_op_isub: {
1277       if (dst.regClass() == s1) {
1278          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1279          break;
1280       }
1281
1282       Temp src0 = get_alu_src(ctx, instr->src[0]);
1283       Temp src1 = get_alu_src(ctx, instr->src[1]);
1284       if (dst.regClass() == v1) {
1285          bld.vsub32(Definition(dst), src0, src1);
1286          break;
1287       }
1288
1289       Temp src00 = bld.tmp(src0.type(), 1);
1290       Temp src01 = bld.tmp(dst.type(), 1);
1291       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1292       Temp src10 = bld.tmp(src1.type(), 1);
1293       Temp src11 = bld.tmp(dst.type(), 1);
1294       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1295       if (dst.regClass() == s2) {
1296          Temp carry = bld.tmp(s1);
1297          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1298          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1299          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1300       } else if (dst.regClass() == v2) {
1301          Temp lower = bld.tmp(v1);
1302          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1303          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1304          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1305       } else {
1306          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1307          nir_print_instr(&instr->instr, stderr);
1308          fprintf(stderr, "\n");
1309       }
1310       break;
1311    }
1312    case nir_op_usub_borrow: {
1313       Temp src0 = get_alu_src(ctx, instr->src[0]);
1314       Temp src1 = get_alu_src(ctx, instr->src[1]);
1315       if (dst.regClass() == s1) {
1316          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1317          break;
1318       } else if (dst.regClass() == v1) {
1319          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1320          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1321          break;
1322       }
1323
1324       Temp src00 = bld.tmp(src0.type(), 1);
1325       Temp src01 = bld.tmp(dst.type(), 1);
1326       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1327       Temp src10 = bld.tmp(src1.type(), 1);
1328       Temp src11 = bld.tmp(dst.type(), 1);
1329       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1330       if (dst.regClass() == s2) {
1331          Temp borrow = bld.tmp(s1);
1332          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1333          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1334          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1335       } else if (dst.regClass() == v2) {
1336          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1337          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1338          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1339          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1340       } else {
1341          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1342          nir_print_instr(&instr->instr, stderr);
1343          fprintf(stderr, "\n");
1344       }
1345       break;
1346    }
1347    case nir_op_imul: {
1348       if (dst.regClass() == v1) {
1349          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1350                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1351       } else if (dst.regClass() == s1) {
1352          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1353       } else {
1354          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1355          nir_print_instr(&instr->instr, stderr);
1356          fprintf(stderr, "\n");
1357       }
1358       break;
1359    }
1360    case nir_op_umul_high: {
1361       if (dst.regClass() == v1) {
1362          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1364          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1365       } else if (dst.regClass() == s1) {
1366          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1367                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1368          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1369       } else {
1370          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1371          nir_print_instr(&instr->instr, stderr);
1372          fprintf(stderr, "\n");
1373       }
1374       break;
1375    }
1376    case nir_op_imul_high: {
1377       if (dst.regClass() == v1) {
1378          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1380          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1381       } else if (dst.regClass() == s1) {
1382          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1383                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1384          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1385       } else {
1386          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1387          nir_print_instr(&instr->instr, stderr);
1388          fprintf(stderr, "\n");
1389       }
1390       break;
1391    }
1392    case nir_op_fmul: {
1393       if (dst.size() == 1) {
1394          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1395       } else if (dst.size() == 2) {
1396          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1397                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1398       } else {
1399          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1400          nir_print_instr(&instr->instr, stderr);
1401          fprintf(stderr, "\n");
1402       }
1403       break;
1404    }
1405    case nir_op_fadd: {
1406       if (dst.size() == 1) {
1407          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1408       } else if (dst.size() == 2) {
1409          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1410                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1411       } else {
1412          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1413          nir_print_instr(&instr->instr, stderr);
1414          fprintf(stderr, "\n");
1415       }
1416       break;
1417    }
1418    case nir_op_fsub: {
1419       Temp src0 = get_alu_src(ctx, instr->src[0]);
1420       Temp src1 = get_alu_src(ctx, instr->src[1]);
1421       if (dst.size() == 1) {
1422          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1424          else
1425             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1426       } else if (dst.size() == 2) {
1427          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1428                                      get_alu_src(ctx, instr->src[0]),
1429                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1430          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1431          sub->neg[1] = true;
1432       } else {
1433          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1434          nir_print_instr(&instr->instr, stderr);
1435          fprintf(stderr, "\n");
1436       }
1437       break;
1438    }
1439    case nir_op_fmax: {
1440       if (dst.size() == 1) {
1441          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1442       } else if (dst.size() == 2) {
1443          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1444             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1445                                 get_alu_src(ctx, instr->src[0]),
1446                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1447             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1448          } else {
1449             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1450                      get_alu_src(ctx, instr->src[0]),
1451                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1452          }
1453       } else {
1454          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1455          nir_print_instr(&instr->instr, stderr);
1456          fprintf(stderr, "\n");
1457       }
1458       break;
1459    }
1460    case nir_op_fmin: {
1461       if (dst.size() == 1) {
1462          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1463       } else if (dst.size() == 2) {
1464          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1465             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1466                                 get_alu_src(ctx, instr->src[0]),
1467                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1468             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1469          } else {
1470             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1471                      get_alu_src(ctx, instr->src[0]),
1472                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1473          }
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_fmax3: {
1482       if (dst.size() == 1) {
1483          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fmin3: {
1492       if (dst.size() == 1) {
1493          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1494       } else {
1495          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1496          nir_print_instr(&instr->instr, stderr);
1497          fprintf(stderr, "\n");
1498       }
1499       break;
1500    }
1501    case nir_op_fmed3: {
1502       if (dst.size() == 1) {
1503          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1504       } else {
1505          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1506          nir_print_instr(&instr->instr, stderr);
1507          fprintf(stderr, "\n");
1508       }
1509       break;
1510    }
1511    case nir_op_umax3: {
1512       if (dst.size() == 1) {
1513          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1514       } else {
1515          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1516          nir_print_instr(&instr->instr, stderr);
1517          fprintf(stderr, "\n");
1518       }
1519       break;
1520    }
1521    case nir_op_umin3: {
1522       if (dst.size() == 1) {
1523          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1524       } else {
1525          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1526          nir_print_instr(&instr->instr, stderr);
1527          fprintf(stderr, "\n");
1528       }
1529       break;
1530    }
1531    case nir_op_umed3: {
1532       if (dst.size() == 1) {
1533          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1534       } else {
1535          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1536          nir_print_instr(&instr->instr, stderr);
1537          fprintf(stderr, "\n");
1538       }
1539       break;
1540    }
1541    case nir_op_imax3: {
1542       if (dst.size() == 1) {
1543          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_imin3: {
1552       if (dst.size() == 1) {
1553          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1554       } else {
1555          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1556          nir_print_instr(&instr->instr, stderr);
1557          fprintf(stderr, "\n");
1558       }
1559       break;
1560    }
1561    case nir_op_imed3: {
1562       if (dst.size() == 1) {
1563          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1564       } else {
1565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1566          nir_print_instr(&instr->instr, stderr);
1567          fprintf(stderr, "\n");
1568       }
1569       break;
1570    }
1571    case nir_op_cube_face_coord: {
1572       Temp in = get_alu_src(ctx, instr->src[0], 3);
1573       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1574                       emit_extract_vector(ctx, in, 1, v1),
1575                       emit_extract_vector(ctx, in, 2, v1) };
1576       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1577       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1578       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1579       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1580       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1581       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1582       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1583       break;
1584    }
1585    case nir_op_cube_face_index: {
1586       Temp in = get_alu_src(ctx, instr->src[0], 3);
1587       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1588                       emit_extract_vector(ctx, in, 1, v1),
1589                       emit_extract_vector(ctx, in, 2, v1) };
1590       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1591       break;
1592    }
1593    case nir_op_bcsel: {
1594       emit_bcsel(ctx, instr, dst);
1595       break;
1596    }
1597    case nir_op_frsq: {
1598       if (dst.size() == 1) {
1599          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1600       } else if (dst.size() == 2) {
1601          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1602       } else {
1603          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1604          nir_print_instr(&instr->instr, stderr);
1605          fprintf(stderr, "\n");
1606       }
1607       break;
1608    }
1609    case nir_op_fneg: {
1610       Temp src = get_alu_src(ctx, instr->src[0]);
1611       if (dst.size() == 1) {
1612          if (ctx->block->fp_mode.must_flush_denorms32)
1613             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1614          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1615       } else if (dst.size() == 2) {
1616          if (ctx->block->fp_mode.must_flush_denorms16_64)
1617             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1618          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1619          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1620          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1621          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1622       } else {
1623          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1624          nir_print_instr(&instr->instr, stderr);
1625          fprintf(stderr, "\n");
1626       }
1627       break;
1628    }
1629    case nir_op_fabs: {
1630       Temp src = get_alu_src(ctx, instr->src[0]);
1631       if (dst.size() == 1) {
1632          if (ctx->block->fp_mode.must_flush_denorms32)
1633             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1634          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1635       } else if (dst.size() == 2) {
1636          if (ctx->block->fp_mode.must_flush_denorms16_64)
1637             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1638          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1639          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1640          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1642       } else {
1643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1644          nir_print_instr(&instr->instr, stderr);
1645          fprintf(stderr, "\n");
1646       }
1647       break;
1648    }
1649    case nir_op_fsat: {
1650       Temp src = get_alu_src(ctx, instr->src[0]);
1651       if (dst.size() == 1) {
1652          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1653          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1654          // TODO: confirm that this holds under any circumstances
1655       } else if (dst.size() == 2) {
1656          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1657          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1658          vop3->clamp = true;
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_flog2: {
1667       if (dst.size() == 1) {
1668          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_frcp: {
1677       if (dst.size() == 1) {
1678          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1679       } else if (dst.size() == 2) {
1680          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1681       } else {
1682          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1683          nir_print_instr(&instr->instr, stderr);
1684          fprintf(stderr, "\n");
1685       }
1686       break;
1687    }
1688    case nir_op_fexp2: {
1689       if (dst.size() == 1) {
1690          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1691       } else {
1692          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1693          nir_print_instr(&instr->instr, stderr);
1694          fprintf(stderr, "\n");
1695       }
1696       break;
1697    }
1698    case nir_op_fsqrt: {
1699       if (dst.size() == 1) {
1700          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1701       } else if (dst.size() == 2) {
1702          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1703       } else {
1704          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1705          nir_print_instr(&instr->instr, stderr);
1706          fprintf(stderr, "\n");
1707       }
1708       break;
1709    }
1710    case nir_op_ffract: {
1711       if (dst.size() == 1) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1713       } else if (dst.size() == 2) {
1714          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1715       } else {
1716          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1717          nir_print_instr(&instr->instr, stderr);
1718          fprintf(stderr, "\n");
1719       }
1720       break;
1721    }
1722    case nir_op_ffloor: {
1723       if (dst.size() == 1) {
1724          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1725       } else if (dst.size() == 2) {
1726          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1727       } else {
1728          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1729          nir_print_instr(&instr->instr, stderr);
1730          fprintf(stderr, "\n");
1731       }
1732       break;
1733    }
1734    case nir_op_fceil: {
1735       if (dst.size() == 1) {
1736          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1737       } else if (dst.size() == 2) {
1738          if (ctx->options->chip_class >= GFX7) {
1739             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1740          } else {
1741             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1742             Temp src0 = get_alu_src(ctx, instr->src[0]);
1743
1744             /* trunc = trunc(src0)
1745              * if (src0 > 0.0 && src0 != trunc)
1746              *    trunc += 1.0
1747              */
1748             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1749             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1750             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1751             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1752             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1753             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1754             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1755          }
1756       } else {
1757          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1758          nir_print_instr(&instr->instr, stderr);
1759          fprintf(stderr, "\n");
1760       }
1761       break;
1762    }
1763    case nir_op_ftrunc: {
1764       if (dst.size() == 1) {
1765          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1766       } else if (dst.size() == 2) {
1767          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1768       } else {
1769          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1770          nir_print_instr(&instr->instr, stderr);
1771          fprintf(stderr, "\n");
1772       }
1773       break;
1774    }
1775    case nir_op_fround_even: {
1776       if (dst.size() == 1) {
1777          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1778       } else if (dst.size() == 2) {
1779          if (ctx->options->chip_class >= GFX7) {
1780             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1781          } else {
1782             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1783             Temp src0 = get_alu_src(ctx, instr->src[0]);
1784
1785             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1786             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1787
1788             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1789             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1790             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1791             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1792             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1793             tmp = sub->definitions[0].getTemp();
1794
1795             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1796             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1797             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1798             Temp cond = vop3->definitions[0].getTemp();
1799
1800             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1801             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1802             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1803             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1804
1805             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1806          }
1807       } else {
1808          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1809          nir_print_instr(&instr->instr, stderr);
1810          fprintf(stderr, "\n");
1811       }
1812       break;
1813    }
1814    case nir_op_fsin:
1815    case nir_op_fcos: {
1816       Temp src = get_alu_src(ctx, instr->src[0]);
1817       aco_ptr<Instruction> norm;
1818       if (dst.size() == 1) {
1819          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1820          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1821
1822          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1823          if (ctx->options->chip_class < GFX9)
1824             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1825
1826          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1827          bld.vop1(opcode, Definition(dst), tmp);
1828       } else {
1829          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1830          nir_print_instr(&instr->instr, stderr);
1831          fprintf(stderr, "\n");
1832       }
1833       break;
1834    }
1835    case nir_op_ldexp: {
1836       if (dst.size() == 1) {
1837          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1838                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1839                   get_alu_src(ctx, instr->src[1]));
1840       } else if (dst.size() == 2) {
1841          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1842                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1843                   get_alu_src(ctx, instr->src[1]));
1844       } else {
1845          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1846          nir_print_instr(&instr->instr, stderr);
1847          fprintf(stderr, "\n");
1848       }
1849       break;
1850    }
1851    case nir_op_frexp_sig: {
1852       if (dst.size() == 1) {
1853          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1854                   get_alu_src(ctx, instr->src[0]));
1855       } else if (dst.size() == 2) {
1856          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1857                   get_alu_src(ctx, instr->src[0]));
1858       } else {
1859          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1860          nir_print_instr(&instr->instr, stderr);
1861          fprintf(stderr, "\n");
1862       }
1863       break;
1864    }
1865    case nir_op_frexp_exp: {
1866       if (instr->src[0].src.ssa->bit_size == 32) {
1867          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1868                   get_alu_src(ctx, instr->src[0]));
1869       } else if (instr->src[0].src.ssa->bit_size == 64) {
1870          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1871                   get_alu_src(ctx, instr->src[0]));
1872       } else {
1873          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1874          nir_print_instr(&instr->instr, stderr);
1875          fprintf(stderr, "\n");
1876       }
1877       break;
1878    }
1879    case nir_op_fsign: {
1880       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1881       if (dst.size() == 1) {
1882          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1884          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1885          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1886       } else if (dst.size() == 2) {
1887          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1888          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1889          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1890
1891          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1892          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1893          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1894
1895          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1896       } else {
1897          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1898          nir_print_instr(&instr->instr, stderr);
1899          fprintf(stderr, "\n");
1900       }
1901       break;
1902    }
1903    case nir_op_f2f32: {
1904       if (instr->src[0].src.ssa->bit_size == 64) {
1905          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1906       } else {
1907          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1908          nir_print_instr(&instr->instr, stderr);
1909          fprintf(stderr, "\n");
1910       }
1911       break;
1912    }
1913    case nir_op_f2f64: {
1914       if (instr->src[0].src.ssa->bit_size == 32) {
1915          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1916       } else {
1917          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1918          nir_print_instr(&instr->instr, stderr);
1919          fprintf(stderr, "\n");
1920       }
1921       break;
1922    }
1923    case nir_op_i2f32: {
1924       assert(dst.size() == 1);
1925       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1926       break;
1927    }
1928    case nir_op_i2f64: {
1929       if (instr->src[0].src.ssa->bit_size == 32) {
1930          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1931       } else if (instr->src[0].src.ssa->bit_size == 64) {
1932          Temp src = get_alu_src(ctx, instr->src[0]);
1933          RegClass rc = RegClass(src.type(), 1);
1934          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1935          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1936          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1937          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1938          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1939          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1940
1941       } else {
1942          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1943          nir_print_instr(&instr->instr, stderr);
1944          fprintf(stderr, "\n");
1945       }
1946       break;
1947    }
1948    case nir_op_u2f32: {
1949       assert(dst.size() == 1);
1950       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1951       break;
1952    }
1953    case nir_op_u2f64: {
1954       if (instr->src[0].src.ssa->bit_size == 32) {
1955          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1956       } else if (instr->src[0].src.ssa->bit_size == 64) {
1957          Temp src = get_alu_src(ctx, instr->src[0]);
1958          RegClass rc = RegClass(src.type(), 1);
1959          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1960          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1961          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1962          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1963          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1964          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1965       } else {
1966          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1967          nir_print_instr(&instr->instr, stderr);
1968          fprintf(stderr, "\n");
1969       }
1970       break;
1971    }
1972    case nir_op_f2i32: {
1973       Temp src = get_alu_src(ctx, instr->src[0]);
1974       if (instr->src[0].src.ssa->bit_size == 32) {
1975          if (dst.type() == RegType::vgpr)
1976             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1977          else
1978             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1979                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1980
1981       } else if (instr->src[0].src.ssa->bit_size == 64) {
1982          if (dst.type() == RegType::vgpr)
1983             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1984          else
1985             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1986                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1987
1988       } else {
1989          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1990          nir_print_instr(&instr->instr, stderr);
1991          fprintf(stderr, "\n");
1992       }
1993       break;
1994    }
1995    case nir_op_f2u32: {
1996       Temp src = get_alu_src(ctx, instr->src[0]);
1997       if (instr->src[0].src.ssa->bit_size == 32) {
1998          if (dst.type() == RegType::vgpr)
1999             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
2000          else
2001             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2002                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2003
2004       } else if (instr->src[0].src.ssa->bit_size == 64) {
2005          if (dst.type() == RegType::vgpr)
2006             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2007          else
2008             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2009                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2010
2011       } else {
2012          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2013          nir_print_instr(&instr->instr, stderr);
2014          fprintf(stderr, "\n");
2015       }
2016       break;
2017    }
2018    case nir_op_f2i64: {
2019       Temp src = get_alu_src(ctx, instr->src[0]);
2020       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2021          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2022          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2023          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2024          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2025          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2026          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2027          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2028          Temp new_exponent = bld.tmp(v1);
2029          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2030          if (ctx->program->chip_class >= GFX8)
2031             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2032          else
2033             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2034          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2035          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2036          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2037          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2038          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2039          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2040          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2041          Temp new_lower = bld.tmp(v1);
2042          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2043          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2044          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2045
2046       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2047          if (src.type() == RegType::vgpr)
2048             src = bld.as_uniform(src);
2049          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2050          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2051          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2052          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2053          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2054          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2055          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2056          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2057          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2058          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2059          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2060          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2061          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2062          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2063          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2064          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2065          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2066          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2067          Temp borrow = bld.tmp(s1);
2068          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2069          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2070          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2071
2072       } else if (instr->src[0].src.ssa->bit_size == 64) {
2073          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2074          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2075          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2076          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2077          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2078          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2079          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2080          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2081          if (dst.type() == RegType::sgpr) {
2082             lower = bld.as_uniform(lower);
2083             upper = bld.as_uniform(upper);
2084          }
2085          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2086
2087       } else {
2088          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2089          nir_print_instr(&instr->instr, stderr);
2090          fprintf(stderr, "\n");
2091       }
2092       break;
2093    }
2094    case nir_op_f2u64: {
2095       Temp src = get_alu_src(ctx, instr->src[0]);
2096       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2097          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2098          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2099          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2100          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2101          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2102          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2103          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2104          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2105          Temp new_exponent = bld.tmp(v1);
2106          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2107          if (ctx->program->chip_class >= GFX8)
2108             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2109          else
2110             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2111          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2112          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2114          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2115          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2116          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2117          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2118
2119       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2120          if (src.type() == RegType::vgpr)
2121             src = bld.as_uniform(src);
2122          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2123          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2124          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2125          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2126          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2127          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2128          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2129          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2130          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2131          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2132          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2133          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2134          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2135          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2136          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2137          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2138          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2139          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2140
2141       } else if (instr->src[0].src.ssa->bit_size == 64) {
2142          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2143          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2144          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2145          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2146          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2147          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2148          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2149          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2150          if (dst.type() == RegType::sgpr) {
2151             lower = bld.as_uniform(lower);
2152             upper = bld.as_uniform(upper);
2153          }
2154          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2155
2156       } else {
2157          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2158          nir_print_instr(&instr->instr, stderr);
2159          fprintf(stderr, "\n");
2160       }
2161       break;
2162    }
2163    case nir_op_b2f32: {
2164       Temp src = get_alu_src(ctx, instr->src[0]);
2165       assert(src.regClass() == bld.lm);
2166
2167       if (dst.regClass() == s1) {
2168          src = bool_to_scalar_condition(ctx, src);
2169          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2170       } else if (dst.regClass() == v1) {
2171          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2172       } else {
2173          unreachable("Wrong destination register class for nir_op_b2f32.");
2174       }
2175       break;
2176    }
2177    case nir_op_b2f64: {
2178       Temp src = get_alu_src(ctx, instr->src[0]);
2179       assert(src.regClass() == bld.lm);
2180
2181       if (dst.regClass() == s2) {
2182          src = bool_to_scalar_condition(ctx, src);
2183          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2184       } else if (dst.regClass() == v2) {
2185          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2186          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2187          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2188       } else {
2189          unreachable("Wrong destination register class for nir_op_b2f64.");
2190       }
2191       break;
2192    }
2193    case nir_op_i2i32: {
2194       Temp src = get_alu_src(ctx, instr->src[0]);
2195       if (instr->src[0].src.ssa->bit_size == 64) {
2196          /* we can actually just say dst = src, as it would map the lower register */
2197          emit_extract_vector(ctx, src, 0, dst);
2198       } else {
2199          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2200          nir_print_instr(&instr->instr, stderr);
2201          fprintf(stderr, "\n");
2202       }
2203       break;
2204    }
2205    case nir_op_u2u32: {
2206       Temp src = get_alu_src(ctx, instr->src[0]);
2207       if (instr->src[0].src.ssa->bit_size == 16) {
2208          if (dst.regClass() == s1) {
2209             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2210          } else {
2211             // TODO: do better with SDWA
2212             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2213          }
2214       } else if (instr->src[0].src.ssa->bit_size == 64) {
2215          /* we can actually just say dst = src, as it would map the lower register */
2216          emit_extract_vector(ctx, src, 0, dst);
2217       } else {
2218          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2219          nir_print_instr(&instr->instr, stderr);
2220          fprintf(stderr, "\n");
2221       }
2222       break;
2223    }
2224    case nir_op_i2i64: {
2225       Temp src = get_alu_src(ctx, instr->src[0]);
2226       if (src.regClass() == s1) {
2227          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2228          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2229       } else if (src.regClass() == v1) {
2230          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2231          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2232       } else {
2233          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2234          nir_print_instr(&instr->instr, stderr);
2235          fprintf(stderr, "\n");
2236       }
2237       break;
2238    }
2239    case nir_op_u2u64: {
2240       Temp src = get_alu_src(ctx, instr->src[0]);
2241       if (instr->src[0].src.ssa->bit_size == 32) {
2242          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2243       } else {
2244          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2245          nir_print_instr(&instr->instr, stderr);
2246          fprintf(stderr, "\n");
2247       }
2248       break;
2249    }
2250    case nir_op_b2i32: {
2251       Temp src = get_alu_src(ctx, instr->src[0]);
2252       assert(src.regClass() == bld.lm);
2253
2254       if (dst.regClass() == s1) {
2255          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2256          bool_to_scalar_condition(ctx, src, dst);
2257       } else if (dst.regClass() == v1) {
2258          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2259       } else {
2260          unreachable("Invalid register class for b2i32");
2261       }
2262       break;
2263    }
2264    case nir_op_i2b1: {
2265       Temp src = get_alu_src(ctx, instr->src[0]);
2266       assert(dst.regClass() == bld.lm);
2267
2268       if (src.type() == RegType::vgpr) {
2269          assert(src.regClass() == v1 || src.regClass() == v2);
2270          assert(dst.regClass() == bld.lm);
2271          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2272                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2273       } else {
2274          assert(src.regClass() == s1 || src.regClass() == s2);
2275          Temp tmp;
2276          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2277             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2278          } else {
2279             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2280                            bld.scc(bld.def(s1)), Operand(0u), src);
2281          }
2282          bool_to_vector_condition(ctx, tmp, dst);
2283       }
2284       break;
2285    }
2286    case nir_op_pack_64_2x32_split: {
2287       Temp src0 = get_alu_src(ctx, instr->src[0]);
2288       Temp src1 = get_alu_src(ctx, instr->src[1]);
2289
2290       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2291       break;
2292    }
2293    case nir_op_unpack_64_2x32_split_x:
2294       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2295       break;
2296    case nir_op_unpack_64_2x32_split_y:
2297       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2298       break;
2299    case nir_op_pack_half_2x16: {
2300       Temp src = get_alu_src(ctx, instr->src[0], 2);
2301
2302       if (dst.regClass() == v1) {
2303          Temp src0 = bld.tmp(v1);
2304          Temp src1 = bld.tmp(v1);
2305          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2306          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2307             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2308          else
2309             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2310                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2311                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2312       } else {
2313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2314          nir_print_instr(&instr->instr, stderr);
2315          fprintf(stderr, "\n");
2316       }
2317       break;
2318    }
2319    case nir_op_unpack_half_2x16_split_x: {
2320       if (dst.regClass() == v1) {
2321          Builder bld(ctx->program, ctx->block);
2322          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2323       } else {
2324          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2325          nir_print_instr(&instr->instr, stderr);
2326          fprintf(stderr, "\n");
2327       }
2328       break;
2329    }
2330    case nir_op_unpack_half_2x16_split_y: {
2331       if (dst.regClass() == v1) {
2332          Builder bld(ctx->program, ctx->block);
2333          /* TODO: use SDWA here */
2334          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2335                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2336       } else {
2337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2338          nir_print_instr(&instr->instr, stderr);
2339          fprintf(stderr, "\n");
2340       }
2341       break;
2342    }
2343    case nir_op_fquantize2f16: {
2344       Temp src = get_alu_src(ctx, instr->src[0]);
2345       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2346       Temp f32, cmp_res;
2347
2348       if (ctx->program->chip_class >= GFX8) {
2349          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2350          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2351          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2352       } else {
2353          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2354           * so compare the result and flush to 0 if it's smaller.
2355           */
2356          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2357          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2358          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2359          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2360          cmp_res = vop3->definitions[0].getTemp();
2361       }
2362
2363       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2364          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2366       } else {
2367          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2368       }
2369       break;
2370    }
2371    case nir_op_bfm: {
2372       Temp bits = get_alu_src(ctx, instr->src[0]);
2373       Temp offset = get_alu_src(ctx, instr->src[1]);
2374
2375       if (dst.regClass() == s1) {
2376          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2377       } else if (dst.regClass() == v1) {
2378          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2379       } else {
2380          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2381          nir_print_instr(&instr->instr, stderr);
2382          fprintf(stderr, "\n");
2383       }
2384       break;
2385    }
2386    case nir_op_bitfield_select: {
2387       /* (mask & insert) | (~mask & base) */
2388       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2389       Temp insert = get_alu_src(ctx, instr->src[1]);
2390       Temp base = get_alu_src(ctx, instr->src[2]);
2391
2392       /* dst = (insert & bitmask) | (base & ~bitmask) */
2393       if (dst.regClass() == s1) {
2394          aco_ptr<Instruction> sop2;
2395          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2396          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2397          Operand lhs;
2398          if (const_insert && const_bitmask) {
2399             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2400          } else {
2401             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2402             lhs = Operand(insert);
2403          }
2404
2405          Operand rhs;
2406          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2407          if (const_base && const_bitmask) {
2408             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2409          } else {
2410             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2411             rhs = Operand(base);
2412          }
2413
2414          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2415
2416       } else if (dst.regClass() == v1) {
2417          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2418             base = as_vgpr(ctx, base);
2419          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2420             insert = as_vgpr(ctx, insert);
2421
2422          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2423
2424       } else {
2425          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2426          nir_print_instr(&instr->instr, stderr);
2427          fprintf(stderr, "\n");
2428       }
2429       break;
2430    }
2431    case nir_op_ubfe:
2432    case nir_op_ibfe: {
2433       Temp base = get_alu_src(ctx, instr->src[0]);
2434       Temp offset = get_alu_src(ctx, instr->src[1]);
2435       Temp bits = get_alu_src(ctx, instr->src[2]);
2436
2437       if (dst.type() == RegType::sgpr) {
2438          Operand extract;
2439          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2440          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2441          if (const_offset && const_bits) {
2442             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2443             extract = Operand(const_extract);
2444          } else {
2445             Operand width;
2446             if (const_bits) {
2447                width = Operand(const_bits->u32 << 16);
2448             } else {
2449                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2450             }
2451             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2452          }
2453
2454          aco_opcode opcode;
2455          if (dst.regClass() == s1) {
2456             if (instr->op == nir_op_ubfe)
2457                opcode = aco_opcode::s_bfe_u32;
2458             else
2459                opcode = aco_opcode::s_bfe_i32;
2460          } else if (dst.regClass() == s2) {
2461             if (instr->op == nir_op_ubfe)
2462                opcode = aco_opcode::s_bfe_u64;
2463             else
2464                opcode = aco_opcode::s_bfe_i64;
2465          } else {
2466             unreachable("Unsupported BFE bit size");
2467          }
2468
2469          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2470
2471       } else {
2472          aco_opcode opcode;
2473          if (dst.regClass() == v1) {
2474             if (instr->op == nir_op_ubfe)
2475                opcode = aco_opcode::v_bfe_u32;
2476             else
2477                opcode = aco_opcode::v_bfe_i32;
2478          } else {
2479             unreachable("Unsupported BFE bit size");
2480          }
2481
2482          emit_vop3a_instruction(ctx, instr, opcode, dst);
2483       }
2484       break;
2485    }
2486    case nir_op_bit_count: {
2487       Temp src = get_alu_src(ctx, instr->src[0]);
2488       if (src.regClass() == s1) {
2489          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2490       } else if (src.regClass() == v1) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2492       } else if (src.regClass() == v2) {
2493          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2494                   emit_extract_vector(ctx, src, 1, v1),
2495                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2496                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2497       } else if (src.regClass() == s2) {
2498          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2499       } else {
2500          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2501          nir_print_instr(&instr->instr, stderr);
2502          fprintf(stderr, "\n");
2503       }
2504       break;
2505    }
2506    case nir_op_flt: {
2507       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2508       break;
2509    }
2510    case nir_op_fge: {
2511       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2512       break;
2513    }
2514    case nir_op_feq: {
2515       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2516       break;
2517    }
2518    case nir_op_fne: {
2519       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2520       break;
2521    }
2522    case nir_op_ilt: {
2523       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2524       break;
2525    }
2526    case nir_op_ige: {
2527       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2528       break;
2529    }
2530    case nir_op_ieq: {
2531       if (instr->src[0].src.ssa->bit_size == 1)
2532          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2533       else
2534          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2535                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2536       break;
2537    }
2538    case nir_op_ine: {
2539       if (instr->src[0].src.ssa->bit_size == 1)
2540          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2541       else
2542          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2543                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2544       break;
2545    }
2546    case nir_op_ult: {
2547       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2548       break;
2549    }
2550    case nir_op_uge: {
2551       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2552       break;
2553    }
2554    case nir_op_fddx:
2555    case nir_op_fddy:
2556    case nir_op_fddx_fine:
2557    case nir_op_fddy_fine:
2558    case nir_op_fddx_coarse:
2559    case nir_op_fddy_coarse: {
2560       Temp src = get_alu_src(ctx, instr->src[0]);
2561       uint16_t dpp_ctrl1, dpp_ctrl2;
2562       if (instr->op == nir_op_fddx_fine) {
2563          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2564          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2565       } else if (instr->op == nir_op_fddy_fine) {
2566          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2567          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2568       } else {
2569          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2570          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2571             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2572          else
2573             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2574       }
2575
2576       Temp tmp;
2577       if (ctx->program->chip_class >= GFX8) {
2578          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2579          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2580       } else {
2581          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2582          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2583          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2584       }
2585       emit_wqm(ctx, tmp, dst, true);
2586       break;
2587    }
2588    default:
2589       fprintf(stderr, "Unknown NIR ALU instr: ");
2590       nir_print_instr(&instr->instr, stderr);
2591       fprintf(stderr, "\n");
2592    }
2593 }
2594
2595 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2596 {
2597    Temp dst = get_ssa_temp(ctx, &instr->def);
2598
2599    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2600    // which get truncated the lsb if double and msb if int
2601    // for now, we only use s_mov_b64 with 64bit inline constants
2602    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2603    assert(dst.type() == RegType::sgpr);
2604
2605    Builder bld(ctx->program, ctx->block);
2606
2607    if (instr->def.bit_size == 1) {
2608       assert(dst.regClass() == bld.lm);
2609       int val = instr->value[0].b ? -1 : 0;
2610       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2611       bld.sop1(Builder::s_mov, Definition(dst), op);
2612    } else if (dst.size() == 1) {
2613       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2614    } else {
2615       assert(dst.size() != 1);
2616       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2617       if (instr->def.bit_size == 64)
2618          for (unsigned i = 0; i < dst.size(); i++)
2619             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2620       else {
2621          for (unsigned i = 0; i < dst.size(); i++)
2622             vec->operands[i] = Operand{instr->value[i].u32};
2623       }
2624       vec->definitions[0] = Definition(dst);
2625       ctx->block->instructions.emplace_back(std::move(vec));
2626    }
2627 }
2628
2629 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2630 {
2631    uint32_t new_mask = 0;
2632    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2633       if (mask & (1u << i))
2634          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2635    return new_mask;
2636 }
2637
2638 Operand load_lds_size_m0(isel_context *ctx)
2639 {
2640    /* TODO: m0 does not need to be initialized on GFX9+ */
2641    Builder bld(ctx->program, ctx->block);
2642    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2643 }
2644
2645 Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2646               Temp address, unsigned base_offset, unsigned align)
2647 {
2648    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2649
2650    Builder bld(ctx->program, ctx->block);
2651
2652    Operand m = load_lds_size_m0(ctx);
2653
2654    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2655    unsigned bytes_read = 0;
2656    unsigned result_size = 0;
2657    unsigned total_bytes = num_components * elem_size_bytes;
2658    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2659    bool large_ds_read = ctx->options->chip_class >= GFX7;
2660    bool usable_read2 = ctx->options->chip_class >= GFX7;
2661
2662    while (bytes_read < total_bytes) {
2663       unsigned todo = total_bytes - bytes_read;
2664       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2665       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2666
2667       aco_opcode op = aco_opcode::last_opcode;
2668       bool read2 = false;
2669       if (todo >= 16 && aligned16 && large_ds_read) {
2670          op = aco_opcode::ds_read_b128;
2671          todo = 16;
2672       } else if (todo >= 16 && aligned8 && usable_read2) {
2673          op = aco_opcode::ds_read2_b64;
2674          read2 = true;
2675          todo = 16;
2676       } else if (todo >= 12 && aligned16 && large_ds_read) {
2677          op = aco_opcode::ds_read_b96;
2678          todo = 12;
2679       } else if (todo >= 8 && aligned8) {
2680          op = aco_opcode::ds_read_b64;
2681          todo = 8;
2682       } else if (todo >= 8 && usable_read2) {
2683          op = aco_opcode::ds_read2_b32;
2684          read2 = true;
2685          todo = 8;
2686       } else if (todo >= 4) {
2687          op = aco_opcode::ds_read_b32;
2688          todo = 4;
2689       } else {
2690          assert(false);
2691       }
2692       assert(todo % elem_size_bytes == 0);
2693       unsigned num_elements = todo / elem_size_bytes;
2694       unsigned offset = base_offset + bytes_read;
2695       unsigned max_offset = read2 ? 1019 : 65535;
2696
2697       Temp address_offset = address;
2698       if (offset > max_offset) {
2699          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2700          offset = bytes_read;
2701       }
2702       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2703
2704       Temp res;
2705       if (num_components == 1 && dst.type() == RegType::vgpr)
2706          res = dst;
2707       else
2708          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2709
2710       if (read2)
2711          res = bld.ds(op, Definition(res), address_offset, m, offset / (todo / 2), (offset / (todo / 2)) + 1);
2712       else
2713          res = bld.ds(op, Definition(res), address_offset, m, offset);
2714
2715       if (num_components == 1) {
2716          assert(todo == total_bytes);
2717          if (dst.type() == RegType::sgpr)
2718             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2719          return dst;
2720       }
2721
2722       if (dst.type() == RegType::sgpr) {
2723          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2724          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2725          res = new_res;
2726       }
2727
2728       if (num_elements == 1) {
2729          result[result_size++] = res;
2730       } else {
2731          assert(res != dst && res.size() % num_elements == 0);
2732          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2733          split->operands[0] = Operand(res);
2734          for (unsigned i = 0; i < num_elements; i++)
2735             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2736          ctx->block->instructions.emplace_back(std::move(split));
2737       }
2738
2739       bytes_read += todo;
2740    }
2741
2742    assert(result_size == num_components && result_size > 1);
2743    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2744    for (unsigned i = 0; i < result_size; i++)
2745       vec->operands[i] = Operand(result[i]);
2746    vec->definitions[0] = Definition(dst);
2747    ctx->block->instructions.emplace_back(std::move(vec));
2748    ctx->allocated_vec.emplace(dst.id(), result);
2749
2750    return dst;
2751 }
2752
2753 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2754 {
2755    if (start == 0 && size == data.size())
2756       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2757
2758    unsigned size_hint = 1;
2759    auto it = ctx->allocated_vec.find(data.id());
2760    if (it != ctx->allocated_vec.end())
2761       size_hint = it->second[0].size();
2762    if (size % size_hint || start % size_hint)
2763       size_hint = 1;
2764
2765    start /= size_hint;
2766    size /= size_hint;
2767
2768    Temp elems[size];
2769    for (unsigned i = 0; i < size; i++)
2770       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2771
2772    if (size == 1)
2773       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2774
2775    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2776    for (unsigned i = 0; i < size; i++)
2777       vec->operands[i] = Operand(elems[i]);
2778    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2779    vec->definitions[0] = Definition(res);
2780    ctx->block->instructions.emplace_back(std::move(vec));
2781    return res;
2782 }
2783
2784 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2785 {
2786    Builder bld(ctx->program, ctx->block);
2787    unsigned bytes_written = 0;
2788    bool large_ds_write = ctx->options->chip_class >= GFX7;
2789    bool usable_write2 = ctx->options->chip_class >= GFX7;
2790
2791    while (bytes_written < total_size * 4) {
2792       unsigned todo = total_size * 4 - bytes_written;
2793       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2794       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2795
2796       aco_opcode op = aco_opcode::last_opcode;
2797       bool write2 = false;
2798       unsigned size = 0;
2799       if (todo >= 16 && aligned16 && large_ds_write) {
2800          op = aco_opcode::ds_write_b128;
2801          size = 4;
2802       } else if (todo >= 16 && aligned8 && usable_write2) {
2803          op = aco_opcode::ds_write2_b64;
2804          write2 = true;
2805          size = 4;
2806       } else if (todo >= 12 && aligned16 && large_ds_write) {
2807          op = aco_opcode::ds_write_b96;
2808          size = 3;
2809       } else if (todo >= 8 && aligned8) {
2810          op = aco_opcode::ds_write_b64;
2811          size = 2;
2812       } else if (todo >= 8 && usable_write2) {
2813          op = aco_opcode::ds_write2_b32;
2814          write2 = true;
2815          size = 2;
2816       } else if (todo >= 4) {
2817          op = aco_opcode::ds_write_b32;
2818          size = 1;
2819       } else {
2820          assert(false);
2821       }
2822
2823       unsigned offset = offset0 + offset1 + bytes_written;
2824       unsigned max_offset = write2 ? 1020 : 65535;
2825       Temp address_offset = address;
2826       if (offset > max_offset) {
2827          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2828          offset = offset1 + bytes_written;
2829       }
2830       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2831
2832       if (write2) {
2833          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2834          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2835          bld.ds(op, address_offset, val0, val1, m, offset / size / 2, (offset / size / 2) + 1);
2836       } else {
2837          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2838          bld.ds(op, address_offset, val, m, offset);
2839       }
2840
2841       bytes_written += size * 4;
2842    }
2843 }
2844
2845 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2846                Temp address, unsigned base_offset, unsigned align)
2847 {
2848    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2849    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
2850
2851    Operand m = load_lds_size_m0(ctx);
2852
2853    /* we need at most two stores, assuming that the writemask is at most 4 bits wide */
2854    assert(wrmask <= 0x0f);
2855    int start[2], count[2];
2856    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2857    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2858    assert(wrmask == 0);
2859
2860    /* one combined store is sufficient */
2861    if (count[0] == count[1] && (align % elem_size_bytes) == 0 && (base_offset % elem_size_bytes) == 0) {
2862       Builder bld(ctx->program, ctx->block);
2863
2864       Temp address_offset = address;
2865       if ((base_offset / elem_size_bytes) + start[1] > 255) {
2866          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2867          base_offset = 0;
2868       }
2869
2870       assert(count[0] == 1);
2871       RegClass xtract_rc(RegType::vgpr, elem_size_bytes / 4);
2872
2873       Temp val0 = emit_extract_vector(ctx, data, start[0], xtract_rc);
2874       Temp val1 = emit_extract_vector(ctx, data, start[1], xtract_rc);
2875       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2876       base_offset = base_offset / elem_size_bytes;
2877       bld.ds(op, address_offset, val0, val1, m,
2878              base_offset + start[0], base_offset + start[1]);
2879       return;
2880    }
2881
2882    for (unsigned i = 0; i < 2; i++) {
2883       if (count[i] == 0)
2884          continue;
2885
2886       unsigned elem_size_words = elem_size_bytes / 4;
2887       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2888                       base_offset, start[i] * elem_size_bytes, align);
2889    }
2890    return;
2891 }
2892
2893 unsigned calculate_lds_alignment(isel_context *ctx, unsigned const_offset)
2894 {
2895    unsigned align = 16;
2896    if (const_offset)
2897       align = std::min(align, 1u << (ffs(const_offset) - 1));
2898
2899    return align;
2900 }
2901
2902
2903 Temp create_vec_from_array(isel_context *ctx, Temp arr[], unsigned cnt, RegType reg_type, unsigned elem_size_bytes,
2904                            unsigned split_cnt = 0u, Temp dst = Temp())
2905 {
2906    Builder bld(ctx->program, ctx->block);
2907    unsigned dword_size = elem_size_bytes / 4;
2908
2909    if (!dst.id())
2910       dst = bld.tmp(RegClass(reg_type, cnt * dword_size));
2911
2912    std::array<Temp, NIR_MAX_VEC_COMPONENTS> allocated_vec;
2913    aco_ptr<Pseudo_instruction> instr {create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, cnt, 1)};
2914    instr->definitions[0] = Definition(dst);
2915
2916    for (unsigned i = 0; i < cnt; ++i) {
2917       if (arr[i].id()) {
2918          assert(arr[i].size() == dword_size);
2919          allocated_vec[i] = arr[i];
2920          instr->operands[i] = Operand(arr[i]);
2921       } else {
2922          Temp zero = bld.copy(bld.def(RegClass(reg_type, dword_size)), Operand(0u, dword_size == 2));
2923          allocated_vec[i] = zero;
2924          instr->operands[i] = Operand(zero);
2925       }
2926    }
2927
2928    bld.insert(std::move(instr));
2929
2930    if (split_cnt)
2931       emit_split_vector(ctx, dst, split_cnt);
2932    else
2933       ctx->allocated_vec.emplace(dst.id(), allocated_vec); /* emit_split_vector already does this */
2934
2935    return dst;
2936 }
2937
2938 inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, unsigned const_offset)
2939 {
2940    if (const_offset >= 4096) {
2941       unsigned excess_const_offset = const_offset / 4096u * 4096u;
2942       const_offset %= 4096u;
2943
2944       if (!voffset.id())
2945          voffset = bld.copy(bld.def(v1), Operand(excess_const_offset));
2946       else if (unlikely(voffset.regClass() == s1))
2947          voffset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), Operand(excess_const_offset), Operand(voffset));
2948       else if (likely(voffset.regClass() == v1))
2949          voffset = bld.vadd32(bld.def(v1), Operand(voffset), Operand(excess_const_offset));
2950       else
2951          unreachable("Unsupported register class of voffset");
2952    }
2953
2954    return const_offset;
2955 }
2956
2957 void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata,
2958                              unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false)
2959 {
2960    assert(vdata.id());
2961    assert(vdata.size() != 3 || ctx->program->chip_class != GFX6);
2962    assert(vdata.size() >= 1 && vdata.size() <= 4);
2963
2964    Builder bld(ctx->program, ctx->block);
2965    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_store_dword + vdata.size() - 1);
2966    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
2967
2968    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
2969    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
2970    Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset,
2971                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
2972                                  /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc);
2973
2974    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
2975 }
2976
2977 void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset,
2978                                    unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask,
2979                                    bool allow_combining = true, bool reorder = true, bool slc = false)
2980 {
2981    Builder bld(ctx->program, ctx->block);
2982    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
2983    assert(write_mask);
2984
2985    if (elem_size_bytes == 8) {
2986       elem_size_bytes = 4;
2987       write_mask = widen_mask(write_mask, 2);
2988    }
2989
2990    while (write_mask) {
2991       int start = 0;
2992       int count = 0;
2993       u_bit_scan_consecutive_range(&write_mask, &start, &count);
2994       assert(count > 0);
2995       assert(start >= 0);
2996
2997       while (count > 0) {
2998          unsigned sub_count = allow_combining ? MIN2(count, 4) : 1;
2999          unsigned const_offset = (unsigned) start * elem_size_bytes + base_const_offset;
3000
3001          /* GFX6 doesn't have buffer_store_dwordx3, so make sure not to emit that here either. */
3002          if (unlikely(ctx->program->chip_class == GFX6 && sub_count == 3))
3003             sub_count = 2;
3004
3005          Temp elem = extract_subvector(ctx, src, start, sub_count, RegType::vgpr);
3006          emit_single_mubuf_store(ctx, descriptor, voffset, soffset, elem, const_offset, reorder, slc);
3007
3008          count -= sub_count;
3009          start += sub_count;
3010       }
3011
3012       assert(count == 0);
3013    }
3014 }
3015
3016 Temp emit_single_mubuf_load(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset,
3017                             unsigned const_offset, unsigned size_dwords, bool allow_reorder = true)
3018 {
3019    assert(size_dwords != 3 || ctx->program->chip_class != GFX6);
3020    assert(size_dwords >= 1 && size_dwords <= 4);
3021
3022    Builder bld(ctx->program, ctx->block);
3023    Temp vdata = bld.tmp(RegClass(RegType::vgpr, size_dwords));
3024    aco_opcode op = (aco_opcode) ((unsigned) aco_opcode::buffer_load_dword + size_dwords - 1);
3025    const_offset = resolve_excess_vmem_const_offset(bld, voffset, const_offset);
3026
3027    Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1);
3028    Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u);
3029    Builder::Result r = bld.mubuf(op, Definition(vdata), Operand(descriptor), voffset_op, soffset_op, const_offset,
3030                                  /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false,
3031                                  /* disable_wqm */ false, /* glc */ true,
3032                                  /* dlc*/ ctx->program->chip_class >= GFX10, /* slc */ false);
3033
3034    static_cast<MUBUF_instruction *>(r.instr)->can_reorder = allow_reorder;
3035
3036    return vdata;
3037 }
3038
3039 void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, Temp soffset,
3040                      unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components,
3041                      unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true)
3042 {
3043    assert(elem_size_bytes == 4 || elem_size_bytes == 8);
3044    assert((num_components * elem_size_bytes / 4) == dst.size());
3045    assert(!!stride != allow_combining);
3046
3047    Builder bld(ctx->program, ctx->block);
3048    unsigned split_cnt = num_components;
3049
3050    if (elem_size_bytes == 8) {
3051       elem_size_bytes = 4;
3052       num_components *= 2;
3053    }
3054
3055    if (!stride)
3056       stride = elem_size_bytes;
3057
3058    unsigned load_size = 1;
3059    if (allow_combining) {
3060       if ((num_components % 4) == 0)
3061          load_size = 4;
3062       else if ((num_components % 3) == 0 && ctx->program->chip_class != GFX6)
3063          load_size = 3;
3064       else if ((num_components % 2) == 0)
3065          load_size = 2;
3066    }
3067
3068    unsigned num_loads = num_components / load_size;
3069    std::array<Temp, NIR_MAX_VEC_COMPONENTS> elems;
3070
3071    for (unsigned i = 0; i < num_loads; ++i) {
3072       unsigned const_offset = i * stride * load_size + base_const_offset;
3073       elems[i] = emit_single_mubuf_load(ctx, descriptor, voffset, soffset, const_offset, load_size, allow_reorder);
3074    }
3075
3076    create_vec_from_array(ctx, elems.data(), num_loads, RegType::vgpr, load_size * 4u, split_cnt, dst);
3077 }
3078
3079 std::pair<Temp, unsigned> offset_add_from_nir(isel_context *ctx, const std::pair<Temp, unsigned> &base_offset, nir_src *off_src, unsigned stride = 1u)
3080 {
3081    Builder bld(ctx->program, ctx->block);
3082    Temp offset = base_offset.first;
3083    unsigned const_offset = base_offset.second;
3084
3085    if (!nir_src_is_const(*off_src)) {
3086       Temp indirect_offset_arg = get_ssa_temp(ctx, off_src->ssa);
3087       Temp with_stride;
3088
3089       /* Calculate indirect offset with stride */
3090       if (likely(indirect_offset_arg.regClass() == v1))
3091          with_stride = bld.v_mul_imm(bld.def(v1), indirect_offset_arg, stride);
3092       else if (indirect_offset_arg.regClass() == s1)
3093          with_stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), indirect_offset_arg);
3094       else
3095          unreachable("Unsupported register class of indirect offset");
3096
3097       /* Add to the supplied base offset */
3098       if (offset.id() == 0)
3099          offset = with_stride;
3100       else if (unlikely(offset.regClass() == s1 && with_stride.regClass() == s1))
3101          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), with_stride, offset);
3102       else if (offset.size() == 1 && with_stride.size() == 1)
3103          offset = bld.vadd32(bld.def(v1), with_stride, offset);
3104       else
3105          unreachable("Unsupported register class of indirect offset");
3106    } else {
3107       unsigned const_offset_arg = nir_src_as_uint(*off_src);
3108       const_offset += const_offset_arg * stride;
3109    }
3110
3111    return std::make_pair(offset, const_offset);
3112 }
3113
3114 std::pair<Temp, unsigned> offset_add(isel_context *ctx, const std::pair<Temp, unsigned> &off1, const std::pair<Temp, unsigned> &off2)
3115 {
3116    Builder bld(ctx->program, ctx->block);
3117    Temp offset;
3118
3119    if (off1.first.id() && off2.first.id()) {
3120       if (unlikely(off1.first.regClass() == s1 && off2.first.regClass() == s1))
3121          offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), off1.first, off2.first);
3122       else if (off1.first.size() == 1 && off2.first.size() == 1)
3123          offset = bld.vadd32(bld.def(v1), off1.first, off2.first);
3124       else
3125          unreachable("Unsupported register class of indirect offset");
3126    } else {
3127       offset = off1.first.id() ? off1.first : off2.first;
3128    }
3129
3130    return std::make_pair(offset, off1.second + off2.second);
3131 }
3132
3133 std::pair<Temp, unsigned> offset_mul(isel_context *ctx, const std::pair<Temp, unsigned> &offs, unsigned multiplier)
3134 {
3135    Builder bld(ctx->program, ctx->block);
3136    unsigned const_offset = offs.second * multiplier;
3137
3138    if (!offs.first.id())
3139       return std::make_pair(offs.first, const_offset);
3140
3141    Temp offset = unlikely(offs.first.regClass() == s1)
3142                  ? bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(multiplier), offs.first)
3143                  : bld.v_mul_imm(bld.def(v1), offs.first, multiplier);
3144
3145    return std::make_pair(offset, const_offset);
3146 }
3147
3148 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride, unsigned component_stride)
3149 {
3150    Builder bld(ctx->program, ctx->block);
3151
3152    /* base is the driver_location, which is already multiplied by 4, so is in dwords */
3153    unsigned const_offset = nir_intrinsic_base(instr) * base_stride;
3154    /* component is in bytes */
3155    const_offset += nir_intrinsic_component(instr) * component_stride;
3156
3157    /* offset should be interpreted in relation to the base, so the instruction effectively reads/writes another input/output when it has an offset */
3158    nir_src *off_src = nir_get_io_offset_src(instr);
3159    return offset_add_from_nir(ctx, std::make_pair(Temp(), const_offset), off_src, 4u * base_stride);
3160 }
3161
3162 std::pair<Temp, unsigned> get_intrinsic_io_basic_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned stride = 1u)
3163 {
3164    return get_intrinsic_io_basic_offset(ctx, instr, stride, stride);
3165 }
3166
3167 Temp get_tess_rel_patch_id(isel_context *ctx)
3168 {
3169    Builder bld(ctx->program, ctx->block);
3170
3171    switch (ctx->shader->info.stage) {
3172    case MESA_SHADER_TESS_CTRL:
3173       return bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffu),
3174                       get_arg(ctx, ctx->args->ac.tcs_rel_ids));
3175    case MESA_SHADER_TESS_EVAL:
3176       return get_arg(ctx, ctx->args->tes_rel_patch_id);
3177    default:
3178       unreachable("Unsupported stage in get_tess_rel_patch_id");
3179    }
3180 }
3181
3182 std::pair<Temp, unsigned> get_tcs_per_vertex_input_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3183 {
3184    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3185    Builder bld(ctx->program, ctx->block);
3186
3187    uint32_t tcs_in_patch_stride = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 4;
3188    uint32_t tcs_in_vertex_stride = ctx->tcs_num_inputs * 4;
3189
3190    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr);
3191
3192    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3193    offs = offset_add_from_nir(ctx, offs, vertex_index_src, tcs_in_vertex_stride);
3194
3195    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3196    Temp tcs_in_current_patch_offset = bld.v_mul24_imm(bld.def(v1), rel_patch_id, tcs_in_patch_stride);
3197    offs = offset_add(ctx, offs, std::make_pair(tcs_in_current_patch_offset, 0));
3198
3199    return offset_mul(ctx, offs, 4u);
3200 }
3201
3202 std::pair<Temp, unsigned> get_tcs_output_lds_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, bool per_vertex = false)
3203 {
3204    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3205    Builder bld(ctx->program, ctx->block);
3206
3207    uint32_t input_patch_size = ctx->args->options->key.tcs.input_vertices * ctx->tcs_num_inputs * 16;
3208    uint32_t num_tcs_outputs = util_last_bit64(ctx->args->shader_info->tcs.outputs_written);
3209    uint32_t num_tcs_patch_outputs = util_last_bit64(ctx->args->shader_info->tcs.patch_outputs_written);
3210    uint32_t output_vertex_size = num_tcs_outputs * 16;
3211    uint32_t pervertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3212    uint32_t output_patch_stride = pervertex_output_patch_size + num_tcs_patch_outputs * 16;
3213
3214    std::pair<Temp, unsigned> offs = instr
3215                                     ? get_intrinsic_io_basic_offset(ctx, instr, 4u)
3216                                     : std::make_pair(Temp(), 0u);
3217
3218    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3219    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, output_patch_stride);
3220
3221    if (per_vertex) {
3222       assert(instr);
3223
3224       nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3225       offs = offset_add_from_nir(ctx, offs, vertex_index_src, output_vertex_size);
3226
3227       uint32_t output_patch0_offset = (input_patch_size * ctx->tcs_num_patches);
3228       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_offset));
3229    } else {
3230       uint32_t output_patch0_patch_data_offset = (input_patch_size * ctx->tcs_num_patches + pervertex_output_patch_size);
3231       offs = offset_add(ctx, offs, std::make_pair(patch_off, output_patch0_patch_data_offset));
3232    }
3233
3234    return offs;
3235 }
3236
3237 std::pair<Temp, unsigned> get_tcs_per_vertex_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr)
3238 {
3239    Builder bld(ctx->program, ctx->block);
3240
3241    unsigned vertices_per_patch = ctx->shader->info.tess.tcs_vertices_out;
3242    unsigned attr_stride = vertices_per_patch * ctx->tcs_num_patches;
3243
3244    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u);
3245
3246    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3247    Temp patch_off = bld.v_mul24_imm(bld.def(v1), rel_patch_id, vertices_per_patch * 16u);
3248    offs = offset_add(ctx, offs, std::make_pair(patch_off, 0u));
3249
3250    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3251    offs = offset_add_from_nir(ctx, offs, vertex_index_src, 16u);
3252
3253    return offs;
3254 }
3255
3256 std::pair<Temp, unsigned> get_tcs_per_patch_output_vmem_offset(isel_context *ctx, nir_intrinsic_instr *instr = nullptr, unsigned const_base_offset = 0u)
3257 {
3258    Builder bld(ctx->program, ctx->block);
3259
3260    unsigned num_tcs_outputs = ctx->shader->info.stage == MESA_SHADER_TESS_CTRL
3261                               ? util_last_bit64(ctx->args->shader_info->tcs.outputs_written)
3262                               : ctx->args->options->key.tes.tcs_num_outputs;
3263
3264    unsigned output_vertex_size = num_tcs_outputs * 16;
3265    unsigned per_vertex_output_patch_size = ctx->shader->info.tess.tcs_vertices_out * output_vertex_size;
3266    unsigned per_patch_data_offset = per_vertex_output_patch_size * ctx->tcs_num_patches;
3267    unsigned attr_stride = ctx->tcs_num_patches;
3268
3269    std::pair<Temp, unsigned> offs = instr
3270                                     ? get_intrinsic_io_basic_offset(ctx, instr, attr_stride * 4u, 4u)
3271                                     : std::make_pair(Temp(), 0u);
3272
3273    if (const_base_offset)
3274       offs.second += const_base_offset * attr_stride;
3275
3276    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
3277    Temp patch_off = bld.v_mul_imm(bld.def(v1), rel_patch_id, 16u);
3278    offs = offset_add(ctx, offs, std::make_pair(patch_off, per_patch_data_offset));
3279
3280    return offs;
3281 }
3282
3283 bool tcs_driver_location_matches_api_mask(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex, uint64_t mask, bool *indirect)
3284 {
3285    unsigned off = nir_intrinsic_base(instr) * 4u;
3286    nir_src *off_src = nir_get_io_offset_src(instr);
3287
3288    if (!nir_src_is_const(*off_src)) {
3289       *indirect = true;
3290       return false;
3291    }
3292
3293    *indirect = false;
3294    off += nir_src_as_uint(*off_src) * 16u;
3295
3296    while (mask) {
3297       unsigned slot = u_bit_scan64(&mask) + (per_vertex ? 0 : VARYING_SLOT_PATCH0);
3298       if (off == shader_io_get_unique_index((gl_varying_slot) slot) * 16u)
3299          return true;
3300    }
3301
3302    return false;
3303 }
3304
3305 bool store_output_to_temps(isel_context *ctx, nir_intrinsic_instr *instr)
3306 {
3307    unsigned write_mask = nir_intrinsic_write_mask(instr);
3308    unsigned component = nir_intrinsic_component(instr);
3309    unsigned idx = nir_intrinsic_base(instr) + component;
3310
3311    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3312    if (off_instr->type != nir_instr_type_load_const)
3313       return false;
3314
3315    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3316    idx += nir_src_as_uint(instr->src[1]) * 4u;
3317
3318    if (instr->src[0].ssa->bit_size == 64)
3319       write_mask = widen_mask(write_mask, 2);
3320
3321    for (unsigned i = 0; i < 8; ++i) {
3322       if (write_mask & (1 << i)) {
3323          ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
3324          ctx->outputs.temps[idx] = emit_extract_vector(ctx, src, i, v1);
3325       }
3326       idx++;
3327    }
3328
3329    return true;
3330 }
3331
3332 bool load_input_from_temps(isel_context *ctx, nir_intrinsic_instr *instr, Temp dst)
3333 {
3334    /* Only TCS per-vertex inputs are supported by this function.
3335     * Per-vertex inputs only match between the VS/TCS invocation id when the number of invocations is the same.
3336     */
3337    if (ctx->shader->info.stage != MESA_SHADER_TESS_CTRL || !ctx->tcs_in_out_eq)
3338       return false;
3339
3340    nir_src *off_src = nir_get_io_offset_src(instr);
3341    nir_src *vertex_index_src = nir_get_io_vertex_index_src(instr);
3342    nir_instr *vertex_index_instr = vertex_index_src->ssa->parent_instr;
3343    bool can_use_temps = nir_src_is_const(*off_src) &&
3344                         vertex_index_instr->type == nir_instr_type_intrinsic &&
3345                         nir_instr_as_intrinsic(vertex_index_instr)->intrinsic == nir_intrinsic_load_invocation_id;
3346
3347    if (!can_use_temps)
3348       return false;
3349
3350    unsigned idx = nir_intrinsic_base(instr) + nir_intrinsic_component(instr) + 4 * nir_src_as_uint(*off_src);
3351    Temp *src = &ctx->inputs.temps[idx];
3352    Temp vec = create_vec_from_array(ctx, src, dst.size(), dst.regClass().type(), 4u);
3353    assert(vec.size() == dst.size());
3354
3355    Builder bld(ctx->program, ctx->block);
3356    bld.copy(Definition(dst), vec);
3357    return true;
3358 }
3359
3360 void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr)
3361 {
3362    Builder bld(ctx->program, ctx->block);
3363
3364    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, 4u);
3365    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3366    unsigned write_mask = nir_intrinsic_write_mask(instr);
3367    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
3368
3369    if (ctx->tcs_in_out_eq && store_output_to_temps(ctx, instr)) {
3370       /* When the TCS only reads this output directly and for the same vertices as its invocation id, it is unnecessary to store the VS output to LDS. */
3371       bool indirect_write;
3372       bool temp_only_input = tcs_driver_location_matches_api_mask(ctx, instr, true, ctx->tcs_temp_only_inputs, &indirect_write);
3373       if (temp_only_input && !indirect_write)
3374          return;
3375    }
3376
3377    if (ctx->stage == vertex_es || ctx->stage == tess_eval_es) {
3378       /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */
3379       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
3380       Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset);
3381       store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true);
3382    } else {
3383       Temp lds_base;
3384
3385       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
3386          /* GFX9+: ES stage is merged into GS, data is passed between them using LDS. */
3387          unsigned itemsize = ctx->stage == vertex_geometry_gs
3388                              ? ctx->program->info->vs.es_info.esgs_itemsize
3389                              : ctx->program->info->tes.es_info.esgs_itemsize;
3390          Temp thread_id = emit_mbcnt(ctx, bld.def(v1));
3391          Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
3392          Temp vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), thread_id,
3393                                bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
3394          lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
3395       } else if (ctx->stage == vertex_ls || ctx->stage == vertex_tess_control_hs) {
3396          /* GFX6-8: VS runs on LS stage when tessellation is used, but LS shares LDS space with HS.
3397           * GFX9+: LS is merged into HS, but still uses the same LDS layout.
3398           */
3399          unsigned num_tcs_inputs = util_last_bit64(ctx->args->shader_info->vs.ls_outputs_written);
3400          Temp vertex_idx = get_arg(ctx, ctx->args->rel_auto_id);
3401          lds_base = bld.v_mul_imm(bld.def(v1), vertex_idx, num_tcs_inputs * 16u);
3402       } else {
3403          unreachable("Invalid LS or ES stage");
3404       }
3405
3406       offs = offset_add(ctx, offs, std::make_pair(lds_base, 0u));
3407       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
3408       store_lds(ctx, elem_size_bytes, src, write_mask, offs.first, offs.second, lds_align);
3409    }
3410 }
3411
3412 bool should_write_tcs_patch_output_to_vmem(isel_context *ctx, nir_intrinsic_instr *instr)
3413 {
3414    unsigned off = nir_intrinsic_base(instr) * 4u;
3415    return off != ctx->tcs_tess_lvl_out_loc &&
3416           off != ctx->tcs_tess_lvl_in_loc;
3417 }
3418
3419 bool should_write_tcs_output_to_lds(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3420 {
3421    /* When none of the appropriate outputs are read, we are OK to never write to LDS */
3422    if (per_vertex ? ctx->shader->info.outputs_read == 0U : ctx->shader->info.patch_outputs_read == 0u)
3423       return false;
3424
3425    uint64_t mask = per_vertex
3426                    ? ctx->shader->info.outputs_read
3427                    : ctx->shader->info.patch_outputs_read;
3428    bool indirect_write;
3429    bool output_read = tcs_driver_location_matches_api_mask(ctx, instr, per_vertex, mask, &indirect_write);
3430    return indirect_write || output_read;
3431 }
3432
3433 void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3434 {
3435    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3436    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3437
3438    Builder bld(ctx->program, ctx->block);
3439
3440    Temp store_val = get_ssa_temp(ctx, instr->src[0].ssa);
3441    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3442    unsigned write_mask = nir_intrinsic_write_mask(instr);
3443
3444    /* Only write to VMEM if the output is per-vertex or it's per-patch non tess factor */
3445    bool write_to_vmem = per_vertex || should_write_tcs_patch_output_to_vmem(ctx, instr);
3446    /* Only write to LDS if the output is read by the shader, or it's per-patch tess factor */
3447    bool write_to_lds = !write_to_vmem || should_write_tcs_output_to_lds(ctx, instr, per_vertex);
3448
3449    if (write_to_vmem) {
3450       std::pair<Temp, unsigned> vmem_offs = per_vertex
3451                                             ? get_tcs_per_vertex_output_vmem_offset(ctx, instr)
3452                                             : get_tcs_per_patch_output_vmem_offset(ctx, instr);
3453
3454       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
3455       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
3456       store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, false);
3457    }
3458
3459    if (write_to_lds) {
3460       std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3461       unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3462       store_lds(ctx, elem_size_bytes, store_val, write_mask, lds_offs.first, lds_offs.second, lds_align);
3463    }
3464 }
3465
3466 void visit_load_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool per_vertex)
3467 {
3468    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
3469    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
3470
3471    Builder bld(ctx->program, ctx->block);
3472
3473    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3474    std::pair<Temp, unsigned> lds_offs = get_tcs_output_lds_offset(ctx, instr, per_vertex);
3475    unsigned lds_align = calculate_lds_alignment(ctx, lds_offs.second);
3476    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
3477
3478    load_lds(ctx, elem_size_bytes, dst, lds_offs.first, lds_offs.second, lds_align);
3479 }
3480
3481 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
3482 {
3483    if (ctx->stage == vertex_vs ||
3484        ctx->stage == tess_eval_vs ||
3485        ctx->stage == fragment_fs ||
3486        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
3487       bool stored_to_temps = store_output_to_temps(ctx, instr);
3488       if (!stored_to_temps) {
3489          fprintf(stderr, "Unimplemented output offset instruction:\n");
3490          nir_print_instr(instr->src[1].ssa->parent_instr, stderr);
3491          fprintf(stderr, "\n");
3492          abort();
3493       }
3494    } else if (ctx->stage == vertex_es ||
3495               ctx->stage == vertex_ls ||
3496               ctx->stage == tess_eval_es ||
3497               (ctx->stage == vertex_tess_control_hs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3498               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX) ||
3499               (ctx->stage == tess_eval_geometry_gs && ctx->shader->info.stage == MESA_SHADER_TESS_EVAL)) {
3500       visit_store_ls_or_es_output(ctx, instr);
3501    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
3502       visit_store_tcs_output(ctx, instr, false);
3503    } else {
3504       unreachable("Shader stage not implemented");
3505    }
3506 }
3507
3508 void visit_load_output(isel_context *ctx, nir_intrinsic_instr *instr)
3509 {
3510    visit_load_tcs_output(ctx, instr, false);
3511 }
3512
3513 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3514 {
3515    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3516    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3517
3518    Builder bld(ctx->program, ctx->block);
3519    Builder::Result interp_p1 = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3520    if (ctx->program->has_16bank_lds)
3521       interp_p1.instr->operands[0].setLateKill(true);
3522    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), interp_p1, idx, component);
3523 }
3524
3525 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3526 {
3527    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3528    for (unsigned i = 0; i < num_components; i++)
3529       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3530    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3531       assert(num_components == 4);
3532       Builder bld(ctx->program, ctx->block);
3533       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3534    }
3535
3536    for (Operand& op : vec->operands)
3537       op = op.isUndefined() ? Operand(0u) : op;
3538
3539    vec->definitions[0] = Definition(dst);
3540    ctx->block->instructions.emplace_back(std::move(vec));
3541    emit_split_vector(ctx, dst, num_components);
3542    return;
3543 }
3544
3545 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3546 {
3547    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3548    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3549    unsigned idx = nir_intrinsic_base(instr);
3550    unsigned component = nir_intrinsic_component(instr);
3551    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3552
3553    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3554    if (offset) {
3555       assert(offset->u32 == 0);
3556    } else {
3557       /* the lower 15bit of the prim_mask contain the offset into LDS
3558        * while the upper bits contain the number of prims */
3559       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3560       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3561       Builder bld(ctx->program, ctx->block);
3562       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3563       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3564       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3565       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3566       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3567    }
3568
3569    if (instr->dest.ssa.num_components == 1) {
3570       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3571    } else {
3572       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3573       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3574       {
3575          Temp tmp = {ctx->program->allocateId(), v1};
3576          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3577          vec->operands[i] = Operand(tmp);
3578       }
3579       vec->definitions[0] = Definition(dst);
3580       ctx->block->instructions.emplace_back(std::move(vec));
3581    }
3582 }
3583
3584 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3585                              unsigned offset, unsigned stride, unsigned channels)
3586 {
3587    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3588    if (vtx_info->chan_byte_size != 4 && channels == 3)
3589       return false;
3590    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3591           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3592 }
3593
3594 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3595                               unsigned offset, unsigned stride, unsigned *channels)
3596 {
3597    if (!vtx_info->chan_byte_size) {
3598       *channels = vtx_info->num_channels;
3599       return vtx_info->chan_format;
3600    }
3601
3602    unsigned num_channels = *channels;
3603    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3604       unsigned new_channels = num_channels + 1;
3605       /* first, assume more loads is worse and try using a larger data format */
3606       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3607          new_channels++;
3608          /* don't make the attribute potentially out-of-bounds */
3609          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3610             new_channels = 5;
3611       }
3612
3613       if (new_channels == 5) {
3614          /* then try decreasing load size (at the cost of more loads) */
3615          new_channels = *channels;
3616          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3617             new_channels--;
3618       }
3619
3620       if (new_channels < *channels)
3621          *channels = new_channels;
3622       num_channels = new_channels;
3623    }
3624
3625    switch (vtx_info->chan_format) {
3626    case V_008F0C_BUF_DATA_FORMAT_8:
3627       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3628                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3629    case V_008F0C_BUF_DATA_FORMAT_16:
3630       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3631                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3632    case V_008F0C_BUF_DATA_FORMAT_32:
3633       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3634                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3635    }
3636    unreachable("shouldn't reach here");
3637    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3638 }
3639
3640 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3641  * so we may need to fix it up. */
3642 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3643 {
3644    Builder bld(ctx->program, ctx->block);
3645
3646    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3647       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3648
3649    /* For the integer-like cases, do a natural sign extension.
3650     *
3651     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3652     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3653     * exponent.
3654     */
3655    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3656    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3657
3658    /* Convert back to the right type. */
3659    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3660       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3661       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3662       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3663    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3664       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3665    }
3666
3667    return alpha;
3668 }
3669
3670 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3671 {
3672    Builder bld(ctx->program, ctx->block);
3673    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3674    if (ctx->shader->info.stage == MESA_SHADER_VERTEX) {
3675
3676       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3677       if (off_instr->type != nir_instr_type_load_const) {
3678          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3679          nir_print_instr(off_instr, stderr);
3680          fprintf(stderr, "\n");
3681       }
3682       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3683
3684       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3685
3686       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3687       unsigned component = nir_intrinsic_component(instr);
3688       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3689       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3690       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3691       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3692
3693       unsigned dfmt = attrib_format & 0xf;
3694       unsigned nfmt = (attrib_format >> 4) & 0x7;
3695       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3696
3697       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3698       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3699       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3700       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3701       if (post_shuffle)
3702          num_channels = MAX2(num_channels, 3);
3703
3704       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3705       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3706
3707       Temp index;
3708       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3709          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3710          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3711          if (divisor) {
3712             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3713             if (divisor != 1) {
3714                Temp divided = bld.tmp(v1);
3715                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3716                index = bld.vadd32(bld.def(v1), start_instance, divided);
3717             } else {
3718                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3719             }
3720          } else {
3721             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3722          }
3723       } else {
3724          index = bld.vadd32(bld.def(v1),
3725                             get_arg(ctx, ctx->args->ac.base_vertex),
3726                             get_arg(ctx, ctx->args->ac.vertex_id));
3727       }
3728
3729       Temp channels[num_channels];
3730       unsigned channel_start = 0;
3731       bool direct_fetch = false;
3732
3733       /* skip unused channels at the start */
3734       if (vtx_info->chan_byte_size && !post_shuffle) {
3735          channel_start = ffs(mask) - 1;
3736          for (unsigned i = 0; i < channel_start; i++)
3737             channels[i] = Temp(0, s1);
3738       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3739          num_channels = 3 - (ffs(mask) - 1);
3740       }
3741
3742       /* load channels */
3743       while (channel_start < num_channels) {
3744          unsigned fetch_size = num_channels - channel_start;
3745          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3746          bool expanded = false;
3747
3748          /* use MUBUF when possible to avoid possible alignment issues */
3749          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3750          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3751                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3752                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3753                           vtx_info->chan_byte_size == 4;
3754          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3755          if (!use_mubuf) {
3756             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3757          } else {
3758             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
3759                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
3760                fetch_size = 4;
3761                expanded = true;
3762             }
3763          }
3764
3765          Temp fetch_index = index;
3766          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3767             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3768             fetch_offset = fetch_offset % attrib_stride;
3769          }
3770
3771          Operand soffset(0u);
3772          if (fetch_offset >= 4096) {
3773             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3774             fetch_offset %= 4096;
3775          }
3776
3777          aco_opcode opcode;
3778          switch (fetch_size) {
3779          case 1:
3780             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3781             break;
3782          case 2:
3783             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3784             break;
3785          case 3:
3786             assert(ctx->options->chip_class >= GFX7 ||
3787                    (!use_mubuf && ctx->options->chip_class == GFX6));
3788             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3789             break;
3790          case 4:
3791             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3792             break;
3793          default:
3794             unreachable("Unimplemented load_input vector size");
3795          }
3796
3797          Temp fetch_dst;
3798          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3799              !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE ||
3800                            num_channels <= 3)) {
3801             direct_fetch = true;
3802             fetch_dst = dst;
3803          } else {
3804             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3805          }
3806
3807          if (use_mubuf) {
3808             Instruction *mubuf = bld.mubuf(opcode,
3809                                            Definition(fetch_dst), list, fetch_index, soffset,
3810                                            fetch_offset, false, true).instr;
3811             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3812          } else {
3813             Instruction *mtbuf = bld.mtbuf(opcode,
3814                                            Definition(fetch_dst), list, fetch_index, soffset,
3815                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3816             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3817          }
3818
3819          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3820
3821          if (fetch_size == 1) {
3822             channels[channel_start] = fetch_dst;
3823          } else {
3824             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3825                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3826          }
3827
3828          channel_start += fetch_size;
3829       }
3830
3831       if (!direct_fetch) {
3832          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3833                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3834
3835          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3836          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3837          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3838
3839          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3840          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3841          unsigned num_temp = 0;
3842          for (unsigned i = 0; i < dst.size(); i++) {
3843             unsigned idx = i + component;
3844             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3845                Temp channel = channels[swizzle[idx]];
3846                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3847                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3848                vec->operands[i] = Operand(channel);
3849
3850                num_temp++;
3851                elems[i] = channel;
3852             } else if (is_float && idx == 3) {
3853                vec->operands[i] = Operand(0x3f800000u);
3854             } else if (!is_float && idx == 3) {
3855                vec->operands[i] = Operand(1u);
3856             } else {
3857                vec->operands[i] = Operand(0u);
3858             }
3859          }
3860          vec->definitions[0] = Definition(dst);
3861          ctx->block->instructions.emplace_back(std::move(vec));
3862          emit_split_vector(ctx, dst, dst.size());
3863
3864          if (num_temp == dst.size())
3865             ctx->allocated_vec.emplace(dst.id(), elems);
3866       }
3867    } else if (ctx->shader->info.stage == MESA_SHADER_FRAGMENT) {
3868       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
3869       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
3870       if (off_instr->type != nir_instr_type_load_const ||
3871           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3872          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3873          nir_print_instr(off_instr, stderr);
3874          fprintf(stderr, "\n");
3875       }
3876
3877       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3878       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
3879       if (offset) {
3880          assert(offset->u32 == 0);
3881       } else {
3882          /* the lower 15bit of the prim_mask contain the offset into LDS
3883           * while the upper bits contain the number of prims */
3884          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
3885          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3886          Builder bld(ctx->program, ctx->block);
3887          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3888          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3889          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3890          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3891          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3892       }
3893
3894       unsigned idx = nir_intrinsic_base(instr);
3895       unsigned component = nir_intrinsic_component(instr);
3896       unsigned vertex_id = 2; /* P0 */
3897
3898       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
3899          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
3900          switch (src0->u32) {
3901          case 0:
3902             vertex_id = 2; /* P0 */
3903             break;
3904          case 1:
3905             vertex_id = 0; /* P10 */
3906             break;
3907          case 2:
3908             vertex_id = 1; /* P20 */
3909             break;
3910          default:
3911             unreachable("invalid vertex index");
3912          }
3913       }
3914
3915       if (dst.size() == 1) {
3916          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
3917       } else {
3918          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3919          for (unsigned i = 0; i < dst.size(); i++)
3920             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
3921          vec->definitions[0] = Definition(dst);
3922          bld.insert(std::move(vec));
3923       }
3924
3925    } else if (ctx->shader->info.stage == MESA_SHADER_TESS_EVAL) {
3926       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
3927       Temp soffset = get_arg(ctx, ctx->args->oc_lds);
3928       std::pair<Temp, unsigned> offs = get_tcs_per_patch_output_vmem_offset(ctx, instr);
3929       unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8u;
3930
3931       load_vmem_mubuf(ctx, dst, ring, offs.first, soffset, offs.second, elem_size_bytes, instr->dest.ssa.num_components);
3932    } else {
3933       unreachable("Shader stage not implemented");
3934    }
3935 }
3936
3937 std::pair<Temp, unsigned> get_gs_per_vertex_input_offset(isel_context *ctx, nir_intrinsic_instr *instr, unsigned base_stride = 1u)
3938 {
3939    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3940
3941    Builder bld(ctx->program, ctx->block);
3942    nir_src *vertex_src = nir_get_io_vertex_index_src(instr);
3943    Temp vertex_offset;
3944
3945    if (!nir_src_is_const(*vertex_src)) {
3946       /* better code could be created, but this case probably doesn't happen
3947        * much in practice */
3948       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, vertex_src->ssa));
3949       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3950          Temp elem;
3951
3952          if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
3953             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3954             if (i % 2u)
3955                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3956          } else {
3957             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3958          }
3959
3960          if (vertex_offset.id()) {
3961             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)),
3962                                  Operand(i), indirect_vertex);
3963             vertex_offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), vertex_offset, elem, cond);
3964          } else {
3965             vertex_offset = elem;
3966          }
3967       }
3968
3969       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
3970          vertex_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), vertex_offset);
3971    } else {
3972       unsigned vertex = nir_src_as_uint(*vertex_src);
3973       if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs)
3974          vertex_offset = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
3975                                   get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3976                                   Operand((vertex % 2u) * 16u), Operand(16u));
3977       else
3978          vertex_offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3979    }
3980
3981    std::pair<Temp, unsigned> offs = get_intrinsic_io_basic_offset(ctx, instr, base_stride);
3982    offs = offset_add(ctx, offs, std::make_pair(vertex_offset, 0u));
3983    return offset_mul(ctx, offs, 4u);
3984 }
3985
3986 void visit_load_gs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3987 {
3988    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3989
3990    Builder bld(ctx->program, ctx->block);
3991    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3992    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3993
3994    if (ctx->stage == geometry_gs) {
3995       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr, ctx->program->wave_size);
3996       Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3997       load_vmem_mubuf(ctx, dst, ring, offs.first, Temp(), offs.second, elem_size_bytes, instr->dest.ssa.num_components, 4u * ctx->program->wave_size, false, true);
3998    } else if (ctx->stage == vertex_geometry_gs || ctx->stage == tess_eval_geometry_gs) {
3999       std::pair<Temp, unsigned> offs = get_gs_per_vertex_input_offset(ctx, instr);
4000       unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4001       load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4002    } else {
4003       unreachable("Unsupported GS stage.");
4004    }
4005 }
4006
4007 void visit_load_tcs_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4008 {
4009    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4010
4011    Builder bld(ctx->program, ctx->block);
4012    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4013
4014    if (load_input_from_temps(ctx, instr, dst))
4015       return;
4016
4017    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_input_lds_offset(ctx, instr);
4018    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4019    unsigned lds_align = calculate_lds_alignment(ctx, offs.second);
4020
4021    load_lds(ctx, elem_size_bytes, dst, offs.first, offs.second, lds_align);
4022 }
4023
4024 void visit_load_tes_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4025 {
4026    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4027
4028    Builder bld(ctx->program, ctx->block);
4029
4030    Temp ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
4031    Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
4032    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4033
4034    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4035    std::pair<Temp, unsigned> offs = get_tcs_per_vertex_output_vmem_offset(ctx, instr);
4036
4037    load_vmem_mubuf(ctx, dst, ring, offs.first, oc_lds, offs.second, elem_size_bytes, instr->dest.ssa.num_components, 0u, true, true);
4038 }
4039
4040 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
4041 {
4042    switch (ctx->shader->info.stage) {
4043    case MESA_SHADER_GEOMETRY:
4044       visit_load_gs_per_vertex_input(ctx, instr);
4045       break;
4046    case MESA_SHADER_TESS_CTRL:
4047       visit_load_tcs_per_vertex_input(ctx, instr);
4048       break;
4049    case MESA_SHADER_TESS_EVAL:
4050       visit_load_tes_per_vertex_input(ctx, instr);
4051       break;
4052    default:
4053       unreachable("Unimplemented shader stage");
4054    }
4055 }
4056
4057 void visit_load_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4058 {
4059    visit_load_tcs_output(ctx, instr, true);
4060 }
4061
4062 void visit_store_per_vertex_output(isel_context *ctx, nir_intrinsic_instr *instr)
4063 {
4064    assert(ctx->stage == tess_control_hs || ctx->stage == vertex_tess_control_hs);
4065    assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL);
4066
4067    visit_store_tcs_output(ctx, instr, true);
4068 }
4069
4070 void visit_load_tess_coord(isel_context *ctx, nir_intrinsic_instr *instr)
4071 {
4072    assert(ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
4073
4074    Builder bld(ctx->program, ctx->block);
4075    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4076
4077    Operand tes_u(get_arg(ctx, ctx->args->tes_u));
4078    Operand tes_v(get_arg(ctx, ctx->args->tes_v));
4079    Operand tes_w(0u);
4080
4081    if (ctx->shader->info.tess.primitive_mode == GL_TRIANGLES) {
4082       Temp tmp = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), tes_u, tes_v);
4083       tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0x3f800000u /* 1.0f */), tmp);
4084       tes_w = Operand(tmp);
4085    }
4086
4087    Temp tess_coord = bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tes_u, tes_v, tes_w);
4088    emit_split_vector(ctx, tess_coord, 3);
4089 }
4090
4091 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
4092 {
4093    if (ctx->program->info->need_indirect_descriptor_sets) {
4094       Builder bld(ctx->program, ctx->block);
4095       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
4096       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
4097       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
4098    }
4099
4100    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
4101 }
4102
4103
4104 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
4105 {
4106    Builder bld(ctx->program, ctx->block);
4107    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
4108    if (!ctx->divergent_vals[instr->dest.ssa.index])
4109       index = bld.as_uniform(index);
4110    unsigned desc_set = nir_intrinsic_desc_set(instr);
4111    unsigned binding = nir_intrinsic_binding(instr);
4112
4113    Temp desc_ptr;
4114    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
4115    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
4116    unsigned offset = layout->binding[binding].offset;
4117    unsigned stride;
4118    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
4119        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
4120       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
4121       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
4122       offset = pipeline_layout->push_constant_size + 16 * idx;
4123       stride = 16;
4124    } else {
4125       desc_ptr = load_desc_ptr(ctx, desc_set);
4126       stride = layout->binding[binding].size;
4127    }
4128
4129    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
4130    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
4131    if (stride != 1) {
4132       if (nir_const_index) {
4133          const_index = const_index * stride;
4134       } else if (index.type() == RegType::vgpr) {
4135          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
4136          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
4137       } else {
4138          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
4139       }
4140    }
4141    if (offset) {
4142       if (nir_const_index) {
4143          const_index = const_index + offset;
4144       } else if (index.type() == RegType::vgpr) {
4145          index = bld.vadd32(bld.def(v1), Operand(offset), index);
4146       } else {
4147          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
4148       }
4149    }
4150
4151    if (nir_const_index && const_index == 0) {
4152       index = desc_ptr;
4153    } else if (index.type() == RegType::vgpr) {
4154       index = bld.vadd32(bld.def(v1),
4155                          nir_const_index ? Operand(const_index) : Operand(index),
4156                          Operand(desc_ptr));
4157    } else {
4158       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4159                        nir_const_index ? Operand(const_index) : Operand(index),
4160                        Operand(desc_ptr));
4161    }
4162
4163    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
4164 }
4165
4166 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
4167                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
4168 {
4169    Builder bld(ctx->program, ctx->block);
4170
4171    unsigned num_bytes = dst.size() * 4;
4172    bool dlc = glc && ctx->options->chip_class >= GFX10;
4173
4174    aco_opcode op;
4175    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
4176       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4177       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4178       unsigned const_offset = 0;
4179
4180       Temp lower = Temp();
4181       if (num_bytes > 16) {
4182          assert(num_components == 3 || num_components == 4);
4183          op = aco_opcode::buffer_load_dwordx4;
4184          lower = bld.tmp(v4);
4185          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4186          mubuf->definitions[0] = Definition(lower);
4187          mubuf->operands[0] = Operand(rsrc);
4188          mubuf->operands[1] = vaddr;
4189          mubuf->operands[2] = soffset;
4190          mubuf->offen = (offset.type() == RegType::vgpr);
4191          mubuf->glc = glc;
4192          mubuf->dlc = dlc;
4193          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4194          mubuf->can_reorder = readonly;
4195          bld.insert(std::move(mubuf));
4196          emit_split_vector(ctx, lower, 2);
4197          num_bytes -= 16;
4198          const_offset = 16;
4199       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
4200          /* GFX6 doesn't support loading vec3, expand to vec4. */
4201          num_bytes = 16;
4202       }
4203
4204       switch (num_bytes) {
4205          case 4:
4206             op = aco_opcode::buffer_load_dword;
4207             break;
4208          case 8:
4209             op = aco_opcode::buffer_load_dwordx2;
4210             break;
4211          case 12:
4212             assert(ctx->options->chip_class > GFX6);
4213             op = aco_opcode::buffer_load_dwordx3;
4214             break;
4215          case 16:
4216             op = aco_opcode::buffer_load_dwordx4;
4217             break;
4218          default:
4219             unreachable("Load SSBO not implemented for this size.");
4220       }
4221       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4222       mubuf->operands[0] = Operand(rsrc);
4223       mubuf->operands[1] = vaddr;
4224       mubuf->operands[2] = soffset;
4225       mubuf->offen = (offset.type() == RegType::vgpr);
4226       mubuf->glc = glc;
4227       mubuf->dlc = dlc;
4228       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
4229       mubuf->can_reorder = readonly;
4230       mubuf->offset = const_offset;
4231       aco_ptr<Instruction> instr = std::move(mubuf);
4232
4233       if (dst.size() > 4) {
4234          assert(lower != Temp());
4235          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
4236          instr->definitions[0] = Definition(upper);
4237          bld.insert(std::move(instr));
4238          if (dst.size() == 8)
4239             emit_split_vector(ctx, upper, 2);
4240          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
4241          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
4242          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
4243          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
4244          if (dst.size() == 8)
4245             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
4246       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
4247          Temp vec = bld.tmp(v4);
4248          instr->definitions[0] = Definition(vec);
4249          bld.insert(std::move(instr));
4250          emit_split_vector(ctx, vec, 4);
4251
4252          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4253          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4254          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4255          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4256       }
4257
4258       if (dst.type() == RegType::sgpr) {
4259          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4260          instr->definitions[0] = Definition(vec);
4261          bld.insert(std::move(instr));
4262          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4263       } else {
4264          instr->definitions[0] = Definition(dst);
4265          bld.insert(std::move(instr));
4266          emit_split_vector(ctx, dst, num_components);
4267       }
4268    } else {
4269       switch (num_bytes) {
4270          case 4:
4271             op = aco_opcode::s_buffer_load_dword;
4272             break;
4273          case 8:
4274             op = aco_opcode::s_buffer_load_dwordx2;
4275             break;
4276          case 12:
4277          case 16:
4278             op = aco_opcode::s_buffer_load_dwordx4;
4279             break;
4280          case 24:
4281          case 32:
4282             op = aco_opcode::s_buffer_load_dwordx8;
4283             break;
4284          default:
4285             unreachable("Load SSBO not implemented for this size.");
4286       }
4287       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4288       load->operands[0] = Operand(rsrc);
4289       load->operands[1] = Operand(bld.as_uniform(offset));
4290       assert(load->operands[1].getTemp().type() == RegType::sgpr);
4291       load->definitions[0] = Definition(dst);
4292       load->glc = glc;
4293       load->dlc = dlc;
4294       load->barrier = readonly ? barrier_none : barrier_buffer;
4295       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
4296       assert(ctx->options->chip_class >= GFX8 || !glc);
4297
4298       /* trim vector */
4299       if (dst.size() == 3) {
4300          Temp vec = bld.tmp(s4);
4301          load->definitions[0] = Definition(vec);
4302          bld.insert(std::move(load));
4303          emit_split_vector(ctx, vec, 4);
4304
4305          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4306                     emit_extract_vector(ctx, vec, 0, s1),
4307                     emit_extract_vector(ctx, vec, 1, s1),
4308                     emit_extract_vector(ctx, vec, 2, s1));
4309       } else if (dst.size() == 6) {
4310          Temp vec = bld.tmp(s8);
4311          load->definitions[0] = Definition(vec);
4312          bld.insert(std::move(load));
4313          emit_split_vector(ctx, vec, 4);
4314
4315          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4316                     emit_extract_vector(ctx, vec, 0, s2),
4317                     emit_extract_vector(ctx, vec, 1, s2),
4318                     emit_extract_vector(ctx, vec, 2, s2));
4319       } else {
4320          bld.insert(std::move(load));
4321       }
4322       emit_split_vector(ctx, dst, num_components);
4323    }
4324 }
4325
4326 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
4327 {
4328    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4329    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
4330
4331    Builder bld(ctx->program, ctx->block);
4332
4333    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
4334    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
4335    unsigned binding = nir_intrinsic_binding(idx_instr);
4336    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
4337
4338    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
4339       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4340                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4341                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4342                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4343       if (ctx->options->chip_class >= GFX10) {
4344          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4345                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4346                       S_008F0C_RESOURCE_LEVEL(1);
4347       } else {
4348          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4349                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4350       }
4351       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
4352                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
4353                                      Operand(0xFFFFFFFFu),
4354                                      Operand(desc_type));
4355       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4356                         rsrc, upper_dwords);
4357    } else {
4358       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
4359       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4360    }
4361
4362    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
4363 }
4364
4365 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4366 {
4367    Builder bld(ctx->program, ctx->block);
4368    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4369
4370    unsigned offset = nir_intrinsic_base(instr);
4371    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
4372    if (index_cv && instr->dest.ssa.bit_size == 32) {
4373
4374       unsigned count = instr->dest.ssa.num_components;
4375       unsigned start = (offset + index_cv->u32) / 4u;
4376       start -= ctx->args->ac.base_inline_push_consts;
4377       if (start + count <= ctx->args->ac.num_inline_push_consts) {
4378          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4379          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4380          for (unsigned i = 0; i < count; ++i) {
4381             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
4382             vec->operands[i] = Operand{elems[i]};
4383          }
4384          vec->definitions[0] = Definition(dst);
4385          ctx->block->instructions.emplace_back(std::move(vec));
4386          ctx->allocated_vec.emplace(dst.id(), elems);
4387          return;
4388       }
4389    }
4390
4391    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
4392    if (offset != 0) // TODO check if index != 0 as well
4393       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
4394    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
4395    Temp vec = dst;
4396    bool trim = false;
4397    aco_opcode op;
4398
4399    switch (dst.size()) {
4400    case 1:
4401       op = aco_opcode::s_load_dword;
4402       break;
4403    case 2:
4404       op = aco_opcode::s_load_dwordx2;
4405       break;
4406    case 3:
4407       vec = bld.tmp(s4);
4408       trim = true;
4409    case 4:
4410       op = aco_opcode::s_load_dwordx4;
4411       break;
4412    case 6:
4413       vec = bld.tmp(s8);
4414       trim = true;
4415    case 8:
4416       op = aco_opcode::s_load_dwordx8;
4417       break;
4418    default:
4419       unreachable("unimplemented or forbidden load_push_constant.");
4420    }
4421
4422    bld.smem(op, Definition(vec), ptr, index);
4423
4424    if (trim) {
4425       emit_split_vector(ctx, vec, 4);
4426       RegClass rc = dst.size() == 3 ? s1 : s2;
4427       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4428                  emit_extract_vector(ctx, vec, 0, rc),
4429                  emit_extract_vector(ctx, vec, 1, rc),
4430                  emit_extract_vector(ctx, vec, 2, rc));
4431
4432    }
4433    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4434 }
4435
4436 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
4437 {
4438    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4439
4440    Builder bld(ctx->program, ctx->block);
4441
4442    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
4443                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
4444                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
4445                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
4446    if (ctx->options->chip_class >= GFX10) {
4447       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4448                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
4449                    S_008F0C_RESOURCE_LEVEL(1);
4450    } else {
4451       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4452                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4453    }
4454
4455    unsigned base = nir_intrinsic_base(instr);
4456    unsigned range = nir_intrinsic_range(instr);
4457
4458    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
4459    if (base && offset.type() == RegType::sgpr)
4460       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
4461    else if (base && offset.type() == RegType::vgpr)
4462       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
4463
4464    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4465                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
4466                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
4467                           Operand(desc_type));
4468
4469    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
4470 }
4471
4472 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
4473 {
4474    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4475       ctx->cf_info.exec_potentially_empty_discard = true;
4476
4477    ctx->program->needs_exact = true;
4478
4479    // TODO: optimize uniform conditions
4480    Builder bld(ctx->program, ctx->block);
4481    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
4482    assert(src.regClass() == bld.lm);
4483    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
4484    bld.pseudo(aco_opcode::p_discard_if, src);
4485    ctx->block->kind |= block_kind_uses_discard_if;
4486    return;
4487 }
4488
4489 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
4490 {
4491    Builder bld(ctx->program, ctx->block);
4492
4493    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
4494       ctx->cf_info.exec_potentially_empty_discard = true;
4495
4496    bool divergent = ctx->cf_info.parent_if.is_divergent ||
4497                     ctx->cf_info.parent_loop.has_divergent_continue;
4498
4499    if (ctx->block->loop_nest_depth &&
4500        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
4501       /* we handle discards the same way as jump instructions */
4502       append_logical_end(ctx->block);
4503
4504       /* in loops, discard behaves like break */
4505       Block *linear_target = ctx->cf_info.parent_loop.exit;
4506       ctx->block->kind |= block_kind_discard;
4507
4508       if (!divergent) {
4509          /* uniform discard - loop ends here */
4510          assert(nir_instr_is_last(&instr->instr));
4511          ctx->block->kind |= block_kind_uniform;
4512          ctx->cf_info.has_branch = true;
4513          bld.branch(aco_opcode::p_branch);
4514          add_linear_edge(ctx->block->index, linear_target);
4515          return;
4516       }
4517
4518       /* we add a break right behind the discard() instructions */
4519       ctx->block->kind |= block_kind_break;
4520       unsigned idx = ctx->block->index;
4521
4522       ctx->cf_info.parent_loop.has_divergent_branch = true;
4523       ctx->cf_info.nir_to_aco[instr->instr.block->index] = idx;
4524
4525       /* remove critical edges from linear CFG */
4526       bld.branch(aco_opcode::p_branch);
4527       Block* break_block = ctx->program->create_and_insert_block();
4528       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
4529       break_block->kind |= block_kind_uniform;
4530       add_linear_edge(idx, break_block);
4531       add_linear_edge(break_block->index, linear_target);
4532       bld.reset(break_block);
4533       bld.branch(aco_opcode::p_branch);
4534
4535       Block* continue_block = ctx->program->create_and_insert_block();
4536       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
4537       add_linear_edge(idx, continue_block);
4538       append_logical_start(continue_block);
4539       ctx->block = continue_block;
4540
4541       return;
4542    }
4543
4544    /* it can currently happen that NIR doesn't remove the unreachable code */
4545    if (!nir_instr_is_last(&instr->instr)) {
4546       ctx->program->needs_exact = true;
4547       /* save exec somewhere temporarily so that it doesn't get
4548        * overwritten before the discard from outer exec masks */
4549       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
4550       bld.pseudo(aco_opcode::p_discard_if, cond);
4551       ctx->block->kind |= block_kind_uses_discard_if;
4552       return;
4553    }
4554
4555    /* This condition is incorrect for uniformly branched discards in a loop
4556     * predicated by a divergent condition, but the above code catches that case
4557     * and the discard would end up turning into a discard_if.
4558     * For example:
4559     * if (divergent) {
4560     *    while (...) {
4561     *       if (uniform) {
4562     *          discard;
4563     *       }
4564     *    }
4565     * }
4566     */
4567    if (!ctx->cf_info.parent_if.is_divergent) {
4568       /* program just ends here */
4569       ctx->block->kind |= block_kind_uniform;
4570       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
4571               0 /* enabled mask */, 9 /* dest */,
4572               false /* compressed */, true/* done */, true /* valid mask */);
4573       bld.sopp(aco_opcode::s_endpgm);
4574       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
4575    } else {
4576       ctx->block->kind |= block_kind_discard;
4577       /* branch and linear edge is added by visit_if() */
4578    }
4579 }
4580
4581 enum aco_descriptor_type {
4582    ACO_DESC_IMAGE,
4583    ACO_DESC_FMASK,
4584    ACO_DESC_SAMPLER,
4585    ACO_DESC_BUFFER,
4586    ACO_DESC_PLANE_0,
4587    ACO_DESC_PLANE_1,
4588    ACO_DESC_PLANE_2,
4589 };
4590
4591 static bool
4592 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
4593    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
4594       return false;
4595    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
4596    return dim == ac_image_cube ||
4597           dim == ac_image_1darray ||
4598           dim == ac_image_2darray ||
4599           dim == ac_image_2darraymsaa;
4600 }
4601
4602 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4603                       enum aco_descriptor_type desc_type,
4604                       const nir_tex_instr *tex_instr, bool image, bool write)
4605 {
4606 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4607    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4608    if (it != ctx->tex_desc.end())
4609       return it->second;
4610 */
4611    Temp index = Temp();
4612    bool index_set = false;
4613    unsigned constant_index = 0;
4614    unsigned descriptor_set;
4615    unsigned base_index;
4616    Builder bld(ctx->program, ctx->block);
4617
4618    if (!deref_instr) {
4619       assert(tex_instr && !image);
4620       descriptor_set = 0;
4621       base_index = tex_instr->sampler_index;
4622    } else {
4623       while(deref_instr->deref_type != nir_deref_type_var) {
4624          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4625          if (!array_size)
4626             array_size = 1;
4627
4628          assert(deref_instr->deref_type == nir_deref_type_array);
4629          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4630          if (const_value) {
4631             constant_index += array_size * const_value->u32;
4632          } else {
4633             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4634             if (indirect.type() == RegType::vgpr)
4635                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4636
4637             if (array_size != 1)
4638                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4639
4640             if (!index_set) {
4641                index = indirect;
4642                index_set = true;
4643             } else {
4644                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4645             }
4646          }
4647
4648          deref_instr = nir_src_as_deref(deref_instr->parent);
4649       }
4650       descriptor_set = deref_instr->var->data.descriptor_set;
4651       base_index = deref_instr->var->data.binding;
4652    }
4653
4654    Temp list = load_desc_ptr(ctx, descriptor_set);
4655    list = convert_pointer_to_64_bit(ctx, list);
4656
4657    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4658    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4659    unsigned offset = binding->offset;
4660    unsigned stride = binding->size;
4661    aco_opcode opcode;
4662    RegClass type;
4663
4664    assert(base_index < layout->binding_count);
4665
4666    switch (desc_type) {
4667    case ACO_DESC_IMAGE:
4668       type = s8;
4669       opcode = aco_opcode::s_load_dwordx8;
4670       break;
4671    case ACO_DESC_FMASK:
4672       type = s8;
4673       opcode = aco_opcode::s_load_dwordx8;
4674       offset += 32;
4675       break;
4676    case ACO_DESC_SAMPLER:
4677       type = s4;
4678       opcode = aco_opcode::s_load_dwordx4;
4679       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4680          offset += radv_combined_image_descriptor_sampler_offset(binding);
4681       break;
4682    case ACO_DESC_BUFFER:
4683       type = s4;
4684       opcode = aco_opcode::s_load_dwordx4;
4685       break;
4686    case ACO_DESC_PLANE_0:
4687    case ACO_DESC_PLANE_1:
4688       type = s8;
4689       opcode = aco_opcode::s_load_dwordx8;
4690       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4691       break;
4692    case ACO_DESC_PLANE_2:
4693       type = s4;
4694       opcode = aco_opcode::s_load_dwordx4;
4695       offset += 64;
4696       break;
4697    default:
4698       unreachable("invalid desc_type\n");
4699    }
4700
4701    offset += constant_index * stride;
4702
4703    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4704       (!index_set || binding->immutable_samplers_equal)) {
4705       if (binding->immutable_samplers_equal)
4706          constant_index = 0;
4707
4708       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4709       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4710                         Operand(samplers[constant_index * 4 + 0]),
4711                         Operand(samplers[constant_index * 4 + 1]),
4712                         Operand(samplers[constant_index * 4 + 2]),
4713                         Operand(samplers[constant_index * 4 + 3]));
4714    }
4715
4716    Operand off;
4717    if (!index_set) {
4718       off = bld.copy(bld.def(s1), Operand(offset));
4719    } else {
4720       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4721                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4722    }
4723
4724    Temp res = bld.smem(opcode, bld.def(type), list, off);
4725
4726    if (desc_type == ACO_DESC_PLANE_2) {
4727       Temp components[8];
4728       for (unsigned i = 0; i < 8; i++)
4729          components[i] = bld.tmp(s1);
4730       bld.pseudo(aco_opcode::p_split_vector,
4731                  Definition(components[0]),
4732                  Definition(components[1]),
4733                  Definition(components[2]),
4734                  Definition(components[3]),
4735                  res);
4736
4737       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4738       bld.pseudo(aco_opcode::p_split_vector,
4739                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4740                  Definition(components[4]),
4741                  Definition(components[5]),
4742                  Definition(components[6]),
4743                  Definition(components[7]),
4744                  desc2);
4745
4746       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4747                        components[0], components[1], components[2], components[3],
4748                        components[4], components[5], components[6], components[7]);
4749    }
4750
4751    return res;
4752 }
4753
4754 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4755 {
4756    switch (dim) {
4757    case GLSL_SAMPLER_DIM_BUF:
4758       return 1;
4759    case GLSL_SAMPLER_DIM_1D:
4760       return array ? 2 : 1;
4761    case GLSL_SAMPLER_DIM_2D:
4762       return array ? 3 : 2;
4763    case GLSL_SAMPLER_DIM_MS:
4764       return array ? 4 : 3;
4765    case GLSL_SAMPLER_DIM_3D:
4766    case GLSL_SAMPLER_DIM_CUBE:
4767       return 3;
4768    case GLSL_SAMPLER_DIM_RECT:
4769    case GLSL_SAMPLER_DIM_SUBPASS:
4770       return 2;
4771    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4772       return 3;
4773    default:
4774       break;
4775    }
4776    return 0;
4777 }
4778
4779
4780 /* Adjust the sample index according to FMASK.
4781  *
4782  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4783  * which is the identity mapping. Each nibble says which physical sample
4784  * should be fetched to get that sample.
4785  *
4786  * For example, 0x11111100 means there are only 2 samples stored and
4787  * the second sample covers 3/4 of the pixel. When reading samples 0
4788  * and 1, return physical sample 0 (determined by the first two 0s
4789  * in FMASK), otherwise return physical sample 1.
4790  *
4791  * The sample index should be adjusted as follows:
4792  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4793  */
4794 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vector<Temp>& coords, Operand sample_index, Temp fmask_desc_ptr)
4795 {
4796    Builder bld(ctx->program, ctx->block);
4797    Temp fmask = bld.tmp(v1);
4798    unsigned dim = ctx->options->chip_class >= GFX10
4799                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4800                   : 0;
4801
4802    Temp coord = da ? bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), coords[0], coords[1], coords[2]) :
4803                      bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), coords[0], coords[1]);
4804    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 3, 1)};
4805    load->operands[0] = Operand(fmask_desc_ptr);
4806    load->operands[1] = Operand(s4); /* no sampler */
4807    load->operands[2] = Operand(coord);
4808    load->definitions[0] = Definition(fmask);
4809    load->glc = false;
4810    load->dlc = false;
4811    load->dmask = 0x1;
4812    load->unrm = true;
4813    load->da = da;
4814    load->dim = dim;
4815    load->can_reorder = true; /* fmask images shouldn't be modified */
4816    ctx->block->instructions.emplace_back(std::move(load));
4817
4818    Operand sample_index4;
4819    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4820       sample_index4 = Operand(sample_index.constantValue() << 2);
4821    } else if (sample_index.regClass() == s1) {
4822       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4823    } else {
4824       assert(sample_index.regClass() == v1);
4825       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4826    }
4827
4828    Temp final_sample;
4829    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4830       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4831    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4832       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4833    else
4834       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4835
4836    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4837     * resource descriptor is 0 (invalid),
4838     */
4839    Temp compare = bld.tmp(bld.lm);
4840    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4841                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4842
4843    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4844
4845    /* Replace the MSAA sample index. */
4846    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4847 }
4848
4849 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4850 {
4851
4852    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4853    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4854    bool is_array = glsl_sampler_type_is_array(type);
4855    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4856    assert(!add_frag_pos && "Input attachments should be lowered.");
4857    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4858    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4859    int count = image_type_to_components_count(dim, is_array);
4860    std::vector<Temp> coords(count);
4861    Builder bld(ctx->program, ctx->block);
4862
4863    if (is_ms) {
4864       count--;
4865       Temp src2 = get_ssa_temp(ctx, instr->src[2].ssa);
4866       /* get sample index */
4867       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4868          nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4869          Operand sample_index = sample_cv ? Operand(sample_cv->u32) : Operand(emit_extract_vector(ctx, src2, 0, v1));
4870          std::vector<Temp> fmask_load_address;
4871          for (unsigned i = 0; i < (is_array ? 3 : 2); i++)
4872             fmask_load_address.emplace_back(emit_extract_vector(ctx, src0, i, v1));
4873
4874          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4875          coords[count] = adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr);
4876       } else {
4877          coords[count] = emit_extract_vector(ctx, src2, 0, v1);
4878       }
4879    }
4880
4881    if (gfx9_1d) {
4882       coords[0] = emit_extract_vector(ctx, src0, 0, v1);
4883       coords.resize(coords.size() + 1);
4884       coords[1] = bld.copy(bld.def(v1), Operand(0u));
4885       if (is_array)
4886          coords[2] = emit_extract_vector(ctx, src0, 1, v1);
4887    } else {
4888       for (int i = 0; i < count; i++)
4889          coords[i] = emit_extract_vector(ctx, src0, i, v1);
4890    }
4891
4892    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4893        instr->intrinsic == nir_intrinsic_image_deref_store) {
4894       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4895       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4896
4897       if (!level_zero)
4898          coords.emplace_back(get_ssa_temp(ctx, instr->src[lod_index].ssa));
4899    }
4900
4901    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4902    for (unsigned i = 0; i < coords.size(); i++)
4903       vec->operands[i] = Operand(coords[i]);
4904    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4905    vec->definitions[0] = Definition(res);
4906    ctx->block->instructions.emplace_back(std::move(vec));
4907    return res;
4908 }
4909
4910
4911 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4912 {
4913    Builder bld(ctx->program, ctx->block);
4914    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4915    const struct glsl_type *type = glsl_without_array(var->type);
4916    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4917    bool is_array = glsl_sampler_type_is_array(type);
4918    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4919
4920    if (dim == GLSL_SAMPLER_DIM_BUF) {
4921       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4922       unsigned num_channels = util_last_bit(mask);
4923       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4924       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4925
4926       aco_opcode opcode;
4927       switch (num_channels) {
4928       case 1:
4929          opcode = aco_opcode::buffer_load_format_x;
4930          break;
4931       case 2:
4932          opcode = aco_opcode::buffer_load_format_xy;
4933          break;
4934       case 3:
4935          opcode = aco_opcode::buffer_load_format_xyz;
4936          break;
4937       case 4:
4938          opcode = aco_opcode::buffer_load_format_xyzw;
4939          break;
4940       default:
4941          unreachable(">4 channel buffer image load");
4942       }
4943       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4944       load->operands[0] = Operand(rsrc);
4945       load->operands[1] = Operand(vindex);
4946       load->operands[2] = Operand((uint32_t) 0);
4947       Temp tmp;
4948       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4949          tmp = dst;
4950       else
4951          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4952       load->definitions[0] = Definition(tmp);
4953       load->idxen = true;
4954       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4955       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4956       load->barrier = barrier_image;
4957       ctx->block->instructions.emplace_back(std::move(load));
4958
4959       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4960       return;
4961    }
4962
4963    Temp coords = get_image_coords(ctx, instr, type);
4964    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4965
4966    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4967    unsigned num_components = util_bitcount(dmask);
4968    Temp tmp;
4969    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4970       tmp = dst;
4971    else
4972       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4973
4974    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4975    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4976
4977    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1)};
4978    load->operands[0] = Operand(resource);
4979    load->operands[1] = Operand(s4); /* no sampler */
4980    load->operands[2] = Operand(coords);
4981    load->definitions[0] = Definition(tmp);
4982    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4983    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4984    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4985    load->dmask = dmask;
4986    load->unrm = true;
4987    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4988    load->barrier = barrier_image;
4989    ctx->block->instructions.emplace_back(std::move(load));
4990
4991    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4992    return;
4993 }
4994
4995 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4996 {
4997    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4998    const struct glsl_type *type = glsl_without_array(var->type);
4999    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5000    bool is_array = glsl_sampler_type_is_array(type);
5001    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5002
5003    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
5004
5005    if (dim == GLSL_SAMPLER_DIM_BUF) {
5006       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5007       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5008       aco_opcode opcode;
5009       switch (data.size()) {
5010       case 1:
5011          opcode = aco_opcode::buffer_store_format_x;
5012          break;
5013       case 2:
5014          opcode = aco_opcode::buffer_store_format_xy;
5015          break;
5016       case 3:
5017          opcode = aco_opcode::buffer_store_format_xyz;
5018          break;
5019       case 4:
5020          opcode = aco_opcode::buffer_store_format_xyzw;
5021          break;
5022       default:
5023          unreachable(">4 channel buffer image store");
5024       }
5025       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
5026       store->operands[0] = Operand(rsrc);
5027       store->operands[1] = Operand(vindex);
5028       store->operands[2] = Operand((uint32_t) 0);
5029       store->operands[3] = Operand(data);
5030       store->idxen = true;
5031       store->glc = glc;
5032       store->dlc = false;
5033       store->disable_wqm = true;
5034       store->barrier = barrier_image;
5035       ctx->program->needs_exact = true;
5036       ctx->block->instructions.emplace_back(std::move(store));
5037       return;
5038    }
5039
5040    assert(data.type() == RegType::vgpr);
5041    Temp coords = get_image_coords(ctx, instr, type);
5042    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5043
5044    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
5045    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
5046
5047    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 0)};
5048    store->operands[0] = Operand(resource);
5049    store->operands[1] = Operand(data);
5050    store->operands[2] = Operand(coords);
5051    store->glc = glc;
5052    store->dlc = false;
5053    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5054    store->dmask = (1 << data.size()) - 1;
5055    store->unrm = true;
5056    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5057    store->disable_wqm = true;
5058    store->barrier = barrier_image;
5059    ctx->program->needs_exact = true;
5060    ctx->block->instructions.emplace_back(std::move(store));
5061    return;
5062 }
5063
5064 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5065 {
5066    /* return the previous value if dest is ever used */
5067    bool return_previous = false;
5068    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5069       return_previous = true;
5070       break;
5071    }
5072    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5073       return_previous = true;
5074       break;
5075    }
5076
5077    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5078    const struct glsl_type *type = glsl_without_array(var->type);
5079    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5080    bool is_array = glsl_sampler_type_is_array(type);
5081    Builder bld(ctx->program, ctx->block);
5082
5083    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
5084    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
5085
5086    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
5087       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
5088
5089    aco_opcode buf_op, image_op;
5090    switch (instr->intrinsic) {
5091       case nir_intrinsic_image_deref_atomic_add:
5092          buf_op = aco_opcode::buffer_atomic_add;
5093          image_op = aco_opcode::image_atomic_add;
5094          break;
5095       case nir_intrinsic_image_deref_atomic_umin:
5096          buf_op = aco_opcode::buffer_atomic_umin;
5097          image_op = aco_opcode::image_atomic_umin;
5098          break;
5099       case nir_intrinsic_image_deref_atomic_imin:
5100          buf_op = aco_opcode::buffer_atomic_smin;
5101          image_op = aco_opcode::image_atomic_smin;
5102          break;
5103       case nir_intrinsic_image_deref_atomic_umax:
5104          buf_op = aco_opcode::buffer_atomic_umax;
5105          image_op = aco_opcode::image_atomic_umax;
5106          break;
5107       case nir_intrinsic_image_deref_atomic_imax:
5108          buf_op = aco_opcode::buffer_atomic_smax;
5109          image_op = aco_opcode::image_atomic_smax;
5110          break;
5111       case nir_intrinsic_image_deref_atomic_and:
5112          buf_op = aco_opcode::buffer_atomic_and;
5113          image_op = aco_opcode::image_atomic_and;
5114          break;
5115       case nir_intrinsic_image_deref_atomic_or:
5116          buf_op = aco_opcode::buffer_atomic_or;
5117          image_op = aco_opcode::image_atomic_or;
5118          break;
5119       case nir_intrinsic_image_deref_atomic_xor:
5120          buf_op = aco_opcode::buffer_atomic_xor;
5121          image_op = aco_opcode::image_atomic_xor;
5122          break;
5123       case nir_intrinsic_image_deref_atomic_exchange:
5124          buf_op = aco_opcode::buffer_atomic_swap;
5125          image_op = aco_opcode::image_atomic_swap;
5126          break;
5127       case nir_intrinsic_image_deref_atomic_comp_swap:
5128          buf_op = aco_opcode::buffer_atomic_cmpswap;
5129          image_op = aco_opcode::image_atomic_cmpswap;
5130          break;
5131       default:
5132          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
5133    }
5134
5135    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5136
5137    if (dim == GLSL_SAMPLER_DIM_BUF) {
5138       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
5139       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
5140       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
5141       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5142       mubuf->operands[0] = Operand(resource);
5143       mubuf->operands[1] = Operand(vindex);
5144       mubuf->operands[2] = Operand((uint32_t)0);
5145       mubuf->operands[3] = Operand(data);
5146       if (return_previous)
5147          mubuf->definitions[0] = Definition(dst);
5148       mubuf->offset = 0;
5149       mubuf->idxen = true;
5150       mubuf->glc = return_previous;
5151       mubuf->dlc = false; /* Not needed for atomics */
5152       mubuf->disable_wqm = true;
5153       mubuf->barrier = barrier_image;
5154       ctx->program->needs_exact = true;
5155       ctx->block->instructions.emplace_back(std::move(mubuf));
5156       return;
5157    }
5158
5159    Temp coords = get_image_coords(ctx, instr, type);
5160    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
5161    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 3, return_previous ? 1 : 0)};
5162    mimg->operands[0] = Operand(resource);
5163    mimg->operands[1] = Operand(data);
5164    mimg->operands[2] = Operand(coords);
5165    if (return_previous)
5166       mimg->definitions[0] = Definition(dst);
5167    mimg->glc = return_previous;
5168    mimg->dlc = false; /* Not needed for atomics */
5169    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5170    mimg->dmask = (1 << data.size()) - 1;
5171    mimg->unrm = true;
5172    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
5173    mimg->disable_wqm = true;
5174    mimg->barrier = barrier_image;
5175    ctx->program->needs_exact = true;
5176    ctx->block->instructions.emplace_back(std::move(mimg));
5177    return;
5178 }
5179
5180 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
5181 {
5182    if (in_elements && ctx->options->chip_class == GFX8) {
5183       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
5184       Builder bld(ctx->program, ctx->block);
5185
5186       Temp size = emit_extract_vector(ctx, desc, 2, s1);
5187
5188       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
5189       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
5190
5191       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
5192       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
5193
5194       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
5195       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
5196
5197       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
5198       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
5199                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
5200       if (dst.type() == RegType::vgpr)
5201          bld.copy(Definition(dst), shr_dst);
5202
5203       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
5204    } else {
5205       emit_extract_vector(ctx, desc, 2, dst);
5206    }
5207 }
5208
5209 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
5210 {
5211    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
5212    const struct glsl_type *type = glsl_without_array(var->type);
5213    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
5214    bool is_array = glsl_sampler_type_is_array(type);
5215    Builder bld(ctx->program, ctx->block);
5216
5217    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
5218       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
5219       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
5220    }
5221
5222    /* LOD */
5223    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
5224
5225    /* Resource */
5226    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
5227
5228    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5229
5230    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1)};
5231    mimg->operands[0] = Operand(resource);
5232    mimg->operands[1] = Operand(s4); /* no sampler */
5233    mimg->operands[2] = Operand(lod);
5234    uint8_t& dmask = mimg->dmask;
5235    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
5236    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
5237    mimg->da = glsl_sampler_type_is_array(type);
5238    mimg->can_reorder = true;
5239    Definition& def = mimg->definitions[0];
5240    ctx->block->instructions.emplace_back(std::move(mimg));
5241
5242    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
5243        glsl_sampler_type_is_array(type)) {
5244
5245       assert(instr->dest.ssa.num_components == 3);
5246       Temp tmp = {ctx->program->allocateId(), v3};
5247       def = Definition(tmp);
5248       emit_split_vector(ctx, tmp, 3);
5249
5250       /* divide 3rd value by 6 by multiplying with magic number */
5251       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
5252       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
5253
5254       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5255                  emit_extract_vector(ctx, tmp, 0, v1),
5256                  emit_extract_vector(ctx, tmp, 1, v1),
5257                  by_6);
5258
5259    } else if (ctx->options->chip_class == GFX9 &&
5260               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
5261               glsl_sampler_type_is_array(type)) {
5262       assert(instr->dest.ssa.num_components == 2);
5263       def = Definition(dst);
5264       dmask = 0x5;
5265    } else {
5266       def = Definition(dst);
5267    }
5268
5269    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
5270 }
5271
5272 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5273 {
5274    Builder bld(ctx->program, ctx->block);
5275    unsigned num_components = instr->num_components;
5276
5277    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5278    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5279    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5280
5281    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5282    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
5283 }
5284
5285 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5286 {
5287    Builder bld(ctx->program, ctx->block);
5288    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5289    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5290    unsigned writemask = nir_intrinsic_write_mask(instr);
5291    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
5292
5293    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5294    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5295
5296    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
5297                ctx->options->chip_class >= GFX8;
5298    if (smem)
5299       offset = bld.as_uniform(offset);
5300    bool smem_nonfs = smem && ctx->stage != fragment_fs;
5301
5302    while (writemask) {
5303       int start, count;
5304       u_bit_scan_consecutive_range(&writemask, &start, &count);
5305       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
5306          /* GFX6 doesn't support storing vec3, split it. */
5307          writemask |= 1u << (start + 2);
5308          count = 2;
5309       }
5310       int num_bytes = count * elem_size_bytes;
5311
5312       if (num_bytes > 16) {
5313          assert(elem_size_bytes == 8);
5314          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5315          count = 2;
5316          num_bytes = 16;
5317       }
5318
5319       // TODO: check alignment of sub-dword stores
5320       // TODO: split 3 bytes. there is no store instruction for that
5321
5322       Temp write_data;
5323       if (count != instr->num_components) {
5324          emit_split_vector(ctx, data, instr->num_components);
5325          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5326          for (int i = 0; i < count; i++) {
5327             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
5328             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
5329          }
5330          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
5331          vec->definitions[0] = Definition(write_data);
5332          ctx->block->instructions.emplace_back(std::move(vec));
5333       } else if (!smem && data.type() != RegType::vgpr) {
5334          assert(num_bytes % 4 == 0);
5335          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
5336       } else if (smem_nonfs && data.type() == RegType::vgpr) {
5337          assert(num_bytes % 4 == 0);
5338          write_data = bld.as_uniform(data);
5339       } else {
5340          write_data = data;
5341       }
5342
5343       aco_opcode vmem_op, smem_op;
5344       switch (num_bytes) {
5345          case 4:
5346             vmem_op = aco_opcode::buffer_store_dword;
5347             smem_op = aco_opcode::s_buffer_store_dword;
5348             break;
5349          case 8:
5350             vmem_op = aco_opcode::buffer_store_dwordx2;
5351             smem_op = aco_opcode::s_buffer_store_dwordx2;
5352             break;
5353          case 12:
5354             vmem_op = aco_opcode::buffer_store_dwordx3;
5355             smem_op = aco_opcode::last_opcode;
5356             assert(!smem && ctx->options->chip_class > GFX6);
5357             break;
5358          case 16:
5359             vmem_op = aco_opcode::buffer_store_dwordx4;
5360             smem_op = aco_opcode::s_buffer_store_dwordx4;
5361             break;
5362          default:
5363             unreachable("Store SSBO not implemented for this size.");
5364       }
5365       if (ctx->stage == fragment_fs)
5366          smem_op = aco_opcode::p_fs_buffer_store_smem;
5367
5368       if (smem) {
5369          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
5370          store->operands[0] = Operand(rsrc);
5371          if (start) {
5372             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
5373                                 offset, Operand(start * elem_size_bytes));
5374             store->operands[1] = Operand(off);
5375          } else {
5376             store->operands[1] = Operand(offset);
5377          }
5378          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
5379             store->operands[1].setFixed(m0);
5380          store->operands[2] = Operand(write_data);
5381          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5382          store->dlc = false;
5383          store->disable_wqm = true;
5384          store->barrier = barrier_buffer;
5385          ctx->block->instructions.emplace_back(std::move(store));
5386          ctx->program->wb_smem_l1_on_end = true;
5387          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
5388             ctx->block->kind |= block_kind_needs_lowering;
5389             ctx->program->needs_exact = true;
5390          }
5391       } else {
5392          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
5393          store->operands[0] = Operand(rsrc);
5394          store->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5395          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5396          store->operands[3] = Operand(write_data);
5397          store->offset = start * elem_size_bytes;
5398          store->offen = (offset.type() == RegType::vgpr);
5399          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5400          store->dlc = false;
5401          store->disable_wqm = true;
5402          store->barrier = barrier_buffer;
5403          ctx->program->needs_exact = true;
5404          ctx->block->instructions.emplace_back(std::move(store));
5405       }
5406    }
5407 }
5408
5409 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
5410 {
5411    /* return the previous value if dest is ever used */
5412    bool return_previous = false;
5413    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5414       return_previous = true;
5415       break;
5416    }
5417    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5418       return_previous = true;
5419       break;
5420    }
5421
5422    Builder bld(ctx->program, ctx->block);
5423    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
5424
5425    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
5426       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5427                         get_ssa_temp(ctx, instr->src[3].ssa), data);
5428
5429    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
5430    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5431    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
5432
5433    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5434
5435    aco_opcode op32, op64;
5436    switch (instr->intrinsic) {
5437       case nir_intrinsic_ssbo_atomic_add:
5438          op32 = aco_opcode::buffer_atomic_add;
5439          op64 = aco_opcode::buffer_atomic_add_x2;
5440          break;
5441       case nir_intrinsic_ssbo_atomic_imin:
5442          op32 = aco_opcode::buffer_atomic_smin;
5443          op64 = aco_opcode::buffer_atomic_smin_x2;
5444          break;
5445       case nir_intrinsic_ssbo_atomic_umin:
5446          op32 = aco_opcode::buffer_atomic_umin;
5447          op64 = aco_opcode::buffer_atomic_umin_x2;
5448          break;
5449       case nir_intrinsic_ssbo_atomic_imax:
5450          op32 = aco_opcode::buffer_atomic_smax;
5451          op64 = aco_opcode::buffer_atomic_smax_x2;
5452          break;
5453       case nir_intrinsic_ssbo_atomic_umax:
5454          op32 = aco_opcode::buffer_atomic_umax;
5455          op64 = aco_opcode::buffer_atomic_umax_x2;
5456          break;
5457       case nir_intrinsic_ssbo_atomic_and:
5458          op32 = aco_opcode::buffer_atomic_and;
5459          op64 = aco_opcode::buffer_atomic_and_x2;
5460          break;
5461       case nir_intrinsic_ssbo_atomic_or:
5462          op32 = aco_opcode::buffer_atomic_or;
5463          op64 = aco_opcode::buffer_atomic_or_x2;
5464          break;
5465       case nir_intrinsic_ssbo_atomic_xor:
5466          op32 = aco_opcode::buffer_atomic_xor;
5467          op64 = aco_opcode::buffer_atomic_xor_x2;
5468          break;
5469       case nir_intrinsic_ssbo_atomic_exchange:
5470          op32 = aco_opcode::buffer_atomic_swap;
5471          op64 = aco_opcode::buffer_atomic_swap_x2;
5472          break;
5473       case nir_intrinsic_ssbo_atomic_comp_swap:
5474          op32 = aco_opcode::buffer_atomic_cmpswap;
5475          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5476          break;
5477       default:
5478          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
5479    }
5480    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5481    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5482    mubuf->operands[0] = Operand(rsrc);
5483    mubuf->operands[1] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
5484    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
5485    mubuf->operands[3] = Operand(data);
5486    if (return_previous)
5487       mubuf->definitions[0] = Definition(dst);
5488    mubuf->offset = 0;
5489    mubuf->offen = (offset.type() == RegType::vgpr);
5490    mubuf->glc = return_previous;
5491    mubuf->dlc = false; /* Not needed for atomics */
5492    mubuf->disable_wqm = true;
5493    mubuf->barrier = barrier_buffer;
5494    ctx->program->needs_exact = true;
5495    ctx->block->instructions.emplace_back(std::move(mubuf));
5496 }
5497
5498 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
5499
5500    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5501    Builder bld(ctx->program, ctx->block);
5502    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
5503    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
5504 }
5505
5506 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
5507 {
5508    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5509                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5510
5511    if (addr.type() == RegType::vgpr)
5512       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
5513    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
5514 }
5515
5516 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
5517 {
5518    Builder bld(ctx->program, ctx->block);
5519    unsigned num_components = instr->num_components;
5520    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
5521
5522    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5523    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5524
5525    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
5526    bool dlc = glc && ctx->options->chip_class >= GFX10;
5527    aco_opcode op;
5528    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
5529       bool global = ctx->options->chip_class >= GFX9;
5530
5531       if (ctx->options->chip_class >= GFX7) {
5532          aco_opcode op;
5533          switch (num_bytes) {
5534          case 4:
5535             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
5536             break;
5537          case 8:
5538             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
5539             break;
5540          case 12:
5541             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
5542             break;
5543          case 16:
5544             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
5545             break;
5546          default:
5547             unreachable("load_global not implemented for this size.");
5548          }
5549
5550          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
5551          flat->operands[0] = Operand(addr);
5552          flat->operands[1] = Operand(s1);
5553          flat->glc = glc;
5554          flat->dlc = dlc;
5555          flat->barrier = barrier_buffer;
5556
5557          if (dst.type() == RegType::sgpr) {
5558             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5559             flat->definitions[0] = Definition(vec);
5560             ctx->block->instructions.emplace_back(std::move(flat));
5561             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5562          } else {
5563             flat->definitions[0] = Definition(dst);
5564             ctx->block->instructions.emplace_back(std::move(flat));
5565          }
5566          emit_split_vector(ctx, dst, num_components);
5567       } else {
5568          assert(ctx->options->chip_class == GFX6);
5569
5570          /* GFX6 doesn't support loading vec3, expand to vec4. */
5571          num_bytes = num_bytes == 12 ? 16 : num_bytes;
5572
5573          aco_opcode op;
5574          switch (num_bytes) {
5575          case 4:
5576             op = aco_opcode::buffer_load_dword;
5577             break;
5578          case 8:
5579             op = aco_opcode::buffer_load_dwordx2;
5580             break;
5581          case 16:
5582             op = aco_opcode::buffer_load_dwordx4;
5583             break;
5584          default:
5585             unreachable("load_global not implemented for this size.");
5586          }
5587
5588          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5589
5590          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
5591          mubuf->operands[0] = Operand(rsrc);
5592          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5593          mubuf->operands[2] = Operand(0u);
5594          mubuf->glc = glc;
5595          mubuf->dlc = false;
5596          mubuf->offset = 0;
5597          mubuf->addr64 = addr.type() == RegType::vgpr;
5598          mubuf->disable_wqm = false;
5599          mubuf->barrier = barrier_buffer;
5600          aco_ptr<Instruction> instr = std::move(mubuf);
5601
5602          /* expand vector */
5603          if (dst.size() == 3) {
5604             Temp vec = bld.tmp(v4);
5605             instr->definitions[0] = Definition(vec);
5606             bld.insert(std::move(instr));
5607             emit_split_vector(ctx, vec, 4);
5608
5609             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5610             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5611             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5612             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5613          }
5614
5615          if (dst.type() == RegType::sgpr) {
5616             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5617             instr->definitions[0] = Definition(vec);
5618             bld.insert(std::move(instr));
5619             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5620             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5621          } else {
5622             instr->definitions[0] = Definition(dst);
5623             bld.insert(std::move(instr));
5624             emit_split_vector(ctx, dst, num_components);
5625          }
5626       }
5627    } else {
5628       switch (num_bytes) {
5629          case 4:
5630             op = aco_opcode::s_load_dword;
5631             break;
5632          case 8:
5633             op = aco_opcode::s_load_dwordx2;
5634             break;
5635          case 12:
5636          case 16:
5637             op = aco_opcode::s_load_dwordx4;
5638             break;
5639          default:
5640             unreachable("load_global not implemented for this size.");
5641       }
5642       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5643       load->operands[0] = Operand(addr);
5644       load->operands[1] = Operand(0u);
5645       load->definitions[0] = Definition(dst);
5646       load->glc = glc;
5647       load->dlc = dlc;
5648       load->barrier = barrier_buffer;
5649       assert(ctx->options->chip_class >= GFX8 || !glc);
5650
5651       if (dst.size() == 3) {
5652          /* trim vector */
5653          Temp vec = bld.tmp(s4);
5654          load->definitions[0] = Definition(vec);
5655          ctx->block->instructions.emplace_back(std::move(load));
5656          emit_split_vector(ctx, vec, 4);
5657
5658          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5659                     emit_extract_vector(ctx, vec, 0, s1),
5660                     emit_extract_vector(ctx, vec, 1, s1),
5661                     emit_extract_vector(ctx, vec, 2, s1));
5662       } else {
5663          ctx->block->instructions.emplace_back(std::move(load));
5664       }
5665    }
5666 }
5667
5668 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5669 {
5670    Builder bld(ctx->program, ctx->block);
5671    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5672
5673    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5674    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5675
5676    if (ctx->options->chip_class >= GFX7)
5677       addr = as_vgpr(ctx, addr);
5678
5679    unsigned writemask = nir_intrinsic_write_mask(instr);
5680    while (writemask) {
5681       int start, count;
5682       u_bit_scan_consecutive_range(&writemask, &start, &count);
5683       if (count == 3 && ctx->options->chip_class == GFX6) {
5684          /* GFX6 doesn't support storing vec3, split it. */
5685          writemask |= 1u << (start + 2);
5686          count = 2;
5687       }
5688       unsigned num_bytes = count * elem_size_bytes;
5689
5690       Temp write_data = data;
5691       if (count != instr->num_components) {
5692          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5693          for (int i = 0; i < count; i++)
5694             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5695          write_data = bld.tmp(RegType::vgpr, count);
5696          vec->definitions[0] = Definition(write_data);
5697          ctx->block->instructions.emplace_back(std::move(vec));
5698       }
5699
5700       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5701       unsigned offset = start * elem_size_bytes;
5702
5703       if (ctx->options->chip_class >= GFX7) {
5704          if (offset > 0 && ctx->options->chip_class < GFX9) {
5705             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5706             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5707             Temp carry = bld.tmp(bld.lm);
5708             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5709
5710             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5711                      Operand(offset), addr0);
5712             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5713                      Operand(0u), addr1,
5714                      carry).def(1).setHint(vcc);
5715
5716             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5717
5718             offset = 0;
5719          }
5720
5721          bool global = ctx->options->chip_class >= GFX9;
5722          aco_opcode op;
5723          switch (num_bytes) {
5724          case 4:
5725             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5726             break;
5727          case 8:
5728             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5729             break;
5730          case 12:
5731             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5732             break;
5733          case 16:
5734             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5735             break;
5736          default:
5737             unreachable("store_global not implemented for this size.");
5738          }
5739
5740          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5741          flat->operands[0] = Operand(addr);
5742          flat->operands[1] = Operand(s1);
5743          flat->operands[2] = Operand(data);
5744          flat->glc = glc;
5745          flat->dlc = false;
5746          flat->offset = offset;
5747          flat->disable_wqm = true;
5748          flat->barrier = barrier_buffer;
5749          ctx->program->needs_exact = true;
5750          ctx->block->instructions.emplace_back(std::move(flat));
5751       } else {
5752          assert(ctx->options->chip_class == GFX6);
5753
5754          aco_opcode op;
5755          switch (num_bytes) {
5756          case 4:
5757             op = aco_opcode::buffer_store_dword;
5758             break;
5759          case 8:
5760             op = aco_opcode::buffer_store_dwordx2;
5761             break;
5762          case 16:
5763             op = aco_opcode::buffer_store_dwordx4;
5764             break;
5765          default:
5766             unreachable("store_global not implemented for this size.");
5767          }
5768
5769          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5770
5771          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5772          mubuf->operands[0] = Operand(rsrc);
5773          mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5774          mubuf->operands[2] = Operand(0u);
5775          mubuf->operands[3] = Operand(write_data);
5776          mubuf->glc = glc;
5777          mubuf->dlc = false;
5778          mubuf->offset = offset;
5779          mubuf->addr64 = addr.type() == RegType::vgpr;
5780          mubuf->disable_wqm = true;
5781          mubuf->barrier = barrier_buffer;
5782          ctx->program->needs_exact = true;
5783          ctx->block->instructions.emplace_back(std::move(mubuf));
5784       }
5785    }
5786 }
5787
5788 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5789 {
5790    /* return the previous value if dest is ever used */
5791    bool return_previous = false;
5792    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5793       return_previous = true;
5794       break;
5795    }
5796    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5797       return_previous = true;
5798       break;
5799    }
5800
5801    Builder bld(ctx->program, ctx->block);
5802    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5803    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5804
5805    if (ctx->options->chip_class >= GFX7)
5806       addr = as_vgpr(ctx, addr);
5807
5808    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5809       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5810                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5811
5812    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5813
5814    aco_opcode op32, op64;
5815
5816    if (ctx->options->chip_class >= GFX7) {
5817       bool global = ctx->options->chip_class >= GFX9;
5818       switch (instr->intrinsic) {
5819          case nir_intrinsic_global_atomic_add:
5820             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5821             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5822             break;
5823          case nir_intrinsic_global_atomic_imin:
5824             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5825             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5826             break;
5827          case nir_intrinsic_global_atomic_umin:
5828             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5829             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5830             break;
5831          case nir_intrinsic_global_atomic_imax:
5832             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5833             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5834             break;
5835          case nir_intrinsic_global_atomic_umax:
5836             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5837             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5838             break;
5839          case nir_intrinsic_global_atomic_and:
5840             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5841             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5842             break;
5843          case nir_intrinsic_global_atomic_or:
5844             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5845             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5846             break;
5847          case nir_intrinsic_global_atomic_xor:
5848             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5849             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5850             break;
5851          case nir_intrinsic_global_atomic_exchange:
5852             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5853             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5854             break;
5855          case nir_intrinsic_global_atomic_comp_swap:
5856             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5857             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5858             break;
5859          default:
5860             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5861       }
5862
5863       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5864       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5865       flat->operands[0] = Operand(addr);
5866       flat->operands[1] = Operand(s1);
5867       flat->operands[2] = Operand(data);
5868       if (return_previous)
5869          flat->definitions[0] = Definition(dst);
5870       flat->glc = return_previous;
5871       flat->dlc = false; /* Not needed for atomics */
5872       flat->offset = 0;
5873       flat->disable_wqm = true;
5874       flat->barrier = barrier_buffer;
5875       ctx->program->needs_exact = true;
5876       ctx->block->instructions.emplace_back(std::move(flat));
5877    } else {
5878       assert(ctx->options->chip_class == GFX6);
5879
5880       switch (instr->intrinsic) {
5881          case nir_intrinsic_global_atomic_add:
5882             op32 = aco_opcode::buffer_atomic_add;
5883             op64 = aco_opcode::buffer_atomic_add_x2;
5884             break;
5885          case nir_intrinsic_global_atomic_imin:
5886             op32 = aco_opcode::buffer_atomic_smin;
5887             op64 = aco_opcode::buffer_atomic_smin_x2;
5888             break;
5889          case nir_intrinsic_global_atomic_umin:
5890             op32 = aco_opcode::buffer_atomic_umin;
5891             op64 = aco_opcode::buffer_atomic_umin_x2;
5892             break;
5893          case nir_intrinsic_global_atomic_imax:
5894             op32 = aco_opcode::buffer_atomic_smax;
5895             op64 = aco_opcode::buffer_atomic_smax_x2;
5896             break;
5897          case nir_intrinsic_global_atomic_umax:
5898             op32 = aco_opcode::buffer_atomic_umax;
5899             op64 = aco_opcode::buffer_atomic_umax_x2;
5900             break;
5901          case nir_intrinsic_global_atomic_and:
5902             op32 = aco_opcode::buffer_atomic_and;
5903             op64 = aco_opcode::buffer_atomic_and_x2;
5904             break;
5905          case nir_intrinsic_global_atomic_or:
5906             op32 = aco_opcode::buffer_atomic_or;
5907             op64 = aco_opcode::buffer_atomic_or_x2;
5908             break;
5909          case nir_intrinsic_global_atomic_xor:
5910             op32 = aco_opcode::buffer_atomic_xor;
5911             op64 = aco_opcode::buffer_atomic_xor_x2;
5912             break;
5913          case nir_intrinsic_global_atomic_exchange:
5914             op32 = aco_opcode::buffer_atomic_swap;
5915             op64 = aco_opcode::buffer_atomic_swap_x2;
5916             break;
5917          case nir_intrinsic_global_atomic_comp_swap:
5918             op32 = aco_opcode::buffer_atomic_cmpswap;
5919             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5920             break;
5921          default:
5922             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5923       }
5924
5925       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5926
5927       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5928
5929       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5930       mubuf->operands[0] = Operand(rsrc);
5931       mubuf->operands[1] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5932       mubuf->operands[2] = Operand(0u);
5933       mubuf->operands[3] = Operand(data);
5934       if (return_previous)
5935          mubuf->definitions[0] = Definition(dst);
5936       mubuf->glc = return_previous;
5937       mubuf->dlc = false;
5938       mubuf->offset = 0;
5939       mubuf->addr64 = addr.type() == RegType::vgpr;
5940       mubuf->disable_wqm = true;
5941       mubuf->barrier = barrier_buffer;
5942       ctx->program->needs_exact = true;
5943       ctx->block->instructions.emplace_back(std::move(mubuf));
5944    }
5945 }
5946
5947 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5948    Builder bld(ctx->program, ctx->block);
5949    switch(instr->intrinsic) {
5950       case nir_intrinsic_group_memory_barrier:
5951       case nir_intrinsic_memory_barrier:
5952          bld.barrier(aco_opcode::p_memory_barrier_common);
5953          break;
5954       case nir_intrinsic_memory_barrier_buffer:
5955          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5956          break;
5957       case nir_intrinsic_memory_barrier_image:
5958          bld.barrier(aco_opcode::p_memory_barrier_image);
5959          break;
5960       case nir_intrinsic_memory_barrier_tcs_patch:
5961       case nir_intrinsic_memory_barrier_shared:
5962          bld.barrier(aco_opcode::p_memory_barrier_shared);
5963          break;
5964       default:
5965          unreachable("Unimplemented memory barrier intrinsic");
5966          break;
5967    }
5968 }
5969
5970 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5971 {
5972    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5973    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5974    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5975    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5976    Builder bld(ctx->program, ctx->block);
5977
5978    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5979    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5980    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5981 }
5982
5983 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5984 {
5985    unsigned writemask = nir_intrinsic_write_mask(instr);
5986    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5987    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5988    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5989    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5990
5991    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5992    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5993 }
5994
5995 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5996 {
5997    unsigned offset = nir_intrinsic_base(instr);
5998    Operand m = load_lds_size_m0(ctx);
5999    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6000    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6001
6002    unsigned num_operands = 3;
6003    aco_opcode op32, op64, op32_rtn, op64_rtn;
6004    switch(instr->intrinsic) {
6005       case nir_intrinsic_shared_atomic_add:
6006          op32 = aco_opcode::ds_add_u32;
6007          op64 = aco_opcode::ds_add_u64;
6008          op32_rtn = aco_opcode::ds_add_rtn_u32;
6009          op64_rtn = aco_opcode::ds_add_rtn_u64;
6010          break;
6011       case nir_intrinsic_shared_atomic_imin:
6012          op32 = aco_opcode::ds_min_i32;
6013          op64 = aco_opcode::ds_min_i64;
6014          op32_rtn = aco_opcode::ds_min_rtn_i32;
6015          op64_rtn = aco_opcode::ds_min_rtn_i64;
6016          break;
6017       case nir_intrinsic_shared_atomic_umin:
6018          op32 = aco_opcode::ds_min_u32;
6019          op64 = aco_opcode::ds_min_u64;
6020          op32_rtn = aco_opcode::ds_min_rtn_u32;
6021          op64_rtn = aco_opcode::ds_min_rtn_u64;
6022          break;
6023       case nir_intrinsic_shared_atomic_imax:
6024          op32 = aco_opcode::ds_max_i32;
6025          op64 = aco_opcode::ds_max_i64;
6026          op32_rtn = aco_opcode::ds_max_rtn_i32;
6027          op64_rtn = aco_opcode::ds_max_rtn_i64;
6028          break;
6029       case nir_intrinsic_shared_atomic_umax:
6030          op32 = aco_opcode::ds_max_u32;
6031          op64 = aco_opcode::ds_max_u64;
6032          op32_rtn = aco_opcode::ds_max_rtn_u32;
6033          op64_rtn = aco_opcode::ds_max_rtn_u64;
6034          break;
6035       case nir_intrinsic_shared_atomic_and:
6036          op32 = aco_opcode::ds_and_b32;
6037          op64 = aco_opcode::ds_and_b64;
6038          op32_rtn = aco_opcode::ds_and_rtn_b32;
6039          op64_rtn = aco_opcode::ds_and_rtn_b64;
6040          break;
6041       case nir_intrinsic_shared_atomic_or:
6042          op32 = aco_opcode::ds_or_b32;
6043          op64 = aco_opcode::ds_or_b64;
6044          op32_rtn = aco_opcode::ds_or_rtn_b32;
6045          op64_rtn = aco_opcode::ds_or_rtn_b64;
6046          break;
6047       case nir_intrinsic_shared_atomic_xor:
6048          op32 = aco_opcode::ds_xor_b32;
6049          op64 = aco_opcode::ds_xor_b64;
6050          op32_rtn = aco_opcode::ds_xor_rtn_b32;
6051          op64_rtn = aco_opcode::ds_xor_rtn_b64;
6052          break;
6053       case nir_intrinsic_shared_atomic_exchange:
6054          op32 = aco_opcode::ds_write_b32;
6055          op64 = aco_opcode::ds_write_b64;
6056          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
6057          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
6058          break;
6059       case nir_intrinsic_shared_atomic_comp_swap:
6060          op32 = aco_opcode::ds_cmpst_b32;
6061          op64 = aco_opcode::ds_cmpst_b64;
6062          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
6063          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
6064          num_operands = 4;
6065          break;
6066       default:
6067          unreachable("Unhandled shared atomic intrinsic");
6068    }
6069
6070    /* return the previous value if dest is ever used */
6071    bool return_previous = false;
6072    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
6073       return_previous = true;
6074       break;
6075    }
6076    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
6077       return_previous = true;
6078       break;
6079    }
6080
6081    aco_opcode op;
6082    if (data.size() == 1) {
6083       assert(instr->dest.ssa.bit_size == 32);
6084       op = return_previous ? op32_rtn : op32;
6085    } else {
6086       assert(instr->dest.ssa.bit_size == 64);
6087       op = return_previous ? op64_rtn : op64;
6088    }
6089
6090    if (offset > 65535) {
6091       Builder bld(ctx->program, ctx->block);
6092       address = bld.vadd32(bld.def(v1), Operand(offset), address);
6093       offset = 0;
6094    }
6095
6096    aco_ptr<DS_instruction> ds;
6097    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
6098    ds->operands[0] = Operand(address);
6099    ds->operands[1] = Operand(data);
6100    if (num_operands == 4)
6101       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
6102    ds->operands[num_operands - 1] = m;
6103    ds->offset0 = offset;
6104    if (return_previous)
6105       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
6106    ctx->block->instructions.emplace_back(std::move(ds));
6107 }
6108
6109 Temp get_scratch_resource(isel_context *ctx)
6110 {
6111    Builder bld(ctx->program, ctx->block);
6112    Temp scratch_addr = ctx->program->private_segment_buffer;
6113    if (ctx->stage != compute_cs)
6114       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
6115
6116    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
6117                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
6118
6119    if (ctx->program->chip_class >= GFX10) {
6120       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
6121                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
6122                    S_008F0C_RESOURCE_LEVEL(1);
6123    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
6124       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6125                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6126    }
6127
6128    /* older generations need element size = 16 bytes. element size removed in GFX9 */
6129    if (ctx->program->chip_class <= GFX8)
6130       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
6131
6132    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
6133 }
6134
6135 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6136    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
6137    Builder bld(ctx->program, ctx->block);
6138    Temp rsrc = get_scratch_resource(ctx);
6139    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6140    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6141
6142    aco_opcode op;
6143    switch (dst.size()) {
6144       case 1:
6145          op = aco_opcode::buffer_load_dword;
6146          break;
6147       case 2:
6148          op = aco_opcode::buffer_load_dwordx2;
6149          break;
6150       case 3:
6151          op = aco_opcode::buffer_load_dwordx3;
6152          break;
6153       case 4:
6154          op = aco_opcode::buffer_load_dwordx4;
6155          break;
6156       case 6:
6157       case 8: {
6158          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
6159          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
6160                                 bld.def(v4), rsrc, offset,
6161                                 ctx->program->scratch_offset, 0, true);
6162          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
6163                                                   aco_opcode::buffer_load_dwordx4,
6164                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
6165                                 rsrc, offset, ctx->program->scratch_offset, 16, true);
6166          emit_split_vector(ctx, lower, 2);
6167          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
6168          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
6169          if (dst.size() == 8) {
6170             emit_split_vector(ctx, upper, 2);
6171             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
6172             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
6173          } else {
6174             elems[2] = upper;
6175          }
6176
6177          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6178                                                                          Format::PSEUDO, dst.size() / 2, 1)};
6179          for (unsigned i = 0; i < dst.size() / 2; i++)
6180             vec->operands[i] = Operand(elems[i]);
6181          vec->definitions[0] = Definition(dst);
6182          bld.insert(std::move(vec));
6183          ctx->allocated_vec.emplace(dst.id(), elems);
6184          return;
6185       }
6186       default:
6187          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
6188    }
6189
6190    bld.mubuf(op, Definition(dst), rsrc, offset, ctx->program->scratch_offset, 0, true);
6191    emit_split_vector(ctx, dst, instr->num_components);
6192 }
6193
6194 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
6195    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
6196    Builder bld(ctx->program, ctx->block);
6197    Temp rsrc = get_scratch_resource(ctx);
6198    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6199    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
6200
6201    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
6202    unsigned writemask = nir_intrinsic_write_mask(instr);
6203
6204    while (writemask) {
6205       int start, count;
6206       u_bit_scan_consecutive_range(&writemask, &start, &count);
6207       int num_bytes = count * elem_size_bytes;
6208
6209       if (num_bytes > 16) {
6210          assert(elem_size_bytes == 8);
6211          writemask |= (((count - 2) << 1) - 1) << (start + 2);
6212          count = 2;
6213          num_bytes = 16;
6214       }
6215
6216       // TODO: check alignment of sub-dword stores
6217       // TODO: split 3 bytes. there is no store instruction for that
6218
6219       Temp write_data;
6220       if (count != instr->num_components) {
6221          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
6222          for (int i = 0; i < count; i++) {
6223             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
6224             vec->operands[i] = Operand(elem);
6225          }
6226          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
6227          vec->definitions[0] = Definition(write_data);
6228          ctx->block->instructions.emplace_back(std::move(vec));
6229       } else {
6230          write_data = data;
6231       }
6232
6233       aco_opcode op;
6234       switch (num_bytes) {
6235          case 4:
6236             op = aco_opcode::buffer_store_dword;
6237             break;
6238          case 8:
6239             op = aco_opcode::buffer_store_dwordx2;
6240             break;
6241          case 12:
6242             op = aco_opcode::buffer_store_dwordx3;
6243             break;
6244          case 16:
6245             op = aco_opcode::buffer_store_dwordx4;
6246             break;
6247          default:
6248             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
6249       }
6250
6251       bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
6252    }
6253 }
6254
6255 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
6256    uint8_t log2_ps_iter_samples;
6257    if (ctx->program->info->ps.force_persample) {
6258       log2_ps_iter_samples =
6259          util_logbase2(ctx->options->key.fs.num_samples);
6260    } else {
6261       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
6262    }
6263
6264    /* The bit pattern matches that used by fixed function fragment
6265     * processing. */
6266    static const unsigned ps_iter_masks[] = {
6267       0xffff, /* not used */
6268       0x5555,
6269       0x1111,
6270       0x0101,
6271       0x0001,
6272    };
6273    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
6274
6275    Builder bld(ctx->program, ctx->block);
6276
6277    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
6278                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6279    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
6280    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
6281    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6282    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
6283 }
6284
6285 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
6286    Builder bld(ctx->program, ctx->block);
6287
6288    unsigned stream = nir_intrinsic_stream_id(instr);
6289    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6290    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
6291    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
6292
6293    /* get GSVS ring */
6294    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
6295
6296    unsigned num_components =
6297       ctx->program->info->gs.num_stream_output_components[stream];
6298    assert(num_components);
6299
6300    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
6301    unsigned stream_offset = 0;
6302    for (unsigned i = 0; i < stream; i++) {
6303       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
6304       stream_offset += prev_stride * ctx->program->wave_size;
6305    }
6306
6307    /* Limit on the stride field for <= GFX7. */
6308    assert(stride < (1 << 14));
6309
6310    Temp gsvs_dwords[4];
6311    for (unsigned i = 0; i < 4; i++)
6312       gsvs_dwords[i] = bld.tmp(s1);
6313    bld.pseudo(aco_opcode::p_split_vector,
6314               Definition(gsvs_dwords[0]),
6315               Definition(gsvs_dwords[1]),
6316               Definition(gsvs_dwords[2]),
6317               Definition(gsvs_dwords[3]),
6318               gsvs_ring);
6319
6320    if (stream_offset) {
6321       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
6322
6323       Temp carry = bld.tmp(s1);
6324       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
6325       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
6326    }
6327
6328    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
6329    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
6330
6331    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6332                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
6333
6334    unsigned offset = 0;
6335    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
6336       if (ctx->program->info->gs.output_streams[i] != stream)
6337          continue;
6338
6339       for (unsigned j = 0; j < 4; j++) {
6340          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
6341             continue;
6342
6343          if (ctx->outputs.mask[i] & (1 << j)) {
6344             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
6345             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
6346             if (const_offset >= 4096u) {
6347                if (vaddr_offset.isUndefined())
6348                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
6349                else
6350                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
6351                const_offset %= 4096u;
6352             }
6353
6354             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
6355             mtbuf->operands[0] = Operand(gsvs_ring);
6356             mtbuf->operands[1] = vaddr_offset;
6357             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
6358             mtbuf->operands[3] = Operand(ctx->outputs.temps[i * 4u + j]);
6359             mtbuf->offen = !vaddr_offset.isUndefined();
6360             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
6361             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
6362             mtbuf->offset = const_offset;
6363             mtbuf->glc = true;
6364             mtbuf->slc = true;
6365             mtbuf->barrier = barrier_gs_data;
6366             mtbuf->can_reorder = true;
6367             bld.insert(std::move(mtbuf));
6368          }
6369
6370          offset += ctx->shader->info.gs.vertices_out;
6371       }
6372
6373       /* outputs for the next vertex are undefined and keeping them around can
6374        * create invalid IR with control flow */
6375       ctx->outputs.mask[i] = 0;
6376    }
6377
6378    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
6379 }
6380
6381 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
6382 {
6383    Builder bld(ctx->program, ctx->block);
6384
6385    if (cluster_size == 1) {
6386       return src;
6387    } if (op == nir_op_iand && cluster_size == 4) {
6388       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
6389       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6390       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
6391                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
6392    } else if (op == nir_op_ior && cluster_size == 4) {
6393       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
6394       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
6395                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
6396    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
6397       //subgroupAnd(val) -> (exec & ~val) == 0
6398       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6399       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6400       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
6401    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
6402       //subgroupOr(val) -> (val & exec) != 0
6403       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
6404       return bool_to_vector_condition(ctx, tmp);
6405    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
6406       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
6407       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6408       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
6409       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
6410       return bool_to_vector_condition(ctx, tmp);
6411    } else {
6412       //subgroupClustered{And,Or,Xor}(val, n) ->
6413       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
6414       //cluster_offset = ~(n - 1) & lane_id
6415       //cluster_mask = ((1 << n) - 1)
6416       //subgroupClusteredAnd():
6417       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
6418       //subgroupClusteredOr():
6419       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
6420       //subgroupClusteredXor():
6421       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
6422       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
6423       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
6424
6425       Temp tmp;
6426       if (op == nir_op_iand)
6427          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6428       else
6429          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6430
6431       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
6432
6433       if (ctx->program->chip_class <= GFX7)
6434          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
6435       else if (ctx->program->wave_size == 64)
6436          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
6437       else
6438          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
6439       tmp = emit_extract_vector(ctx, tmp, 0, v1);
6440       if (cluster_mask != 0xffffffff)
6441          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
6442
6443       Definition cmp_def = Definition();
6444       if (op == nir_op_iand) {
6445          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
6446       } else if (op == nir_op_ior) {
6447          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6448       } else if (op == nir_op_ixor) {
6449          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
6450                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
6451          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
6452       }
6453       cmp_def.setHint(vcc);
6454       return cmp_def.getTemp();
6455    }
6456 }
6457
6458 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
6459 {
6460    Builder bld(ctx->program, ctx->block);
6461
6462    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
6463    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
6464    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
6465    Temp tmp;
6466    if (op == nir_op_iand)
6467       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
6468    else
6469       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
6470
6471    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
6472    Temp lo = lohi.def(0).getTemp();
6473    Temp hi = lohi.def(1).getTemp();
6474    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
6475
6476    Definition cmp_def = Definition();
6477    if (op == nir_op_iand)
6478       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6479    else if (op == nir_op_ior)
6480       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
6481    else if (op == nir_op_ixor)
6482       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
6483                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
6484    cmp_def.setHint(vcc);
6485    return cmp_def.getTemp();
6486 }
6487
6488 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
6489 {
6490    Builder bld(ctx->program, ctx->block);
6491
6492    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
6493    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
6494    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
6495    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
6496    if (op == nir_op_iand)
6497       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6498    else if (op == nir_op_ior)
6499       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6500    else if (op == nir_op_ixor)
6501       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
6502
6503    assert(false);
6504    return Temp();
6505 }
6506
6507 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
6508 {
6509    Builder bld(ctx->program, ctx->block);
6510    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
6511    if (src.regClass().type() == RegType::vgpr) {
6512       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
6513    } else if (src.regClass() == s1) {
6514       bld.sop1(aco_opcode::s_mov_b32, dst, src);
6515    } else if (src.regClass() == s2) {
6516       bld.sop1(aco_opcode::s_mov_b64, dst, src);
6517    } else {
6518       fprintf(stderr, "Unimplemented NIR instr bit size: ");
6519       nir_print_instr(&instr->instr, stderr);
6520       fprintf(stderr, "\n");
6521    }
6522 }
6523
6524 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
6525 {
6526    Builder bld(ctx->program, ctx->block);
6527    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
6528    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
6529    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
6530
6531    Temp ddx_1, ddx_2, ddy_1, ddy_2;
6532    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
6533    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
6534    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
6535
6536    /* Build DD X/Y */
6537    if (ctx->program->chip_class >= GFX8) {
6538       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
6539       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
6540       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
6541       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
6542       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
6543       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
6544    } else {
6545       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
6546       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
6547       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
6548       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
6549       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
6550       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
6551       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
6552       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
6553       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
6554       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
6555    }
6556
6557    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
6558    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
6559    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
6560    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
6561    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
6562    Temp wqm1 = bld.tmp(v1);
6563    emit_wqm(ctx, tmp1, wqm1, true);
6564    Temp wqm2 = bld.tmp(v1);
6565    emit_wqm(ctx, tmp2, wqm2, true);
6566    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
6567    return;
6568 }
6569
6570 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
6571 {
6572    Builder bld(ctx->program, ctx->block);
6573    switch(instr->intrinsic) {
6574    case nir_intrinsic_load_barycentric_sample:
6575    case nir_intrinsic_load_barycentric_pixel:
6576    case nir_intrinsic_load_barycentric_centroid: {
6577       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
6578       Temp bary = Temp(0, s2);
6579       switch (mode) {
6580       case INTERP_MODE_SMOOTH:
6581       case INTERP_MODE_NONE:
6582          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6583             bary = get_arg(ctx, ctx->args->ac.persp_center);
6584          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6585             bary = ctx->persp_centroid;
6586          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6587             bary = get_arg(ctx, ctx->args->ac.persp_sample);
6588          break;
6589       case INTERP_MODE_NOPERSPECTIVE:
6590          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6591             bary = get_arg(ctx, ctx->args->ac.linear_center);
6592          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6593             bary = ctx->linear_centroid;
6594          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6595             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6596          break;
6597       default:
6598          break;
6599       }
6600       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6601       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6602       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6603       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6604                  Operand(p1), Operand(p2));
6605       emit_split_vector(ctx, dst, 2);
6606       break;
6607    }
6608    case nir_intrinsic_load_barycentric_model: {
6609       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
6610
6611       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6612       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
6613       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
6614       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
6615       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6616                  Operand(p1), Operand(p2), Operand(p3));
6617       emit_split_vector(ctx, dst, 3);
6618       break;
6619    }
6620    case nir_intrinsic_load_barycentric_at_sample: {
6621       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6622       switch (ctx->options->key.fs.num_samples) {
6623          case 2: sample_pos_offset += 1 << 3; break;
6624          case 4: sample_pos_offset += 3 << 3; break;
6625          case 8: sample_pos_offset += 7 << 3; break;
6626          default: break;
6627       }
6628       Temp sample_pos;
6629       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6630       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6631       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6632       if (addr.type() == RegType::sgpr) {
6633          Operand offset;
6634          if (const_addr) {
6635             sample_pos_offset += const_addr->u32 << 3;
6636             offset = Operand(sample_pos_offset);
6637          } else if (ctx->options->chip_class >= GFX9) {
6638             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6639          } else {
6640             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6641             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6642          }
6643
6644          Operand off = bld.copy(bld.def(s1), Operand(offset));
6645          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6646
6647       } else if (ctx->options->chip_class >= GFX9) {
6648          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6649          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6650       } else if (ctx->options->chip_class >= GFX7) {
6651          /* addr += private_segment_buffer + sample_pos_offset */
6652          Temp tmp0 = bld.tmp(s1);
6653          Temp tmp1 = bld.tmp(s1);
6654          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6655          Definition scc_tmp = bld.def(s1, scc);
6656          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6657          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6658          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6659          Temp pck0 = bld.tmp(v1);
6660          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6661          tmp1 = as_vgpr(ctx, tmp1);
6662          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6663          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6664
6665          /* sample_pos = flat_load_dwordx2 addr */
6666          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6667       } else {
6668          assert(ctx->options->chip_class == GFX6);
6669
6670          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6671                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6672          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6673
6674          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6675          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6676
6677          sample_pos = bld.tmp(v2);
6678
6679          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6680          load->definitions[0] = Definition(sample_pos);
6681          load->operands[0] = Operand(rsrc);
6682          load->operands[1] = Operand(addr);
6683          load->operands[2] = Operand(0u);
6684          load->offset = sample_pos_offset;
6685          load->offen = 0;
6686          load->addr64 = true;
6687          load->glc = false;
6688          load->dlc = false;
6689          load->disable_wqm = false;
6690          load->barrier = barrier_none;
6691          load->can_reorder = true;
6692          ctx->block->instructions.emplace_back(std::move(load));
6693       }
6694
6695       /* sample_pos -= 0.5 */
6696       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6697       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6698       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6699       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6700       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6701
6702       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6703       break;
6704    }
6705    case nir_intrinsic_load_barycentric_at_offset: {
6706       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6707       RegClass rc = RegClass(offset.type(), 1);
6708       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6709       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6710       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6711       break;
6712    }
6713    case nir_intrinsic_load_front_face: {
6714       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6715                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6716       break;
6717    }
6718    case nir_intrinsic_load_view_index: {
6719       if (ctx->stage & (sw_vs | sw_gs | sw_tcs | sw_tes)) {
6720          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6721          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6722          break;
6723       }
6724
6725       /* fallthrough */
6726    }
6727    case nir_intrinsic_load_layer_id: {
6728       unsigned idx = nir_intrinsic_base(instr);
6729       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6730                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6731       break;
6732    }
6733    case nir_intrinsic_load_frag_coord: {
6734       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6735       break;
6736    }
6737    case nir_intrinsic_load_sample_pos: {
6738       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6739       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6740       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6741                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6742                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6743       break;
6744    }
6745    case nir_intrinsic_load_tess_coord:
6746       visit_load_tess_coord(ctx, instr);
6747       break;
6748    case nir_intrinsic_load_interpolated_input:
6749       visit_load_interpolated_input(ctx, instr);
6750       break;
6751    case nir_intrinsic_store_output:
6752       visit_store_output(ctx, instr);
6753       break;
6754    case nir_intrinsic_load_input:
6755    case nir_intrinsic_load_input_vertex:
6756       visit_load_input(ctx, instr);
6757       break;
6758    case nir_intrinsic_load_output:
6759       visit_load_output(ctx, instr);
6760       break;
6761    case nir_intrinsic_load_per_vertex_input:
6762       visit_load_per_vertex_input(ctx, instr);
6763       break;
6764    case nir_intrinsic_load_per_vertex_output:
6765       visit_load_per_vertex_output(ctx, instr);
6766       break;
6767    case nir_intrinsic_store_per_vertex_output:
6768       visit_store_per_vertex_output(ctx, instr);
6769       break;
6770    case nir_intrinsic_load_ubo:
6771       visit_load_ubo(ctx, instr);
6772       break;
6773    case nir_intrinsic_load_push_constant:
6774       visit_load_push_constant(ctx, instr);
6775       break;
6776    case nir_intrinsic_load_constant:
6777       visit_load_constant(ctx, instr);
6778       break;
6779    case nir_intrinsic_vulkan_resource_index:
6780       visit_load_resource(ctx, instr);
6781       break;
6782    case nir_intrinsic_discard:
6783       visit_discard(ctx, instr);
6784       break;
6785    case nir_intrinsic_discard_if:
6786       visit_discard_if(ctx, instr);
6787       break;
6788    case nir_intrinsic_load_shared:
6789       visit_load_shared(ctx, instr);
6790       break;
6791    case nir_intrinsic_store_shared:
6792       visit_store_shared(ctx, instr);
6793       break;
6794    case nir_intrinsic_shared_atomic_add:
6795    case nir_intrinsic_shared_atomic_imin:
6796    case nir_intrinsic_shared_atomic_umin:
6797    case nir_intrinsic_shared_atomic_imax:
6798    case nir_intrinsic_shared_atomic_umax:
6799    case nir_intrinsic_shared_atomic_and:
6800    case nir_intrinsic_shared_atomic_or:
6801    case nir_intrinsic_shared_atomic_xor:
6802    case nir_intrinsic_shared_atomic_exchange:
6803    case nir_intrinsic_shared_atomic_comp_swap:
6804       visit_shared_atomic(ctx, instr);
6805       break;
6806    case nir_intrinsic_image_deref_load:
6807       visit_image_load(ctx, instr);
6808       break;
6809    case nir_intrinsic_image_deref_store:
6810       visit_image_store(ctx, instr);
6811       break;
6812    case nir_intrinsic_image_deref_atomic_add:
6813    case nir_intrinsic_image_deref_atomic_umin:
6814    case nir_intrinsic_image_deref_atomic_imin:
6815    case nir_intrinsic_image_deref_atomic_umax:
6816    case nir_intrinsic_image_deref_atomic_imax:
6817    case nir_intrinsic_image_deref_atomic_and:
6818    case nir_intrinsic_image_deref_atomic_or:
6819    case nir_intrinsic_image_deref_atomic_xor:
6820    case nir_intrinsic_image_deref_atomic_exchange:
6821    case nir_intrinsic_image_deref_atomic_comp_swap:
6822       visit_image_atomic(ctx, instr);
6823       break;
6824    case nir_intrinsic_image_deref_size:
6825       visit_image_size(ctx, instr);
6826       break;
6827    case nir_intrinsic_load_ssbo:
6828       visit_load_ssbo(ctx, instr);
6829       break;
6830    case nir_intrinsic_store_ssbo:
6831       visit_store_ssbo(ctx, instr);
6832       break;
6833    case nir_intrinsic_load_global:
6834       visit_load_global(ctx, instr);
6835       break;
6836    case nir_intrinsic_store_global:
6837       visit_store_global(ctx, instr);
6838       break;
6839    case nir_intrinsic_global_atomic_add:
6840    case nir_intrinsic_global_atomic_imin:
6841    case nir_intrinsic_global_atomic_umin:
6842    case nir_intrinsic_global_atomic_imax:
6843    case nir_intrinsic_global_atomic_umax:
6844    case nir_intrinsic_global_atomic_and:
6845    case nir_intrinsic_global_atomic_or:
6846    case nir_intrinsic_global_atomic_xor:
6847    case nir_intrinsic_global_atomic_exchange:
6848    case nir_intrinsic_global_atomic_comp_swap:
6849       visit_global_atomic(ctx, instr);
6850       break;
6851    case nir_intrinsic_ssbo_atomic_add:
6852    case nir_intrinsic_ssbo_atomic_imin:
6853    case nir_intrinsic_ssbo_atomic_umin:
6854    case nir_intrinsic_ssbo_atomic_imax:
6855    case nir_intrinsic_ssbo_atomic_umax:
6856    case nir_intrinsic_ssbo_atomic_and:
6857    case nir_intrinsic_ssbo_atomic_or:
6858    case nir_intrinsic_ssbo_atomic_xor:
6859    case nir_intrinsic_ssbo_atomic_exchange:
6860    case nir_intrinsic_ssbo_atomic_comp_swap:
6861       visit_atomic_ssbo(ctx, instr);
6862       break;
6863    case nir_intrinsic_load_scratch:
6864       visit_load_scratch(ctx, instr);
6865       break;
6866    case nir_intrinsic_store_scratch:
6867       visit_store_scratch(ctx, instr);
6868       break;
6869    case nir_intrinsic_get_buffer_size:
6870       visit_get_buffer_size(ctx, instr);
6871       break;
6872    case nir_intrinsic_control_barrier: {
6873       if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
6874          /* GFX6 only (thanks to a hw bug workaround):
6875           * The real barrier instruction isn’t needed, because an entire patch
6876           * always fits into a single wave.
6877           */
6878          break;
6879       }
6880
6881       if (ctx->program->workgroup_size > ctx->program->wave_size)
6882          bld.sopp(aco_opcode::s_barrier);
6883
6884       break;
6885    }
6886    case nir_intrinsic_memory_barrier_tcs_patch:
6887    case nir_intrinsic_group_memory_barrier:
6888    case nir_intrinsic_memory_barrier:
6889    case nir_intrinsic_memory_barrier_buffer:
6890    case nir_intrinsic_memory_barrier_image:
6891    case nir_intrinsic_memory_barrier_shared:
6892       emit_memory_barrier(ctx, instr);
6893       break;
6894    case nir_intrinsic_load_num_work_groups: {
6895       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6896       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6897       emit_split_vector(ctx, dst, 3);
6898       break;
6899    }
6900    case nir_intrinsic_load_local_invocation_id: {
6901       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6902       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6903       emit_split_vector(ctx, dst, 3);
6904       break;
6905    }
6906    case nir_intrinsic_load_work_group_id: {
6907       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6908       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6909       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6910                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6911                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6912                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6913       emit_split_vector(ctx, dst, 3);
6914       break;
6915    }
6916    case nir_intrinsic_load_local_invocation_index: {
6917       Temp id = emit_mbcnt(ctx, bld.def(v1));
6918
6919       /* The tg_size bits [6:11] contain the subgroup id,
6920        * we need this multiplied by the wave size, and then OR the thread id to it.
6921        */
6922       if (ctx->program->wave_size == 64) {
6923          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6924          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6925                                 get_arg(ctx, ctx->args->ac.tg_size));
6926          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6927       } else {
6928          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6929          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6930                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6931          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6932       }
6933       break;
6934    }
6935    case nir_intrinsic_load_subgroup_id: {
6936       if (ctx->stage == compute_cs) {
6937          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6938                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6939       } else {
6940          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6941       }
6942       break;
6943    }
6944    case nir_intrinsic_load_subgroup_invocation: {
6945       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6946       break;
6947    }
6948    case nir_intrinsic_load_num_subgroups: {
6949       if (ctx->stage == compute_cs)
6950          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6951                   get_arg(ctx, ctx->args->ac.tg_size));
6952       else
6953          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6954       break;
6955    }
6956    case nir_intrinsic_ballot: {
6957       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6958       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6959       Definition tmp = bld.def(dst.regClass());
6960       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6961       if (instr->src[0].ssa->bit_size == 1) {
6962          assert(src.regClass() == bld.lm);
6963          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6964       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6965          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6966       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6967          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6968       } else {
6969          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6970          nir_print_instr(&instr->instr, stderr);
6971          fprintf(stderr, "\n");
6972       }
6973       if (dst.size() != bld.lm.size()) {
6974          /* Wave32 with ballot size set to 64 */
6975          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6976       }
6977       emit_wqm(ctx, tmp.getTemp(), dst);
6978       break;
6979    }
6980    case nir_intrinsic_shuffle:
6981    case nir_intrinsic_read_invocation: {
6982       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6983       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6984          emit_uniform_subgroup(ctx, instr, src);
6985       } else {
6986          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6987          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6988             tid = bld.as_uniform(tid);
6989          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6990          if (src.regClass() == v1) {
6991             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6992          } else if (src.regClass() == v2) {
6993             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6994             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6995             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6996             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6997             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6998             emit_split_vector(ctx, dst, 2);
6999          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
7000             assert(src.regClass() == bld.lm);
7001             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
7002             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7003          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
7004             assert(src.regClass() == bld.lm);
7005             Temp tmp;
7006             if (ctx->program->chip_class <= GFX7)
7007                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
7008             else if (ctx->program->wave_size == 64)
7009                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
7010             else
7011                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
7012             tmp = emit_extract_vector(ctx, tmp, 0, v1);
7013             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
7014             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
7015          } else {
7016             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7017             nir_print_instr(&instr->instr, stderr);
7018             fprintf(stderr, "\n");
7019          }
7020       }
7021       break;
7022    }
7023    case nir_intrinsic_load_sample_id: {
7024       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7025                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
7026       break;
7027    }
7028    case nir_intrinsic_load_sample_mask_in: {
7029       visit_load_sample_mask_in(ctx, instr);
7030       break;
7031    }
7032    case nir_intrinsic_read_first_invocation: {
7033       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7034       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7035       if (src.regClass() == v1) {
7036          emit_wqm(ctx,
7037                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
7038                   dst);
7039       } else if (src.regClass() == v2) {
7040          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7041          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7042          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
7043          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
7044          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7045          emit_split_vector(ctx, dst, 2);
7046       } else if (instr->dest.ssa.bit_size == 1) {
7047          assert(src.regClass() == bld.lm);
7048          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
7049                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
7050          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7051       } else if (src.regClass() == s1) {
7052          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
7053       } else if (src.regClass() == s2) {
7054          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
7055       } else {
7056          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7057          nir_print_instr(&instr->instr, stderr);
7058          fprintf(stderr, "\n");
7059       }
7060       break;
7061    }
7062    case nir_intrinsic_vote_all: {
7063       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7064       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7065       assert(src.regClass() == bld.lm);
7066       assert(dst.regClass() == bld.lm);
7067
7068       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
7069       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
7070       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
7071       break;
7072    }
7073    case nir_intrinsic_vote_any: {
7074       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7075       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7076       assert(src.regClass() == bld.lm);
7077       assert(dst.regClass() == bld.lm);
7078
7079       Temp tmp = bool_to_scalar_condition(ctx, src);
7080       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
7081       break;
7082    }
7083    case nir_intrinsic_reduce:
7084    case nir_intrinsic_inclusive_scan:
7085    case nir_intrinsic_exclusive_scan: {
7086       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7087       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7088       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
7089       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
7090          nir_intrinsic_cluster_size(instr) : 0;
7091       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
7092
7093       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
7094          emit_uniform_subgroup(ctx, instr, src);
7095       } else if (instr->dest.ssa.bit_size == 1) {
7096          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
7097             op = nir_op_iand;
7098          else if (op == nir_op_iadd)
7099             op = nir_op_ixor;
7100          else if (op == nir_op_umax || op == nir_op_imax)
7101             op = nir_op_ior;
7102          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
7103
7104          switch (instr->intrinsic) {
7105          case nir_intrinsic_reduce:
7106             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
7107             break;
7108          case nir_intrinsic_exclusive_scan:
7109             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
7110             break;
7111          case nir_intrinsic_inclusive_scan:
7112             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
7113             break;
7114          default:
7115             assert(false);
7116          }
7117       } else if (cluster_size == 1) {
7118          bld.copy(Definition(dst), src);
7119       } else {
7120          src = as_vgpr(ctx, src);
7121
7122          ReduceOp reduce_op;
7123          switch (op) {
7124          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
7125             CASE(iadd)
7126             CASE(imul)
7127             CASE(fadd)
7128             CASE(fmul)
7129             CASE(imin)
7130             CASE(umin)
7131             CASE(fmin)
7132             CASE(imax)
7133             CASE(umax)
7134             CASE(fmax)
7135             CASE(iand)
7136             CASE(ior)
7137             CASE(ixor)
7138             default:
7139                unreachable("unknown reduction op");
7140          #undef CASE
7141          }
7142
7143          aco_opcode aco_op;
7144          switch (instr->intrinsic) {
7145             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
7146             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
7147             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
7148             default:
7149                unreachable("unknown reduce intrinsic");
7150          }
7151
7152          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
7153          reduce->operands[0] = Operand(src);
7154          // filled in by aco_reduce_assign.cpp, used internally as part of the
7155          // reduce sequence
7156          assert(dst.size() == 1 || dst.size() == 2);
7157          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
7158          reduce->operands[2] = Operand(v1.as_linear());
7159
7160          Temp tmp_dst = bld.tmp(dst.regClass());
7161          reduce->definitions[0] = Definition(tmp_dst);
7162          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
7163          reduce->definitions[2] = Definition();
7164          reduce->definitions[3] = Definition(scc, s1);
7165          reduce->definitions[4] = Definition();
7166          reduce->reduce_op = reduce_op;
7167          reduce->cluster_size = cluster_size;
7168          ctx->block->instructions.emplace_back(std::move(reduce));
7169
7170          emit_wqm(ctx, tmp_dst, dst);
7171       }
7172       break;
7173    }
7174    case nir_intrinsic_quad_broadcast: {
7175       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7176       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7177          emit_uniform_subgroup(ctx, instr, src);
7178       } else {
7179          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7180          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
7181          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
7182
7183          if (instr->dest.ssa.bit_size == 1) {
7184             assert(src.regClass() == bld.lm);
7185             assert(dst.regClass() == bld.lm);
7186             uint32_t half_mask = 0x11111111u << lane;
7187             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
7188             Temp tmp = bld.tmp(bld.lm);
7189             bld.sop1(Builder::s_wqm, Definition(tmp),
7190                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
7191                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
7192             emit_wqm(ctx, tmp, dst);
7193          } else if (instr->dest.ssa.bit_size == 32) {
7194             if (ctx->program->chip_class >= GFX8)
7195                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
7196             else
7197                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
7198          } else if (instr->dest.ssa.bit_size == 64) {
7199             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7200             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7201             if (ctx->program->chip_class >= GFX8) {
7202                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7203                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7204             } else {
7205                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
7206                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
7207             }
7208             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7209             emit_split_vector(ctx, dst, 2);
7210          } else {
7211             fprintf(stderr, "Unimplemented NIR instr bit size: ");
7212             nir_print_instr(&instr->instr, stderr);
7213             fprintf(stderr, "\n");
7214          }
7215       }
7216       break;
7217    }
7218    case nir_intrinsic_quad_swap_horizontal:
7219    case nir_intrinsic_quad_swap_vertical:
7220    case nir_intrinsic_quad_swap_diagonal:
7221    case nir_intrinsic_quad_swizzle_amd: {
7222       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7223       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7224          emit_uniform_subgroup(ctx, instr, src);
7225          break;
7226       }
7227       uint16_t dpp_ctrl = 0;
7228       switch (instr->intrinsic) {
7229       case nir_intrinsic_quad_swap_horizontal:
7230          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
7231          break;
7232       case nir_intrinsic_quad_swap_vertical:
7233          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
7234          break;
7235       case nir_intrinsic_quad_swap_diagonal:
7236          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
7237          break;
7238       case nir_intrinsic_quad_swizzle_amd:
7239          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
7240          break;
7241       default:
7242          break;
7243       }
7244       if (ctx->program->chip_class < GFX8)
7245          dpp_ctrl |= (1 << 15);
7246
7247       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7248       if (instr->dest.ssa.bit_size == 1) {
7249          assert(src.regClass() == bld.lm);
7250          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
7251          if (ctx->program->chip_class >= GFX8)
7252             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7253          else
7254             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7255          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
7256          emit_wqm(ctx, tmp, dst);
7257       } else if (instr->dest.ssa.bit_size == 32) {
7258          Temp tmp;
7259          if (ctx->program->chip_class >= GFX8)
7260             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
7261          else
7262             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
7263          emit_wqm(ctx, tmp, dst);
7264       } else if (instr->dest.ssa.bit_size == 64) {
7265          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7266          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7267          if (ctx->program->chip_class >= GFX8) {
7268             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
7269             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
7270          } else {
7271             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
7272             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
7273          }
7274          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7275          emit_split_vector(ctx, dst, 2);
7276       } else {
7277          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7278          nir_print_instr(&instr->instr, stderr);
7279          fprintf(stderr, "\n");
7280       }
7281       break;
7282    }
7283    case nir_intrinsic_masked_swizzle_amd: {
7284       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7285       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
7286          emit_uniform_subgroup(ctx, instr, src);
7287          break;
7288       }
7289       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7290       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
7291       if (dst.regClass() == v1) {
7292          emit_wqm(ctx,
7293                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
7294                   dst);
7295       } else if (dst.regClass() == v2) {
7296          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
7297          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
7298          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
7299          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
7300          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7301          emit_split_vector(ctx, dst, 2);
7302       } else {
7303          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7304          nir_print_instr(&instr->instr, stderr);
7305          fprintf(stderr, "\n");
7306       }
7307       break;
7308    }
7309    case nir_intrinsic_write_invocation_amd: {
7310       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
7311       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
7312       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
7313       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7314       if (dst.regClass() == v1) {
7315          /* src2 is ignored for writelane. RA assigns the same reg for dst */
7316          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
7317       } else if (dst.regClass() == v2) {
7318          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
7319          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
7320          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
7321          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
7322          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
7323          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
7324          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
7325          emit_split_vector(ctx, dst, 2);
7326       } else {
7327          fprintf(stderr, "Unimplemented NIR instr bit size: ");
7328          nir_print_instr(&instr->instr, stderr);
7329          fprintf(stderr, "\n");
7330       }
7331       break;
7332    }
7333    case nir_intrinsic_mbcnt_amd: {
7334       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7335       RegClass rc = RegClass(src.type(), 1);
7336       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
7337       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
7338       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7339       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
7340       emit_wqm(ctx, wqm_tmp, dst);
7341       break;
7342    }
7343    case nir_intrinsic_load_helper_invocation: {
7344       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7345       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
7346       ctx->block->kind |= block_kind_needs_lowering;
7347       ctx->program->needs_exact = true;
7348       break;
7349    }
7350    case nir_intrinsic_is_helper_invocation: {
7351       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7352       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
7353       ctx->block->kind |= block_kind_needs_lowering;
7354       ctx->program->needs_exact = true;
7355       break;
7356    }
7357    case nir_intrinsic_demote:
7358       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
7359
7360       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7361          ctx->cf_info.exec_potentially_empty_discard = true;
7362       ctx->block->kind |= block_kind_uses_demote;
7363       ctx->program->needs_exact = true;
7364       break;
7365    case nir_intrinsic_demote_if: {
7366       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
7367       assert(src.regClass() == bld.lm);
7368       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
7369       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
7370
7371       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
7372          ctx->cf_info.exec_potentially_empty_discard = true;
7373       ctx->block->kind |= block_kind_uses_demote;
7374       ctx->program->needs_exact = true;
7375       break;
7376    }
7377    case nir_intrinsic_first_invocation: {
7378       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
7379                get_ssa_temp(ctx, &instr->dest.ssa));
7380       break;
7381    }
7382    case nir_intrinsic_shader_clock:
7383       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
7384       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
7385       break;
7386    case nir_intrinsic_load_vertex_id_zero_base: {
7387       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7388       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
7389       break;
7390    }
7391    case nir_intrinsic_load_first_vertex: {
7392       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7393       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
7394       break;
7395    }
7396    case nir_intrinsic_load_base_instance: {
7397       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7398       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
7399       break;
7400    }
7401    case nir_intrinsic_load_instance_id: {
7402       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7403       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
7404       break;
7405    }
7406    case nir_intrinsic_load_draw_id: {
7407       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7408       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
7409       break;
7410    }
7411    case nir_intrinsic_load_invocation_id: {
7412       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7413
7414       if (ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
7415          if (ctx->options->chip_class >= GFX10)
7416             bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7417          else
7418             bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
7419       } else if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) {
7420          bld.vop3(aco_opcode::v_bfe_u32, Definition(dst),
7421                   get_arg(ctx, ctx->args->ac.tcs_rel_ids), Operand(8u), Operand(5u));
7422       } else {
7423          unreachable("Unsupported stage for load_invocation_id");
7424       }
7425
7426       break;
7427    }
7428    case nir_intrinsic_load_primitive_id: {
7429       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7430
7431       switch (ctx->shader->info.stage) {
7432       case MESA_SHADER_GEOMETRY:
7433          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
7434          break;
7435       case MESA_SHADER_TESS_CTRL:
7436          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tcs_patch_id));
7437          break;
7438       case MESA_SHADER_TESS_EVAL:
7439          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.tes_patch_id));
7440          break;
7441       default:
7442          unreachable("Unimplemented shader stage for nir_intrinsic_load_primitive_id");
7443       }
7444
7445       break;
7446    }
7447    case nir_intrinsic_load_patch_vertices_in: {
7448       assert(ctx->shader->info.stage == MESA_SHADER_TESS_CTRL ||
7449              ctx->shader->info.stage == MESA_SHADER_TESS_EVAL);
7450
7451       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7452       bld.copy(Definition(dst), Operand(ctx->args->options->key.tcs.input_vertices));
7453       break;
7454    }
7455    case nir_intrinsic_emit_vertex_with_counter: {
7456       visit_emit_vertex_with_counter(ctx, instr);
7457       break;
7458    }
7459    case nir_intrinsic_end_primitive_with_counter: {
7460       unsigned stream = nir_intrinsic_stream_id(instr);
7461       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
7462       break;
7463    }
7464    case nir_intrinsic_set_vertex_count: {
7465       /* unused, the HW keeps track of this for us */
7466       break;
7467    }
7468    default:
7469       fprintf(stderr, "Unimplemented intrinsic instr: ");
7470       nir_print_instr(&instr->instr, stderr);
7471       fprintf(stderr, "\n");
7472       abort();
7473
7474       break;
7475    }
7476 }
7477
7478
7479 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
7480                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
7481                     enum glsl_base_type *stype)
7482 {
7483    nir_deref_instr *texture_deref_instr = NULL;
7484    nir_deref_instr *sampler_deref_instr = NULL;
7485    int plane = -1;
7486
7487    for (unsigned i = 0; i < instr->num_srcs; i++) {
7488       switch (instr->src[i].src_type) {
7489       case nir_tex_src_texture_deref:
7490          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
7491          break;
7492       case nir_tex_src_sampler_deref:
7493          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
7494          break;
7495       case nir_tex_src_plane:
7496          plane = nir_src_as_int(instr->src[i].src);
7497          break;
7498       default:
7499          break;
7500       }
7501    }
7502
7503    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
7504
7505    if (!sampler_deref_instr)
7506       sampler_deref_instr = texture_deref_instr;
7507
7508    if (plane >= 0) {
7509       assert(instr->op != nir_texop_txf_ms &&
7510              instr->op != nir_texop_samples_identical);
7511       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
7512       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
7513    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
7514       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
7515    } else if (instr->op == nir_texop_fragment_mask_fetch) {
7516       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
7517    } else {
7518       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
7519    }
7520    if (samp_ptr) {
7521       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
7522
7523       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
7524          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
7525          Builder bld(ctx->program, ctx->block);
7526
7527          /* to avoid unnecessary moves, we split and recombine sampler and image */
7528          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
7529                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
7530          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
7531          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
7532                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
7533                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
7534          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
7535                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
7536
7537          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
7538          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
7539                                img[0], img[1], img[2], img[3],
7540                                img[4], img[5], img[6], img[7]);
7541          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
7542                                 samp[0], samp[1], samp[2], samp[3]);
7543       }
7544    }
7545    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
7546                      instr->op == nir_texop_samples_identical))
7547       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
7548 }
7549
7550 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
7551                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
7552 {
7553    Builder bld(ctx->program, ctx->block);
7554
7555    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
7556    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
7557    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
7558
7559    Operand neg_one(0xbf800000u);
7560    Operand one(0x3f800000u);
7561    Operand two(0x40000000u);
7562    Operand four(0x40800000u);
7563
7564    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
7565    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
7566    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
7567
7568    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
7569    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
7570    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
7571    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
7572
7573    // select sc
7574    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
7575    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
7576                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
7577                        one, is_ma_y);
7578    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
7579
7580    // select tc
7581    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
7582    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
7583    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
7584
7585    // select ma
7586    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7587                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
7588                   deriv_z, is_ma_z);
7589    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
7590    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
7591 }
7592
7593 void prepare_cube_coords(isel_context *ctx, std::vector<Temp>& coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
7594 {
7595    Builder bld(ctx->program, ctx->block);
7596    Temp ma, tc, sc, id;
7597
7598    if (is_array) {
7599       coords[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[3]);
7600
7601       // see comment in ac_prepare_cube_coords()
7602       if (ctx->options->chip_class <= GFX8)
7603          coords[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coords[3]);
7604    }
7605
7606    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7607
7608    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
7609    vop3a->operands[0] = Operand(ma);
7610    vop3a->abs[0] = true;
7611    Temp invma = bld.tmp(v1);
7612    vop3a->definitions[0] = Definition(invma);
7613    ctx->block->instructions.emplace_back(std::move(vop3a));
7614
7615    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7616    if (!is_deriv)
7617       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
7618
7619    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7620    if (!is_deriv)
7621       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
7622
7623    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coords[0], coords[1], coords[2]);
7624
7625    if (is_deriv) {
7626       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
7627       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
7628
7629       for (unsigned i = 0; i < 2; i++) {
7630          // see comment in ac_prepare_cube_coords()
7631          Temp deriv_ma;
7632          Temp deriv_sc, deriv_tc;
7633          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
7634                            &deriv_ma, &deriv_sc, &deriv_tc);
7635
7636          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
7637
7638          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7639                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
7640                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
7641          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7642                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
7643                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
7644          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
7645       }
7646
7647       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
7648       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
7649    }
7650
7651    if (is_array)
7652       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coords[3], id, Operand(0x41000000u/*8.0*/));
7653    coords.resize(3);
7654    coords[0] = sc;
7655    coords[1] = tc;
7656    coords[2] = id;
7657 }
7658
7659 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7660 {
7661    if (vec->parent_instr->type != nir_instr_type_alu)
7662       return;
7663    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7664    if (vec_instr->op != nir_op_vec(vec->num_components))
7665       return;
7666
7667    for (unsigned i = 0; i < vec->num_components; i++) {
7668       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7669               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7670    }
7671 }
7672
7673 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7674 {
7675    Builder bld(ctx->program, ctx->block);
7676    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7677         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7678    Temp resource, sampler, fmask_ptr, bias = Temp(), compare = Temp(), sample_index = Temp(),
7679         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp();
7680    std::vector<Temp> coords;
7681    std::vector<Temp> derivs;
7682    nir_const_value *sample_index_cv = NULL;
7683    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7684    enum glsl_base_type stype;
7685    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7686
7687    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7688                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7689    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7690                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7691
7692    for (unsigned i = 0; i < instr->num_srcs; i++) {
7693       switch (instr->src[i].src_type) {
7694       case nir_tex_src_coord: {
7695          Temp coord = get_ssa_temp(ctx, instr->src[i].src.ssa);
7696          for (unsigned i = 0; i < coord.size(); i++)
7697             coords.emplace_back(emit_extract_vector(ctx, coord, i, v1));
7698          break;
7699       }
7700       case nir_tex_src_bias:
7701          if (instr->op == nir_texop_txb) {
7702             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7703             has_bias = true;
7704          }
7705          break;
7706       case nir_tex_src_lod: {
7707          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7708
7709          if (val && val->f32 <= 0.0) {
7710             level_zero = true;
7711          } else {
7712             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7713             has_lod = true;
7714          }
7715          break;
7716       }
7717       case nir_tex_src_comparator:
7718          if (instr->is_shadow) {
7719             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7720             has_compare = true;
7721          }
7722          break;
7723       case nir_tex_src_offset:
7724          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7725          get_const_vec(instr->src[i].src.ssa, const_offset);
7726          has_offset = true;
7727          break;
7728       case nir_tex_src_ddx:
7729          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7730          has_ddx = true;
7731          break;
7732       case nir_tex_src_ddy:
7733          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7734          has_ddy = true;
7735          break;
7736       case nir_tex_src_ms_index:
7737          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7738          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7739          has_sample_index = true;
7740          break;
7741       case nir_tex_src_texture_offset:
7742       case nir_tex_src_sampler_offset:
7743       default:
7744          break;
7745       }
7746    }
7747
7748    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7749       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7750
7751    if (instr->op == nir_texop_texture_samples) {
7752       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7753
7754       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7755       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7756       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7757       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7758
7759       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7760                samples, Operand(1u), bld.scc(is_msaa));
7761       return;
7762    }
7763
7764    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7765       aco_ptr<Instruction> tmp_instr;
7766       Temp acc, pack = Temp();
7767
7768       uint32_t pack_const = 0;
7769       for (unsigned i = 0; i < offset.size(); i++) {
7770          if (!const_offset[i])
7771             continue;
7772          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7773       }
7774
7775       if (offset.type() == RegType::sgpr) {
7776          for (unsigned i = 0; i < offset.size(); i++) {
7777             if (const_offset[i])
7778                continue;
7779
7780             acc = emit_extract_vector(ctx, offset, i, s1);
7781             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7782
7783             if (i) {
7784                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7785             }
7786
7787             if (pack == Temp()) {
7788                pack = acc;
7789             } else {
7790                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7791             }
7792          }
7793
7794          if (pack_const && pack != Temp())
7795             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7796       } else {
7797          for (unsigned i = 0; i < offset.size(); i++) {
7798             if (const_offset[i])
7799                continue;
7800
7801             acc = emit_extract_vector(ctx, offset, i, v1);
7802             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7803
7804             if (i) {
7805                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7806             }
7807
7808             if (pack == Temp()) {
7809                pack = acc;
7810             } else {
7811                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7812             }
7813          }
7814
7815          if (pack_const && pack != Temp())
7816             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7817       }
7818       if (pack_const && pack == Temp())
7819          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7820       else if (pack == Temp())
7821          has_offset = false;
7822       else
7823          offset = pack;
7824    }
7825
7826    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7827       prepare_cube_coords(ctx, coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7828
7829    /* pack derivatives */
7830    if (has_ddx || has_ddy) {
7831       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7832          assert(has_ddx && has_ddy && ddx.size() == 1 && ddy.size() == 1);
7833          Temp zero = bld.copy(bld.def(v1), Operand(0u));
7834          derivs = {ddy, zero, ddy, zero};
7835       } else {
7836          for (unsigned i = 0; has_ddx && i < ddx.size(); i++)
7837             derivs.emplace_back(emit_extract_vector(ctx, ddx, i, v1));
7838          for (unsigned i = 0; has_ddy && i < ddy.size(); i++)
7839             derivs.emplace_back(emit_extract_vector(ctx, ddy, i, v1));
7840       }
7841       has_derivs = true;
7842    }
7843
7844    if (instr->coord_components > 1 &&
7845        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7846        instr->is_array &&
7847        instr->op != nir_texop_txf)
7848       coords[1] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[1]);
7849
7850    if (instr->coord_components > 2 &&
7851       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7852        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7853        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7854        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7855        instr->is_array &&
7856        instr->op != nir_texop_txf &&
7857        instr->op != nir_texop_txf_ms &&
7858        instr->op != nir_texop_fragment_fetch &&
7859        instr->op != nir_texop_fragment_mask_fetch)
7860       coords[2] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coords[2]);
7861
7862    if (ctx->options->chip_class == GFX9 &&
7863        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7864        instr->op != nir_texop_lod && instr->coord_components) {
7865       assert(coords.size() > 0 && coords.size() < 3);
7866
7867       coords.insert(std::next(coords.begin()), bld.copy(bld.def(v1), instr->op == nir_texop_txf ?
7868                                                                      Operand((uint32_t) 0) :
7869                                                                      Operand((uint32_t) 0x3f000000)));
7870    }
7871
7872    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7873
7874    if (instr->op == nir_texop_samples_identical)
7875       resource = fmask_ptr;
7876
7877    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7878              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7879             instr->op != nir_texop_txs &&
7880             instr->op != nir_texop_fragment_fetch &&
7881             instr->op != nir_texop_fragment_mask_fetch) {
7882       assert(has_sample_index);
7883       Operand op(sample_index);
7884       if (sample_index_cv)
7885          op = Operand(sample_index_cv->u32);
7886       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7887    }
7888
7889    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7890       for (unsigned i = 0; i < std::min(offset.size(), instr->coord_components); i++) {
7891          Temp off = emit_extract_vector(ctx, offset, i, v1);
7892          coords[i] = bld.vadd32(bld.def(v1), coords[i], off);
7893       }
7894       has_offset = false;
7895    }
7896
7897    /* Build tex instruction */
7898    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7899    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7900                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7901                   : 0;
7902    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7903    Temp tmp_dst = dst;
7904
7905    /* gather4 selects the component by dmask and always returns vec4 */
7906    if (instr->op == nir_texop_tg4) {
7907       assert(instr->dest.ssa.num_components == 4);
7908       if (instr->is_shadow)
7909          dmask = 1;
7910       else
7911          dmask = 1 << instr->component;
7912       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7913          tmp_dst = bld.tmp(v4);
7914    } else if (instr->op == nir_texop_samples_identical) {
7915       tmp_dst = bld.tmp(v1);
7916    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7917       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7918    }
7919
7920    aco_ptr<MIMG_instruction> tex;
7921    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7922       if (!has_lod)
7923          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7924
7925       bool div_by_6 = instr->op == nir_texop_txs &&
7926                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7927                       instr->is_array &&
7928                       (dmask & (1 << 2));
7929       if (tmp_dst.id() == dst.id() && div_by_6)
7930          tmp_dst = bld.tmp(tmp_dst.regClass());
7931
7932       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7933       tex->operands[0] = Operand(resource);
7934       tex->operands[1] = Operand(s4); /* no sampler */
7935       tex->operands[2] = Operand(as_vgpr(ctx,lod));
7936       if (ctx->options->chip_class == GFX9 &&
7937           instr->op == nir_texop_txs &&
7938           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7939           instr->is_array) {
7940          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7941       } else if (instr->op == nir_texop_query_levels) {
7942          tex->dmask = 1 << 3;
7943       } else {
7944          tex->dmask = dmask;
7945       }
7946       tex->da = da;
7947       tex->definitions[0] = Definition(tmp_dst);
7948       tex->dim = dim;
7949       tex->can_reorder = true;
7950       ctx->block->instructions.emplace_back(std::move(tex));
7951
7952       if (div_by_6) {
7953          /* divide 3rd value by 6 by multiplying with magic number */
7954          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7955          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7956          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7957          assert(instr->dest.ssa.num_components == 3);
7958          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7959          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7960                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7961                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7962                               by_6);
7963
7964       }
7965
7966       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7967       return;
7968    }
7969
7970    Temp tg4_compare_cube_wa64 = Temp();
7971
7972    if (tg4_integer_workarounds) {
7973       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 3, 1));
7974       tex->operands[0] = Operand(resource);
7975       tex->operands[1] = Operand(s4); /* no sampler */
7976       tex->operands[2] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7977       tex->dim = dim;
7978       tex->dmask = 0x3;
7979       tex->da = da;
7980       Temp size = bld.tmp(v2);
7981       tex->definitions[0] = Definition(size);
7982       tex->can_reorder = true;
7983       ctx->block->instructions.emplace_back(std::move(tex));
7984       emit_split_vector(ctx, size, size.size());
7985
7986       Temp half_texel[2];
7987       for (unsigned i = 0; i < 2; i++) {
7988          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7989          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7990          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7991          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7992       }
7993
7994       Temp new_coords[2] = {
7995          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[0], half_texel[0]),
7996          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), coords[1], half_texel[1])
7997       };
7998
7999       if (tg4_integer_cube_workaround) {
8000          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
8001          Temp desc[resource.size()];
8002          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
8003                                                                            Format::PSEUDO, 1, resource.size())};
8004          split->operands[0] = Operand(resource);
8005          for (unsigned i = 0; i < resource.size(); i++) {
8006             desc[i] = bld.tmp(s1);
8007             split->definitions[i] = Definition(desc[i]);
8008          }
8009          ctx->block->instructions.emplace_back(std::move(split));
8010
8011          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
8012          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
8013                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
8014
8015          Temp nfmt;
8016          if (stype == GLSL_TYPE_UINT) {
8017             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8018                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
8019                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
8020                             bld.scc(compare_cube_wa));
8021          } else {
8022             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
8023                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
8024                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
8025                             bld.scc(compare_cube_wa));
8026          }
8027          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
8028          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
8029
8030          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
8031
8032          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
8033                             Operand((uint32_t)C_008F14_NUM_FORMAT));
8034          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
8035
8036          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
8037                                                                          Format::PSEUDO, resource.size(), 1)};
8038          for (unsigned i = 0; i < resource.size(); i++)
8039             vec->operands[i] = Operand(desc[i]);
8040          resource = bld.tmp(resource.regClass());
8041          vec->definitions[0] = Definition(resource);
8042          ctx->block->instructions.emplace_back(std::move(vec));
8043
8044          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8045                                   new_coords[0], coords[0], tg4_compare_cube_wa64);
8046          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8047                                   new_coords[1], coords[1], tg4_compare_cube_wa64);
8048       }
8049       coords[0] = new_coords[0];
8050       coords[1] = new_coords[1];
8051    }
8052
8053    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
8054       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
8055
8056       assert(coords.size() == 1);
8057       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
8058       aco_opcode op;
8059       switch (last_bit) {
8060       case 1:
8061          op = aco_opcode::buffer_load_format_x; break;
8062       case 2:
8063          op = aco_opcode::buffer_load_format_xy; break;
8064       case 3:
8065          op = aco_opcode::buffer_load_format_xyz; break;
8066       case 4:
8067          op = aco_opcode::buffer_load_format_xyzw; break;
8068       default:
8069          unreachable("Tex instruction loads more than 4 components.");
8070       }
8071
8072       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
8073       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
8074          tmp_dst = dst;
8075       else
8076          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
8077
8078       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
8079       mubuf->operands[0] = Operand(resource);
8080       mubuf->operands[1] = Operand(coords[0]);
8081       mubuf->operands[2] = Operand((uint32_t) 0);
8082       mubuf->definitions[0] = Definition(tmp_dst);
8083       mubuf->idxen = true;
8084       mubuf->can_reorder = true;
8085       ctx->block->instructions.emplace_back(std::move(mubuf));
8086
8087       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
8088       return;
8089    }
8090
8091    /* gather MIMG address components */
8092    std::vector<Temp> args;
8093    if (has_offset)
8094       args.emplace_back(offset);
8095    if (has_bias)
8096       args.emplace_back(bias);
8097    if (has_compare)
8098       args.emplace_back(compare);
8099    if (has_derivs)
8100       args.insert(args.end(), derivs.begin(), derivs.end());
8101
8102    args.insert(args.end(), coords.begin(), coords.end());
8103    if (has_sample_index)
8104       args.emplace_back(sample_index);
8105    if (has_lod)
8106       args.emplace_back(lod);
8107
8108    Temp arg = bld.tmp(RegClass(RegType::vgpr, args.size()));
8109    aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
8110    vec->definitions[0] = Definition(arg);
8111    for (unsigned i = 0; i < args.size(); i++)
8112       vec->operands[i] = Operand(args[i]);
8113    ctx->block->instructions.emplace_back(std::move(vec));
8114
8115
8116    if (instr->op == nir_texop_txf ||
8117        instr->op == nir_texop_txf_ms ||
8118        instr->op == nir_texop_samples_identical ||
8119        instr->op == nir_texop_fragment_fetch ||
8120        instr->op == nir_texop_fragment_mask_fetch) {
8121       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
8122       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 3, 1));
8123       tex->operands[0] = Operand(resource);
8124       tex->operands[1] = Operand(s4); /* no sampler */
8125       tex->operands[2] = Operand(arg);
8126       tex->dim = dim;
8127       tex->dmask = dmask;
8128       tex->unrm = true;
8129       tex->da = da;
8130       tex->definitions[0] = Definition(tmp_dst);
8131       tex->can_reorder = true;
8132       ctx->block->instructions.emplace_back(std::move(tex));
8133
8134       if (instr->op == nir_texop_samples_identical) {
8135          assert(dmask == 1 && dst.regClass() == v1);
8136          assert(dst.id() != tmp_dst.id());
8137
8138          Temp tmp = bld.tmp(bld.lm);
8139          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
8140          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
8141
8142       } else {
8143          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
8144       }
8145       return;
8146    }
8147
8148    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
8149    aco_opcode opcode = aco_opcode::image_sample;
8150    if (has_offset) { /* image_sample_*_o */
8151       if (has_compare) {
8152          opcode = aco_opcode::image_sample_c_o;
8153          if (has_derivs)
8154             opcode = aco_opcode::image_sample_c_d_o;
8155          if (has_bias)
8156             opcode = aco_opcode::image_sample_c_b_o;
8157          if (level_zero)
8158             opcode = aco_opcode::image_sample_c_lz_o;
8159          if (has_lod)
8160             opcode = aco_opcode::image_sample_c_l_o;
8161       } else {
8162          opcode = aco_opcode::image_sample_o;
8163          if (has_derivs)
8164             opcode = aco_opcode::image_sample_d_o;
8165          if (has_bias)
8166             opcode = aco_opcode::image_sample_b_o;
8167          if (level_zero)
8168             opcode = aco_opcode::image_sample_lz_o;
8169          if (has_lod)
8170             opcode = aco_opcode::image_sample_l_o;
8171       }
8172    } else { /* no offset */
8173       if (has_compare) {
8174          opcode = aco_opcode::image_sample_c;
8175          if (has_derivs)
8176             opcode = aco_opcode::image_sample_c_d;
8177          if (has_bias)
8178             opcode = aco_opcode::image_sample_c_b;
8179          if (level_zero)
8180             opcode = aco_opcode::image_sample_c_lz;
8181          if (has_lod)
8182             opcode = aco_opcode::image_sample_c_l;
8183       } else {
8184          opcode = aco_opcode::image_sample;
8185          if (has_derivs)
8186             opcode = aco_opcode::image_sample_d;
8187          if (has_bias)
8188             opcode = aco_opcode::image_sample_b;
8189          if (level_zero)
8190             opcode = aco_opcode::image_sample_lz;
8191          if (has_lod)
8192             opcode = aco_opcode::image_sample_l;
8193       }
8194    }
8195
8196    if (instr->op == nir_texop_tg4) {
8197       if (has_offset) {
8198          opcode = aco_opcode::image_gather4_lz_o;
8199          if (has_compare)
8200             opcode = aco_opcode::image_gather4_c_lz_o;
8201       } else {
8202          opcode = aco_opcode::image_gather4_lz;
8203          if (has_compare)
8204             opcode = aco_opcode::image_gather4_c_lz;
8205       }
8206    } else if (instr->op == nir_texop_lod) {
8207       opcode = aco_opcode::image_get_lod;
8208    }
8209
8210    /* we don't need the bias, sample index, compare value or offset to be
8211     * computed in WQM but if the p_create_vector copies the coordinates, then it
8212     * needs to be in WQM */
8213    if (ctx->stage == fragment_fs &&
8214        !has_derivs && !has_lod && !level_zero &&
8215        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
8216        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
8217       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
8218
8219    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
8220    tex->operands[0] = Operand(resource);
8221    tex->operands[1] = Operand(sampler);
8222    tex->operands[2] = Operand(arg);
8223    tex->dim = dim;
8224    tex->dmask = dmask;
8225    tex->da = da;
8226    tex->definitions[0] = Definition(tmp_dst);
8227    tex->can_reorder = true;
8228    ctx->block->instructions.emplace_back(std::move(tex));
8229
8230    if (tg4_integer_cube_workaround) {
8231       assert(tmp_dst.id() != dst.id());
8232       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
8233
8234       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
8235       Temp val[4];
8236       for (unsigned i = 0; i < dst.size(); i++) {
8237          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
8238          Temp cvt_val;
8239          if (stype == GLSL_TYPE_UINT)
8240             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
8241          else
8242             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
8243          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
8244       }
8245       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
8246       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
8247                            val[0], val[1], val[2], val[3]);
8248    }
8249    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
8250    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
8251
8252 }
8253
8254
8255 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
8256 {
8257    Temp tmp = get_ssa_temp(ctx, ssa);
8258    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
8259       return Operand(tmp.regClass());
8260    else
8261       return Operand(tmp);
8262 }
8263
8264 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
8265 {
8266    aco_ptr<Pseudo_instruction> phi;
8267    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
8268    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
8269
8270    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
8271    logical |= ctx->block->kind & block_kind_merge;
8272    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
8273
8274    /* we want a sorted list of sources, since the predecessor list is also sorted */
8275    std::map<unsigned, nir_ssa_def*> phi_src;
8276    nir_foreach_phi_src(src, instr)
8277       phi_src[src->pred->index] = src->src.ssa;
8278
8279    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
8280    unsigned num_operands = 0;
8281    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size()) + 1];
8282    unsigned num_defined = 0;
8283    unsigned cur_pred_idx = 0;
8284    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
8285       if (cur_pred_idx < preds.size()) {
8286          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
8287          unsigned block = ctx->cf_info.nir_to_aco[src.first];
8288          unsigned skipped = 0;
8289          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
8290             skipped++;
8291          if (cur_pred_idx + skipped < preds.size()) {
8292             for (unsigned i = 0; i < skipped; i++)
8293                operands[num_operands++] = Operand(dst.regClass());
8294             cur_pred_idx += skipped;
8295          } else {
8296             continue;
8297          }
8298       }
8299       /* Handle missing predecessors at the end. This shouldn't happen with loop
8300        * headers and we can't ignore these sources for loop header phis. */
8301       if (!(ctx->block->kind & block_kind_loop_header) && cur_pred_idx >= preds.size())
8302          continue;
8303       cur_pred_idx++;
8304       Operand op = get_phi_operand(ctx, src.second);
8305       operands[num_operands++] = op;
8306       num_defined += !op.isUndefined();
8307    }
8308    /* handle block_kind_continue_or_break at loop exit blocks */
8309    while (cur_pred_idx++ < preds.size())
8310       operands[num_operands++] = Operand(dst.regClass());
8311
8312    /* If the loop ends with a break, still add a linear continue edge in case
8313     * that break is divergent or continue_or_break is used. We'll either remove
8314     * this operand later in visit_loop() if it's not necessary or replace the
8315     * undef with something correct. */
8316    if (!logical && ctx->block->kind & block_kind_loop_header) {
8317       nir_loop *loop = nir_cf_node_as_loop(instr->instr.block->cf_node.parent);
8318       nir_block *last = nir_loop_last_block(loop);
8319       if (last->successors[0] != instr->instr.block)
8320          operands[num_operands++] = Operand(RegClass());
8321    }
8322
8323    if (num_defined == 0) {
8324       Builder bld(ctx->program, ctx->block);
8325       if (dst.regClass() == s1) {
8326          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
8327       } else if (dst.regClass() == v1) {
8328          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
8329       } else {
8330          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8331          for (unsigned i = 0; i < dst.size(); i++)
8332             vec->operands[i] = Operand(0u);
8333          vec->definitions[0] = Definition(dst);
8334          ctx->block->instructions.emplace_back(std::move(vec));
8335       }
8336       return;
8337    }
8338
8339    /* we can use a linear phi in some cases if one src is undef */
8340    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
8341       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
8342
8343       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
8344       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
8345       assert(invert->kind & block_kind_invert);
8346
8347       unsigned then_block = invert->linear_preds[0];
8348
8349       Block* insert_block = NULL;
8350       for (unsigned i = 0; i < num_operands; i++) {
8351          Operand op = operands[i];
8352          if (op.isUndefined())
8353             continue;
8354          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
8355          phi->operands[0] = op;
8356          break;
8357       }
8358       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
8359       phi->operands[1] = Operand(dst.regClass());
8360       phi->definitions[0] = Definition(dst);
8361       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
8362       return;
8363    }
8364
8365    /* try to scalarize vector phis */
8366    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
8367       // TODO: scalarize linear phis on divergent ifs
8368       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
8369       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
8370       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
8371          Operand src = operands[i];
8372          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
8373             can_scalarize = false;
8374       }
8375       if (can_scalarize) {
8376          unsigned num_components = instr->dest.ssa.num_components;
8377          assert(dst.size() % num_components == 0);
8378          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
8379
8380          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
8381          for (unsigned k = 0; k < num_components; k++) {
8382             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8383             for (unsigned i = 0; i < num_operands; i++) {
8384                Operand src = operands[i];
8385                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
8386             }
8387             Temp phi_dst = {ctx->program->allocateId(), rc};
8388             phi->definitions[0] = Definition(phi_dst);
8389             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8390             new_vec[k] = phi_dst;
8391             vec->operands[k] = Operand(phi_dst);
8392          }
8393          vec->definitions[0] = Definition(dst);
8394          ctx->block->instructions.emplace_back(std::move(vec));
8395          ctx->allocated_vec.emplace(dst.id(), new_vec);
8396          return;
8397       }
8398    }
8399
8400    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
8401    for (unsigned i = 0; i < num_operands; i++)
8402       phi->operands[i] = operands[i];
8403    phi->definitions[0] = Definition(dst);
8404    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
8405 }
8406
8407
8408 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
8409 {
8410    Temp dst = get_ssa_temp(ctx, &instr->def);
8411
8412    assert(dst.type() == RegType::sgpr);
8413
8414    if (dst.size() == 1) {
8415       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
8416    } else {
8417       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
8418       for (unsigned i = 0; i < dst.size(); i++)
8419          vec->operands[i] = Operand(0u);
8420       vec->definitions[0] = Definition(dst);
8421       ctx->block->instructions.emplace_back(std::move(vec));
8422    }
8423 }
8424
8425 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
8426 {
8427    Builder bld(ctx->program, ctx->block);
8428    Block *logical_target;
8429    append_logical_end(ctx->block);
8430    unsigned idx = ctx->block->index;
8431
8432    switch (instr->type) {
8433    case nir_jump_break:
8434       logical_target = ctx->cf_info.parent_loop.exit;
8435       add_logical_edge(idx, logical_target);
8436       ctx->block->kind |= block_kind_break;
8437
8438       if (!ctx->cf_info.parent_if.is_divergent &&
8439           !ctx->cf_info.parent_loop.has_divergent_continue) {
8440          /* uniform break - directly jump out of the loop */
8441          ctx->block->kind |= block_kind_uniform;
8442          ctx->cf_info.has_branch = true;
8443          bld.branch(aco_opcode::p_branch);
8444          add_linear_edge(idx, logical_target);
8445          return;
8446       }
8447       ctx->cf_info.parent_loop.has_divergent_branch = true;
8448       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8449       break;
8450    case nir_jump_continue:
8451       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8452       add_logical_edge(idx, logical_target);
8453       ctx->block->kind |= block_kind_continue;
8454
8455       if (ctx->cf_info.parent_if.is_divergent) {
8456          /* for potential uniform breaks after this continue,
8457             we must ensure that they are handled correctly */
8458          ctx->cf_info.parent_loop.has_divergent_continue = true;
8459          ctx->cf_info.parent_loop.has_divergent_branch = true;
8460          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
8461       } else {
8462          /* uniform continue - directly jump to the loop header */
8463          ctx->block->kind |= block_kind_uniform;
8464          ctx->cf_info.has_branch = true;
8465          bld.branch(aco_opcode::p_branch);
8466          add_linear_edge(idx, logical_target);
8467          return;
8468       }
8469       break;
8470    default:
8471       fprintf(stderr, "Unknown NIR jump instr: ");
8472       nir_print_instr(&instr->instr, stderr);
8473       fprintf(stderr, "\n");
8474       abort();
8475    }
8476
8477    if (ctx->cf_info.parent_if.is_divergent && !ctx->cf_info.exec_potentially_empty_break) {
8478       ctx->cf_info.exec_potentially_empty_break = true;
8479       ctx->cf_info.exec_potentially_empty_break_depth = ctx->cf_info.loop_nest_depth;
8480    }
8481
8482    /* remove critical edges from linear CFG */
8483    bld.branch(aco_opcode::p_branch);
8484    Block* break_block = ctx->program->create_and_insert_block();
8485    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8486    break_block->kind |= block_kind_uniform;
8487    add_linear_edge(idx, break_block);
8488    /* the loop_header pointer might be invalidated by this point */
8489    if (instr->type == nir_jump_continue)
8490       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
8491    add_linear_edge(break_block->index, logical_target);
8492    bld.reset(break_block);
8493    bld.branch(aco_opcode::p_branch);
8494
8495    Block* continue_block = ctx->program->create_and_insert_block();
8496    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8497    add_linear_edge(idx, continue_block);
8498    append_logical_start(continue_block);
8499    ctx->block = continue_block;
8500    return;
8501 }
8502
8503 void visit_block(isel_context *ctx, nir_block *block)
8504 {
8505    nir_foreach_instr(instr, block) {
8506       switch (instr->type) {
8507       case nir_instr_type_alu:
8508          visit_alu_instr(ctx, nir_instr_as_alu(instr));
8509          break;
8510       case nir_instr_type_load_const:
8511          visit_load_const(ctx, nir_instr_as_load_const(instr));
8512          break;
8513       case nir_instr_type_intrinsic:
8514          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
8515          break;
8516       case nir_instr_type_tex:
8517          visit_tex(ctx, nir_instr_as_tex(instr));
8518          break;
8519       case nir_instr_type_phi:
8520          visit_phi(ctx, nir_instr_as_phi(instr));
8521          break;
8522       case nir_instr_type_ssa_undef:
8523          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
8524          break;
8525       case nir_instr_type_deref:
8526          break;
8527       case nir_instr_type_jump:
8528          visit_jump(ctx, nir_instr_as_jump(instr));
8529          break;
8530       default:
8531          fprintf(stderr, "Unknown NIR instr type: ");
8532          nir_print_instr(instr, stderr);
8533          fprintf(stderr, "\n");
8534          //abort();
8535       }
8536    }
8537
8538    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8539       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
8540 }
8541
8542
8543
8544 static Operand create_continue_phis(isel_context *ctx, unsigned first, unsigned last,
8545                                     aco_ptr<Instruction>& header_phi, Operand *vals)
8546 {
8547    vals[0] = Operand(header_phi->definitions[0].getTemp());
8548    RegClass rc = vals[0].regClass();
8549
8550    unsigned loop_nest_depth = ctx->program->blocks[first].loop_nest_depth;
8551
8552    unsigned next_pred = 1;
8553
8554    for (unsigned idx = first + 1; idx <= last; idx++) {
8555       Block& block = ctx->program->blocks[idx];
8556       if (block.loop_nest_depth != loop_nest_depth) {
8557          vals[idx - first] = vals[idx - 1 - first];
8558          continue;
8559       }
8560
8561       if (block.kind & block_kind_continue) {
8562          vals[idx - first] = header_phi->operands[next_pred];
8563          next_pred++;
8564          continue;
8565       }
8566
8567       bool all_same = true;
8568       for (unsigned i = 1; all_same && (i < block.linear_preds.size()); i++)
8569          all_same = vals[block.linear_preds[i] - first] == vals[block.linear_preds[0] - first];
8570
8571       Operand val;
8572       if (all_same) {
8573          val = vals[block.linear_preds[0] - first];
8574       } else {
8575          aco_ptr<Instruction> phi(create_instruction<Pseudo_instruction>(
8576             aco_opcode::p_linear_phi, Format::PSEUDO, block.linear_preds.size(), 1));
8577          for (unsigned i = 0; i < block.linear_preds.size(); i++)
8578             phi->operands[i] = vals[block.linear_preds[i] - first];
8579          val = Operand(Temp(ctx->program->allocateId(), rc));
8580          phi->definitions[0] = Definition(val.getTemp());
8581          block.instructions.emplace(block.instructions.begin(), std::move(phi));
8582       }
8583       vals[idx - first] = val;
8584    }
8585
8586    return vals[last - first];
8587 }
8588
8589 static void visit_loop(isel_context *ctx, nir_loop *loop)
8590 {
8591    //TODO: we might want to wrap the loop around a branch if exec_potentially_empty=true
8592    append_logical_end(ctx->block);
8593    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
8594    Builder bld(ctx->program, ctx->block);
8595    bld.branch(aco_opcode::p_branch);
8596    unsigned loop_preheader_idx = ctx->block->index;
8597
8598    Block loop_exit = Block();
8599    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8600    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
8601
8602    Block* loop_header = ctx->program->create_and_insert_block();
8603    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
8604    loop_header->kind |= block_kind_loop_header;
8605    add_edge(loop_preheader_idx, loop_header);
8606    ctx->block = loop_header;
8607
8608    /* emit loop body */
8609    unsigned loop_header_idx = loop_header->index;
8610    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
8611    append_logical_start(ctx->block);
8612    bool unreachable = visit_cf_list(ctx, &loop->body);
8613
8614    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
8615    if (!ctx->cf_info.has_branch) {
8616       append_logical_end(ctx->block);
8617       if (ctx->cf_info.exec_potentially_empty_discard || ctx->cf_info.exec_potentially_empty_break) {
8618          /* Discards can result in code running with an empty exec mask.
8619           * This would result in divergent breaks not ever being taken. As a
8620           * workaround, break the loop when the loop mask is empty instead of
8621           * always continuing. */
8622          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
8623          unsigned block_idx = ctx->block->index;
8624
8625          /* create helper blocks to avoid critical edges */
8626          Block *break_block = ctx->program->create_and_insert_block();
8627          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8628          break_block->kind = block_kind_uniform;
8629          bld.reset(break_block);
8630          bld.branch(aco_opcode::p_branch);
8631          add_linear_edge(block_idx, break_block);
8632          add_linear_edge(break_block->index, &loop_exit);
8633
8634          Block *continue_block = ctx->program->create_and_insert_block();
8635          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8636          continue_block->kind = block_kind_uniform;
8637          bld.reset(continue_block);
8638          bld.branch(aco_opcode::p_branch);
8639          add_linear_edge(block_idx, continue_block);
8640          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
8641
8642          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8643             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
8644          ctx->block = &ctx->program->blocks[block_idx];
8645       } else {
8646          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
8647          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8648             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8649          else
8650             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8651       }
8652
8653       bld.reset(ctx->block);
8654       bld.branch(aco_opcode::p_branch);
8655    }
8656
8657    /* Fixup phis in loop header from unreachable blocks.
8658     * has_branch/has_divergent_branch also indicates if the loop ends with a
8659     * break/continue instruction, but we don't emit those if unreachable=true */
8660    if (unreachable) {
8661       assert(ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch);
8662       bool linear = ctx->cf_info.has_branch;
8663       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
8664       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
8665          if ((logical && instr->opcode == aco_opcode::p_phi) ||
8666              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
8667             /* the last operand should be the one that needs to be removed */
8668             instr->operands.pop_back();
8669          } else if (!is_phi(instr)) {
8670             break;
8671          }
8672       }
8673    }
8674
8675    /* Fixup linear phis in loop header from expecting a continue. Both this fixup
8676     * and the previous one shouldn't both happen at once because a break in the
8677     * merge block would get CSE'd */
8678    if (nir_loop_last_block(loop)->successors[0] != nir_loop_first_block(loop)) {
8679       unsigned num_vals = ctx->cf_info.has_branch ? 1 : (ctx->block->index - loop_header_idx + 1);
8680       Operand vals[num_vals];
8681       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
8682          if (instr->opcode == aco_opcode::p_linear_phi) {
8683             if (ctx->cf_info.has_branch)
8684                instr->operands.pop_back();
8685             else
8686                instr->operands.back() = create_continue_phis(ctx, loop_header_idx, ctx->block->index, instr, vals);
8687          } else if (!is_phi(instr)) {
8688             break;
8689          }
8690       }
8691    }
8692
8693    ctx->cf_info.has_branch = false;
8694
8695    // TODO: if the loop has not a single exit, we must add one °°
8696    /* emit loop successor block */
8697    ctx->block = ctx->program->insert_block(std::move(loop_exit));
8698    append_logical_start(ctx->block);
8699
8700    #if 0
8701    // TODO: check if it is beneficial to not branch on continues
8702    /* trim linear phis in loop header */
8703    for (auto&& instr : loop_entry->instructions) {
8704       if (instr->opcode == aco_opcode::p_linear_phi) {
8705          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8706          new_phi->definitions[0] = instr->definitions[0];
8707          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8708             new_phi->operands[i] = instr->operands[i];
8709          /* check that the remaining operands are all the same */
8710          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8711             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8712          instr.swap(new_phi);
8713       } else if (instr->opcode == aco_opcode::p_phi) {
8714          continue;
8715       } else {
8716          break;
8717       }
8718    }
8719    #endif
8720 }
8721
8722 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8723 {
8724    ic->cond = cond;
8725
8726    append_logical_end(ctx->block);
8727    ctx->block->kind |= block_kind_branch;
8728
8729    /* branch to linear then block */
8730    assert(cond.regClass() == ctx->program->lane_mask);
8731    aco_ptr<Pseudo_branch_instruction> branch;
8732    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8733    branch->operands[0] = Operand(cond);
8734    ctx->block->instructions.push_back(std::move(branch));
8735
8736    ic->BB_if_idx = ctx->block->index;
8737    ic->BB_invert = Block();
8738    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8739    /* Invert blocks are intentionally not marked as top level because they
8740     * are not part of the logical cfg. */
8741    ic->BB_invert.kind |= block_kind_invert;
8742    ic->BB_endif = Block();
8743    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8744    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8745
8746    ic->exec_potentially_empty_discard_old = ctx->cf_info.exec_potentially_empty_discard;
8747    ic->exec_potentially_empty_break_old = ctx->cf_info.exec_potentially_empty_break;
8748    ic->exec_potentially_empty_break_depth_old = ctx->cf_info.exec_potentially_empty_break_depth;
8749    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8750    ctx->cf_info.parent_if.is_divergent = true;
8751
8752    /* divergent branches use cbranch_execz */
8753    ctx->cf_info.exec_potentially_empty_discard = false;
8754    ctx->cf_info.exec_potentially_empty_break = false;
8755    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8756
8757    /** emit logical then block */
8758    Block* BB_then_logical = ctx->program->create_and_insert_block();
8759    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8760    add_edge(ic->BB_if_idx, BB_then_logical);
8761    ctx->block = BB_then_logical;
8762    append_logical_start(BB_then_logical);
8763 }
8764
8765 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8766 {
8767    Block *BB_then_logical = ctx->block;
8768    append_logical_end(BB_then_logical);
8769     /* branch from logical then block to invert block */
8770    aco_ptr<Pseudo_branch_instruction> branch;
8771    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8772    BB_then_logical->instructions.emplace_back(std::move(branch));
8773    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8774    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8775       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8776    BB_then_logical->kind |= block_kind_uniform;
8777    assert(!ctx->cf_info.has_branch);
8778    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8779    ctx->cf_info.parent_loop.has_divergent_branch = false;
8780
8781    /** emit linear then block */
8782    Block* BB_then_linear = ctx->program->create_and_insert_block();
8783    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8784    BB_then_linear->kind |= block_kind_uniform;
8785    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8786    /* branch from linear then block to invert block */
8787    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8788    BB_then_linear->instructions.emplace_back(std::move(branch));
8789    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8790
8791    /** emit invert merge block */
8792    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8793    ic->invert_idx = ctx->block->index;
8794
8795    /* branch to linear else block (skip else) */
8796    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8797    branch->operands[0] = Operand(ic->cond);
8798    ctx->block->instructions.push_back(std::move(branch));
8799
8800    ic->exec_potentially_empty_discard_old |= ctx->cf_info.exec_potentially_empty_discard;
8801    ic->exec_potentially_empty_break_old |= ctx->cf_info.exec_potentially_empty_break;
8802    ic->exec_potentially_empty_break_depth_old =
8803       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8804    /* divergent branches use cbranch_execz */
8805    ctx->cf_info.exec_potentially_empty_discard = false;
8806    ctx->cf_info.exec_potentially_empty_break = false;
8807    ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8808
8809    /** emit logical else block */
8810    Block* BB_else_logical = ctx->program->create_and_insert_block();
8811    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8812    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8813    add_linear_edge(ic->invert_idx, BB_else_logical);
8814    ctx->block = BB_else_logical;
8815    append_logical_start(BB_else_logical);
8816 }
8817
8818 static void end_divergent_if(isel_context *ctx, if_context *ic)
8819 {
8820    Block *BB_else_logical = ctx->block;
8821    append_logical_end(BB_else_logical);
8822
8823    /* branch from logical else block to endif block */
8824    aco_ptr<Pseudo_branch_instruction> branch;
8825    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8826    BB_else_logical->instructions.emplace_back(std::move(branch));
8827    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8828    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8829       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8830    BB_else_logical->kind |= block_kind_uniform;
8831
8832    assert(!ctx->cf_info.has_branch);
8833    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8834
8835
8836    /** emit linear else block */
8837    Block* BB_else_linear = ctx->program->create_and_insert_block();
8838    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8839    BB_else_linear->kind |= block_kind_uniform;
8840    add_linear_edge(ic->invert_idx, BB_else_linear);
8841
8842    /* branch from linear else block to endif block */
8843    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8844    BB_else_linear->instructions.emplace_back(std::move(branch));
8845    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8846
8847
8848    /** emit endif merge block */
8849    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8850    append_logical_start(ctx->block);
8851
8852
8853    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8854    ctx->cf_info.exec_potentially_empty_discard |= ic->exec_potentially_empty_discard_old;
8855    ctx->cf_info.exec_potentially_empty_break |= ic->exec_potentially_empty_break_old;
8856    ctx->cf_info.exec_potentially_empty_break_depth =
8857       std::min(ic->exec_potentially_empty_break_depth_old, ctx->cf_info.exec_potentially_empty_break_depth);
8858    if (ctx->cf_info.loop_nest_depth == ctx->cf_info.exec_potentially_empty_break_depth &&
8859        !ctx->cf_info.parent_if.is_divergent) {
8860       ctx->cf_info.exec_potentially_empty_break = false;
8861       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8862    }
8863    /* uniform control flow never has an empty exec-mask */
8864    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent) {
8865       ctx->cf_info.exec_potentially_empty_discard = false;
8866       ctx->cf_info.exec_potentially_empty_break = false;
8867       ctx->cf_info.exec_potentially_empty_break_depth = UINT16_MAX;
8868    }
8869 }
8870
8871 static bool visit_if(isel_context *ctx, nir_if *if_stmt)
8872 {
8873    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8874    Builder bld(ctx->program, ctx->block);
8875    aco_ptr<Pseudo_branch_instruction> branch;
8876
8877    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8878       /**
8879        * Uniform conditionals are represented in the following way*) :
8880        *
8881        * The linear and logical CFG:
8882        *                        BB_IF
8883        *                        /    \
8884        *       BB_THEN (logical)      BB_ELSE (logical)
8885        *                        \    /
8886        *                        BB_ENDIF
8887        *
8888        * *) Exceptions may be due to break and continue statements within loops
8889        *    If a break/continue happens within uniform control flow, it branches
8890        *    to the loop exit/entry block. Otherwise, it branches to the next
8891        *    merge block.
8892        **/
8893       append_logical_end(ctx->block);
8894       ctx->block->kind |= block_kind_uniform;
8895
8896       /* emit branch */
8897       assert(cond.regClass() == bld.lm);
8898       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8899       cond = bool_to_scalar_condition(ctx, cond);
8900
8901       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8902       branch->operands[0] = Operand(cond);
8903       branch->operands[0].setFixed(scc);
8904       ctx->block->instructions.emplace_back(std::move(branch));
8905
8906       unsigned BB_if_idx = ctx->block->index;
8907       Block BB_endif = Block();
8908       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8909       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8910
8911       /** emit then block */
8912       Block* BB_then = ctx->program->create_and_insert_block();
8913       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8914       add_edge(BB_if_idx, BB_then);
8915       append_logical_start(BB_then);
8916       ctx->block = BB_then;
8917       visit_cf_list(ctx, &if_stmt->then_list);
8918       BB_then = ctx->block;
8919       bool then_branch = ctx->cf_info.has_branch;
8920       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8921
8922       if (!then_branch) {
8923          append_logical_end(BB_then);
8924          /* branch from then block to endif block */
8925          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8926          BB_then->instructions.emplace_back(std::move(branch));
8927          add_linear_edge(BB_then->index, &BB_endif);
8928          if (!then_branch_divergent)
8929             add_logical_edge(BB_then->index, &BB_endif);
8930          BB_then->kind |= block_kind_uniform;
8931       }
8932
8933       ctx->cf_info.has_branch = false;
8934       ctx->cf_info.parent_loop.has_divergent_branch = false;
8935
8936       /** emit else block */
8937       Block* BB_else = ctx->program->create_and_insert_block();
8938       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8939       add_edge(BB_if_idx, BB_else);
8940       append_logical_start(BB_else);
8941       ctx->block = BB_else;
8942       visit_cf_list(ctx, &if_stmt->else_list);
8943       BB_else = ctx->block;
8944
8945       if (!ctx->cf_info.has_branch) {
8946          append_logical_end(BB_else);
8947          /* branch from then block to endif block */
8948          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8949          BB_else->instructions.emplace_back(std::move(branch));
8950          add_linear_edge(BB_else->index, &BB_endif);
8951          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8952             add_logical_edge(BB_else->index, &BB_endif);
8953          BB_else->kind |= block_kind_uniform;
8954       }
8955
8956       ctx->cf_info.has_branch &= then_branch;
8957       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8958
8959       /** emit endif merge block */
8960       if (!ctx->cf_info.has_branch) {
8961          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8962          append_logical_start(ctx->block);
8963       }
8964       return !ctx->cf_info.has_branch;
8965    } else { /* non-uniform condition */
8966       /**
8967        * To maintain a logical and linear CFG without critical edges,
8968        * non-uniform conditionals are represented in the following way*) :
8969        *
8970        * The linear CFG:
8971        *                        BB_IF
8972        *                        /    \
8973        *       BB_THEN (logical)      BB_THEN (linear)
8974        *                        \    /
8975        *                        BB_INVERT (linear)
8976        *                        /    \
8977        *       BB_ELSE (logical)      BB_ELSE (linear)
8978        *                        \    /
8979        *                        BB_ENDIF
8980        *
8981        * The logical CFG:
8982        *                        BB_IF
8983        *                        /    \
8984        *       BB_THEN (logical)      BB_ELSE (logical)
8985        *                        \    /
8986        *                        BB_ENDIF
8987        *
8988        * *) Exceptions may be due to break and continue statements within loops
8989        **/
8990
8991       if_context ic;
8992
8993       begin_divergent_if_then(ctx, &ic, cond);
8994       visit_cf_list(ctx, &if_stmt->then_list);
8995
8996       begin_divergent_if_else(ctx, &ic);
8997       visit_cf_list(ctx, &if_stmt->else_list);
8998
8999       end_divergent_if(ctx, &ic);
9000
9001       return true;
9002    }
9003 }
9004
9005 static bool visit_cf_list(isel_context *ctx,
9006                           struct exec_list *list)
9007 {
9008    foreach_list_typed(nir_cf_node, node, node, list) {
9009       switch (node->type) {
9010       case nir_cf_node_block:
9011          visit_block(ctx, nir_cf_node_as_block(node));
9012          break;
9013       case nir_cf_node_if:
9014          if (!visit_if(ctx, nir_cf_node_as_if(node)))
9015             return true;
9016          break;
9017       case nir_cf_node_loop:
9018          visit_loop(ctx, nir_cf_node_as_loop(node));
9019          break;
9020       default:
9021          unreachable("unimplemented cf list type");
9022       }
9023    }
9024    return false;
9025 }
9026
9027 static void create_null_export(isel_context *ctx)
9028 {
9029    /* Some shader stages always need to have exports.
9030     * So when there is none, we need to add a null export.
9031     */
9032
9033    unsigned dest = (ctx->program->stage & hw_fs) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS;
9034    bool vm = (ctx->program->stage & hw_fs) || ctx->program->chip_class >= GFX10;
9035    Builder bld(ctx->program, ctx->block);
9036    bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
9037            /* enabled_mask */ 0, dest, /* compr */ false, /* done */ true, vm);
9038 }
9039
9040 static bool export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
9041 {
9042    assert(ctx->stage == vertex_vs ||
9043           ctx->stage == tess_eval_vs ||
9044           ctx->stage == gs_copy_vs);
9045
9046    int offset = ctx->stage == tess_eval_vs
9047                 ? ctx->program->info->tes.outinfo.vs_output_param_offset[slot]
9048                 : ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
9049    uint64_t mask = ctx->outputs.mask[slot];
9050    if (!is_pos && !mask)
9051       return false;
9052    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
9053       return false;
9054    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9055    exp->enabled_mask = mask;
9056    for (unsigned i = 0; i < 4; ++i) {
9057       if (mask & (1 << i))
9058          exp->operands[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9059       else
9060          exp->operands[i] = Operand(v1);
9061    }
9062    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
9063     * Setting valid_mask=1 prevents it and has no other effect.
9064     */
9065    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
9066    exp->done = false;
9067    exp->compressed = false;
9068    if (is_pos)
9069       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9070    else
9071       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
9072    ctx->block->instructions.emplace_back(std::move(exp));
9073
9074    return true;
9075 }
9076
9077 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
9078 {
9079    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
9080    exp->enabled_mask = 0;
9081    for (unsigned i = 0; i < 4; ++i)
9082       exp->operands[i] = Operand(v1);
9083    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
9084       exp->operands[0] = Operand(ctx->outputs.temps[VARYING_SLOT_PSIZ * 4u]);
9085       exp->enabled_mask |= 0x1;
9086    }
9087    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
9088       exp->operands[2] = Operand(ctx->outputs.temps[VARYING_SLOT_LAYER * 4u]);
9089       exp->enabled_mask |= 0x4;
9090    }
9091    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
9092       if (ctx->options->chip_class < GFX9) {
9093          exp->operands[3] = Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]);
9094          exp->enabled_mask |= 0x8;
9095       } else {
9096          Builder bld(ctx->program, ctx->block);
9097
9098          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
9099                              Operand(ctx->outputs.temps[VARYING_SLOT_VIEWPORT * 4u]));
9100          if (exp->operands[2].isTemp())
9101             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
9102
9103          exp->operands[2] = Operand(out);
9104          exp->enabled_mask |= 0x4;
9105       }
9106    }
9107    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
9108    exp->done = false;
9109    exp->compressed = false;
9110    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
9111    ctx->block->instructions.emplace_back(std::move(exp));
9112 }
9113
9114 static void create_vs_exports(isel_context *ctx)
9115 {
9116    assert(ctx->stage == vertex_vs ||
9117           ctx->stage == tess_eval_vs ||
9118           ctx->stage == gs_copy_vs);
9119
9120    radv_vs_output_info *outinfo = ctx->stage == tess_eval_vs
9121                                   ? &ctx->program->info->tes.outinfo
9122                                   : &ctx->program->info->vs.outinfo;
9123
9124    if (outinfo->export_prim_id) {
9125       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
9126       ctx->outputs.temps[VARYING_SLOT_PRIMITIVE_ID * 4u] = get_arg(ctx, ctx->args->vs_prim_id);
9127    }
9128
9129    if (ctx->options->key.has_multiview_view_index) {
9130       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
9131       ctx->outputs.temps[VARYING_SLOT_LAYER * 4u] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
9132    }
9133
9134    /* the order these position exports are created is important */
9135    int next_pos = 0;
9136    bool exported_pos = export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
9137    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
9138       export_vs_psiz_layer_viewport(ctx, &next_pos);
9139       exported_pos = true;
9140    }
9141    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9142       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
9143    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9144       exported_pos |= export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
9145
9146    if (ctx->export_clip_dists) {
9147       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
9148          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
9149       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
9150          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
9151    }
9152
9153    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9154       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
9155           i != VARYING_SLOT_PRIMITIVE_ID)
9156          continue;
9157
9158       export_vs_varying(ctx, i, false, NULL);
9159    }
9160
9161    if (!exported_pos)
9162       create_null_export(ctx);
9163 }
9164
9165 static bool export_fs_mrt_z(isel_context *ctx)
9166 {
9167    Builder bld(ctx->program, ctx->block);
9168    unsigned enabled_channels = 0;
9169    bool compr = false;
9170    Operand values[4];
9171
9172    for (unsigned i = 0; i < 4; ++i) {
9173       values[i] = Operand(v1);
9174    }
9175
9176    /* Both stencil and sample mask only need 16-bits. */
9177    if (!ctx->program->info->ps.writes_z &&
9178        (ctx->program->info->ps.writes_stencil ||
9179         ctx->program->info->ps.writes_sample_mask)) {
9180       compr = true; /* COMPR flag */
9181
9182       if (ctx->program->info->ps.writes_stencil) {
9183          /* Stencil should be in X[23:16]. */
9184          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9185          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
9186          enabled_channels |= 0x3;
9187       }
9188
9189       if (ctx->program->info->ps.writes_sample_mask) {
9190          /* SampleMask should be in Y[15:0]. */
9191          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9192          enabled_channels |= 0xc;
9193      }
9194    } else {
9195       if (ctx->program->info->ps.writes_z) {
9196          values[0] = Operand(ctx->outputs.temps[FRAG_RESULT_DEPTH * 4u]);
9197          enabled_channels |= 0x1;
9198       }
9199
9200       if (ctx->program->info->ps.writes_stencil) {
9201          values[1] = Operand(ctx->outputs.temps[FRAG_RESULT_STENCIL * 4u]);
9202          enabled_channels |= 0x2;
9203       }
9204
9205       if (ctx->program->info->ps.writes_sample_mask) {
9206          values[2] = Operand(ctx->outputs.temps[FRAG_RESULT_SAMPLE_MASK * 4u]);
9207          enabled_channels |= 0x4;
9208       }
9209    }
9210
9211    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
9212     * writemask component.
9213     */
9214    if (ctx->options->chip_class == GFX6 &&
9215        ctx->options->family != CHIP_OLAND &&
9216        ctx->options->family != CHIP_HAINAN) {
9217             enabled_channels |= 0x1;
9218    }
9219
9220    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9221            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
9222
9223    return true;
9224 }
9225
9226 static bool export_fs_mrt_color(isel_context *ctx, int slot)
9227 {
9228    Builder bld(ctx->program, ctx->block);
9229    unsigned write_mask = ctx->outputs.mask[slot];
9230    Operand values[4];
9231
9232    for (unsigned i = 0; i < 4; ++i) {
9233       if (write_mask & (1 << i)) {
9234          values[i] = Operand(ctx->outputs.temps[slot * 4u + i]);
9235       } else {
9236          values[i] = Operand(v1);
9237       }
9238    }
9239
9240    unsigned target, col_format;
9241    unsigned enabled_channels = 0;
9242    aco_opcode compr_op = (aco_opcode)0;
9243
9244    slot -= FRAG_RESULT_DATA0;
9245    target = V_008DFC_SQ_EXP_MRT + slot;
9246    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
9247
9248    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
9249    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
9250
9251    switch (col_format)
9252    {
9253    case V_028714_SPI_SHADER_ZERO:
9254       enabled_channels = 0; /* writemask */
9255       target = V_008DFC_SQ_EXP_NULL;
9256       break;
9257
9258    case V_028714_SPI_SHADER_32_R:
9259       enabled_channels = 1;
9260       break;
9261
9262    case V_028714_SPI_SHADER_32_GR:
9263       enabled_channels = 0x3;
9264       break;
9265
9266    case V_028714_SPI_SHADER_32_AR:
9267       if (ctx->options->chip_class >= GFX10) {
9268          /* Special case: on GFX10, the outputs are different for 32_AR */
9269          enabled_channels = 0x3;
9270          values[1] = values[3];
9271          values[3] = Operand(v1);
9272       } else {
9273          enabled_channels = 0x9;
9274       }
9275       break;
9276
9277    case V_028714_SPI_SHADER_FP16_ABGR:
9278       enabled_channels = 0x5;
9279       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
9280       break;
9281
9282    case V_028714_SPI_SHADER_UNORM16_ABGR:
9283       enabled_channels = 0x5;
9284       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
9285       break;
9286
9287    case V_028714_SPI_SHADER_SNORM16_ABGR:
9288       enabled_channels = 0x5;
9289       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
9290       break;
9291
9292    case V_028714_SPI_SHADER_UINT16_ABGR: {
9293       enabled_channels = 0x5;
9294       compr_op = aco_opcode::v_cvt_pk_u16_u32;
9295       if (is_int8 || is_int10) {
9296          /* clamp */
9297          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
9298          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9299
9300          for (unsigned i = 0; i < 4; i++) {
9301             if ((write_mask >> i) & 1) {
9302                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
9303                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
9304                                     values[i]);
9305             }
9306          }
9307       }
9308       break;
9309    }
9310
9311    case V_028714_SPI_SHADER_SINT16_ABGR:
9312       enabled_channels = 0x5;
9313       compr_op = aco_opcode::v_cvt_pk_i16_i32;
9314       if (is_int8 || is_int10) {
9315          /* clamp */
9316          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
9317          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
9318          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
9319          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
9320
9321          for (unsigned i = 0; i < 4; i++) {
9322             if ((write_mask >> i) & 1) {
9323                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
9324                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
9325                                     values[i]);
9326                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
9327                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
9328                                     values[i]);
9329             }
9330          }
9331       }
9332       break;
9333
9334    case V_028714_SPI_SHADER_32_ABGR:
9335       enabled_channels = 0xF;
9336       break;
9337
9338    default:
9339       break;
9340    }
9341
9342    if (target == V_008DFC_SQ_EXP_NULL)
9343       return false;
9344
9345    if ((bool) compr_op) {
9346       for (int i = 0; i < 2; i++) {
9347          /* check if at least one of the values to be compressed is enabled */
9348          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
9349          if (enabled) {
9350             enabled_channels |= enabled << (i*2);
9351             values[i] = bld.vop3(compr_op, bld.def(v1),
9352                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
9353                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
9354          } else {
9355             values[i] = Operand(v1);
9356          }
9357       }
9358       values[2] = Operand(v1);
9359       values[3] = Operand(v1);
9360    } else {
9361       for (int i = 0; i < 4; i++)
9362          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
9363    }
9364
9365    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
9366            enabled_channels, target, (bool) compr_op);
9367    return true;
9368 }
9369
9370 static void create_fs_exports(isel_context *ctx)
9371 {
9372    bool exported = false;
9373
9374    /* Export depth, stencil and sample mask. */
9375    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
9376        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
9377        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK])
9378       exported |= export_fs_mrt_z(ctx);
9379
9380    /* Export all color render targets. */
9381    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i)
9382       if (ctx->outputs.mask[i])
9383          exported |= export_fs_mrt_color(ctx, i);
9384
9385    if (!exported)
9386       create_null_export(ctx);
9387 }
9388
9389 static void write_tcs_tess_factors(isel_context *ctx)
9390 {
9391    unsigned outer_comps;
9392    unsigned inner_comps;
9393
9394    switch (ctx->args->options->key.tcs.primitive_mode) {
9395    case GL_ISOLINES:
9396       outer_comps = 2;
9397       inner_comps = 0;
9398       break;
9399    case GL_TRIANGLES:
9400       outer_comps = 3;
9401       inner_comps = 1;
9402       break;
9403    case GL_QUADS:
9404       outer_comps = 4;
9405       inner_comps = 2;
9406       break;
9407    default:
9408       return;
9409    }
9410
9411    Builder bld(ctx->program, ctx->block);
9412
9413    bld.barrier(aco_opcode::p_memory_barrier_shared);
9414    if (unlikely(ctx->program->chip_class != GFX6 && ctx->program->workgroup_size > ctx->program->wave_size))
9415       bld.sopp(aco_opcode::s_barrier);
9416
9417    Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids);
9418    Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u));
9419
9420    Temp invocation_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), invocation_id);
9421    if_context ic_invocation_id_is_zero;
9422    begin_divergent_if_then(ctx, &ic_invocation_id_is_zero, invocation_id_is_zero);
9423    bld.reset(ctx->block);
9424
9425    Temp hs_ring_tess_factor = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_FACTOR * 16u));
9426
9427    std::pair<Temp, unsigned> lds_base = get_tcs_output_lds_offset(ctx);
9428    unsigned stride = inner_comps + outer_comps;
9429    unsigned lds_align = calculate_lds_alignment(ctx, lds_base.second);
9430    Temp tf_inner_vec;
9431    Temp tf_outer_vec;
9432    Temp out[6];
9433    assert(stride <= (sizeof(out) / sizeof(Temp)));
9434
9435    if (ctx->args->options->key.tcs.primitive_mode == GL_ISOLINES) {
9436       // LINES reversal
9437       tf_outer_vec = load_lds(ctx, 4, bld.tmp(v2), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
9438       out[1] = emit_extract_vector(ctx, tf_outer_vec, 0, v1);
9439       out[0] = emit_extract_vector(ctx, tf_outer_vec, 1, v1);
9440    } else {
9441       tf_outer_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, outer_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_out_loc, lds_align);
9442       tf_inner_vec = load_lds(ctx, 4, bld.tmp(RegClass(RegType::vgpr, inner_comps)), lds_base.first, lds_base.second + ctx->tcs_tess_lvl_in_loc, lds_align);
9443
9444       for (unsigned i = 0; i < outer_comps; ++i)
9445          out[i] = emit_extract_vector(ctx, tf_outer_vec, i, v1);
9446       for (unsigned i = 0; i < inner_comps; ++i)
9447          out[outer_comps + i] = emit_extract_vector(ctx, tf_inner_vec, i, v1);
9448    }
9449
9450    Temp rel_patch_id = get_tess_rel_patch_id(ctx);
9451    Temp tf_base = get_arg(ctx, ctx->args->tess_factor_offset);
9452    Temp byte_offset = bld.v_mul_imm(bld.def(v1), rel_patch_id, stride * 4u);
9453    unsigned tf_const_offset = 0;
9454
9455    if (ctx->program->chip_class <= GFX8) {
9456       Temp rel_patch_id_is_zero = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), rel_patch_id);
9457       if_context ic_rel_patch_id_is_zero;
9458       begin_divergent_if_then(ctx, &ic_rel_patch_id_is_zero, rel_patch_id_is_zero);
9459       bld.reset(ctx->block);
9460
9461       /* Store the dynamic HS control word. */
9462       Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u));
9463       bld.mubuf(aco_opcode::buffer_store_dword,
9464                 /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word,
9465                 /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false,
9466                 /* disable_wqm */ false, /* glc */ true);
9467       tf_const_offset += 4;
9468
9469       begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero);
9470       end_divergent_if(ctx, &ic_rel_patch_id_is_zero);
9471       bld.reset(ctx->block);
9472    }
9473
9474    assert(stride == 2 || stride == 4 || stride == 6);
9475    Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr, 4u);
9476    store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false);
9477
9478    /* Store to offchip for TES to read - only if TES reads them */
9479    if (ctx->args->options->key.tcs.tes_reads_tess_factors) {
9480       Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u));
9481       Temp oc_lds = get_arg(ctx, ctx->args->oc_lds);
9482
9483       std::pair<Temp, unsigned> vmem_offs_outer = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_out_loc);
9484       store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, false);
9485
9486       if (likely(inner_comps)) {
9487          std::pair<Temp, unsigned> vmem_offs_inner = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_in_loc);
9488          store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, false);
9489       }
9490    }
9491
9492    begin_divergent_if_else(ctx, &ic_invocation_id_is_zero);
9493    end_divergent_if(ctx, &ic_invocation_id_is_zero);
9494 }
9495
9496 static void emit_stream_output(isel_context *ctx,
9497                                Temp const *so_buffers,
9498                                Temp const *so_write_offset,
9499                                const struct radv_stream_output *output)
9500 {
9501    unsigned num_comps = util_bitcount(output->component_mask);
9502    unsigned writemask = (1 << num_comps) - 1;
9503    unsigned loc = output->location;
9504    unsigned buf = output->buffer;
9505
9506    assert(num_comps && num_comps <= 4);
9507    if (!num_comps || num_comps > 4)
9508       return;
9509
9510    unsigned start = ffs(output->component_mask) - 1;
9511
9512    Temp out[4];
9513    bool all_undef = true;
9514    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
9515    for (unsigned i = 0; i < num_comps; i++) {
9516       out[i] = ctx->outputs.temps[loc * 4 + start + i];
9517       all_undef = all_undef && !out[i].id();
9518    }
9519    if (all_undef)
9520       return;
9521
9522    while (writemask) {
9523       int start, count;
9524       u_bit_scan_consecutive_range(&writemask, &start, &count);
9525       if (count == 3 && ctx->options->chip_class == GFX6) {
9526          /* GFX6 doesn't support storing vec3, split it. */
9527          writemask |= 1u << (start + 2);
9528          count = 2;
9529       }
9530
9531       unsigned offset = output->offset + start * 4;
9532
9533       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
9534       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
9535       for (int i = 0; i < count; ++i)
9536          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
9537       vec->definitions[0] = Definition(write_data);
9538       ctx->block->instructions.emplace_back(std::move(vec));
9539
9540       aco_opcode opcode;
9541       switch (count) {
9542       case 1:
9543          opcode = aco_opcode::buffer_store_dword;
9544          break;
9545       case 2:
9546          opcode = aco_opcode::buffer_store_dwordx2;
9547          break;
9548       case 3:
9549          opcode = aco_opcode::buffer_store_dwordx3;
9550          break;
9551       case 4:
9552          opcode = aco_opcode::buffer_store_dwordx4;
9553          break;
9554       default:
9555          unreachable("Unsupported dword count.");
9556       }
9557
9558       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
9559       store->operands[0] = Operand(so_buffers[buf]);
9560       store->operands[1] = Operand(so_write_offset[buf]);
9561       store->operands[2] = Operand((uint32_t) 0);
9562       store->operands[3] = Operand(write_data);
9563       if (offset > 4095) {
9564          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
9565          Builder bld(ctx->program, ctx->block);
9566          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
9567       } else {
9568          store->offset = offset;
9569       }
9570       store->offen = true;
9571       store->glc = true;
9572       store->dlc = false;
9573       store->slc = true;
9574       store->can_reorder = true;
9575       ctx->block->instructions.emplace_back(std::move(store));
9576    }
9577 }
9578
9579 static void emit_streamout(isel_context *ctx, unsigned stream)
9580 {
9581    Builder bld(ctx->program, ctx->block);
9582
9583    Temp so_buffers[4];
9584    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
9585    for (unsigned i = 0; i < 4; i++) {
9586       unsigned stride = ctx->program->info->so.strides[i];
9587       if (!stride)
9588          continue;
9589
9590       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
9591       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
9592    }
9593
9594    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9595                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
9596
9597    Temp tid = emit_mbcnt(ctx, bld.def(v1));
9598
9599    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
9600
9601    if_context ic;
9602    begin_divergent_if_then(ctx, &ic, can_emit);
9603
9604    bld.reset(ctx->block);
9605
9606    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
9607
9608    Temp so_write_offset[4];
9609
9610    for (unsigned i = 0; i < 4; i++) {
9611       unsigned stride = ctx->program->info->so.strides[i];
9612       if (!stride)
9613          continue;
9614
9615       if (stride == 1) {
9616          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
9617                                 get_arg(ctx, ctx->args->streamout_write_idx),
9618                                 get_arg(ctx, ctx->args->streamout_offset[i]));
9619          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
9620
9621          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
9622       } else {
9623          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
9624          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
9625                                  get_arg(ctx, ctx->args->streamout_offset[i]));
9626          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
9627       }
9628    }
9629
9630    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
9631       struct radv_stream_output *output =
9632          &ctx->program->info->so.outputs[i];
9633       if (stream != output->stream)
9634          continue;
9635
9636       emit_stream_output(ctx, so_buffers, so_write_offset, output);
9637    }
9638
9639    begin_divergent_if_else(ctx, &ic);
9640    end_divergent_if(ctx, &ic);
9641 }
9642
9643 } /* end namespace */
9644
9645 void fix_ls_vgpr_init_bug(isel_context *ctx, Pseudo_instruction *startpgm)
9646 {
9647    assert(ctx->shader->info.stage == MESA_SHADER_VERTEX);
9648    Builder bld(ctx->program, ctx->block);
9649    constexpr unsigned hs_idx = 1u;
9650    Builder::Result hs_thread_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9651                                               get_arg(ctx, ctx->args->merged_wave_info),
9652                                               Operand((8u << 16) | (hs_idx * 8u)));
9653    Temp ls_has_nonzero_hs_threads = bool_to_vector_condition(ctx, hs_thread_count.def(1).getTemp());
9654
9655    /* If there are no HS threads, SPI mistakenly loads the LS VGPRs starting at VGPR 0. */
9656
9657    Temp instance_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9658                                get_arg(ctx, ctx->args->rel_auto_id),
9659                                get_arg(ctx, ctx->args->ac.instance_id),
9660                                ls_has_nonzero_hs_threads);
9661    Temp rel_auto_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9662                                get_arg(ctx, ctx->args->ac.tcs_rel_ids),
9663                                get_arg(ctx, ctx->args->rel_auto_id),
9664                                ls_has_nonzero_hs_threads);
9665    Temp vertex_id = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9666                              get_arg(ctx, ctx->args->ac.tcs_patch_id),
9667                              get_arg(ctx, ctx->args->ac.vertex_id),
9668                              ls_has_nonzero_hs_threads);
9669
9670    ctx->arg_temps[ctx->args->ac.instance_id.arg_index] = instance_id;
9671    ctx->arg_temps[ctx->args->rel_auto_id.arg_index] = rel_auto_id;
9672    ctx->arg_temps[ctx->args->ac.vertex_id.arg_index] = vertex_id;
9673 }
9674
9675 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
9676 {
9677    /* Split all arguments except for the first (ring_offsets) and the last
9678     * (exec) so that the dead channels don't stay live throughout the program.
9679     */
9680    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
9681       if (startpgm->definitions[i].regClass().size() > 1) {
9682          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
9683                            startpgm->definitions[i].regClass().size());
9684       }
9685    }
9686 }
9687
9688 void handle_bc_optimize(isel_context *ctx)
9689 {
9690    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
9691    Builder bld(ctx->program, ctx->block);
9692    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
9693    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
9694    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
9695    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
9696    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
9697    if (uses_center && uses_centroid) {
9698       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
9699                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
9700
9701       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
9702          Temp new_coord[2];
9703          for (unsigned i = 0; i < 2; i++) {
9704             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
9705             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
9706             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9707                                     persp_centroid, persp_center, sel);
9708          }
9709          ctx->persp_centroid = bld.tmp(v2);
9710          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
9711                     Operand(new_coord[0]), Operand(new_coord[1]));
9712          emit_split_vector(ctx, ctx->persp_centroid, 2);
9713       }
9714
9715       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
9716          Temp new_coord[2];
9717          for (unsigned i = 0; i < 2; i++) {
9718             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
9719             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
9720             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
9721                                     linear_centroid, linear_center, sel);
9722          }
9723          ctx->linear_centroid = bld.tmp(v2);
9724          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
9725                     Operand(new_coord[0]), Operand(new_coord[1]));
9726          emit_split_vector(ctx, ctx->linear_centroid, 2);
9727       }
9728    }
9729 }
9730
9731 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
9732 {
9733    Program *program = ctx->program;
9734
9735    unsigned float_controls = shader->info.float_controls_execution_mode;
9736
9737    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
9738       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
9739    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
9740       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
9741                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
9742
9743    program->next_fp_mode.must_flush_denorms32 =
9744       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
9745    program->next_fp_mode.must_flush_denorms16_64 =
9746       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
9747                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
9748
9749    program->next_fp_mode.care_about_round32 =
9750       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
9751
9752    program->next_fp_mode.care_about_round16_64 =
9753       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
9754                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
9755
9756    /* default to preserving fp16 and fp64 denorms, since it's free */
9757    if (program->next_fp_mode.must_flush_denorms16_64)
9758       program->next_fp_mode.denorm16_64 = 0;
9759    else
9760       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9761
9762    /* preserving fp32 denorms is expensive, so only do it if asked */
9763    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
9764       program->next_fp_mode.denorm32 = fp_denorm_keep;
9765    else
9766       program->next_fp_mode.denorm32 = 0;
9767
9768    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
9769       program->next_fp_mode.round32 = fp_round_tz;
9770    else
9771       program->next_fp_mode.round32 = fp_round_ne;
9772
9773    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
9774       program->next_fp_mode.round16_64 = fp_round_tz;
9775    else
9776       program->next_fp_mode.round16_64 = fp_round_ne;
9777
9778    ctx->block->fp_mode = program->next_fp_mode;
9779 }
9780
9781 void cleanup_cfg(Program *program)
9782 {
9783    /* create linear_succs/logical_succs */
9784    for (Block& BB : program->blocks) {
9785       for (unsigned idx : BB.linear_preds)
9786          program->blocks[idx].linear_succs.emplace_back(BB.index);
9787       for (unsigned idx : BB.logical_preds)
9788          program->blocks[idx].logical_succs.emplace_back(BB.index);
9789    }
9790 }
9791
9792 void select_program(Program *program,
9793                     unsigned shader_count,
9794                     struct nir_shader *const *shaders,
9795                     ac_shader_config* config,
9796                     struct radv_shader_args *args)
9797 {
9798    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
9799    if_context ic_merged_wave_info;
9800
9801    for (unsigned i = 0; i < shader_count; i++) {
9802       nir_shader *nir = shaders[i];
9803       init_context(&ctx, nir);
9804
9805       setup_fp_mode(&ctx, nir);
9806
9807       if (!i) {
9808          /* needs to be after init_context() for FS */
9809          Pseudo_instruction *startpgm = add_startpgm(&ctx);
9810          append_logical_start(ctx.block);
9811
9812          if (unlikely(args->options->has_ls_vgpr_init_bug && ctx.stage == vertex_tess_control_hs))
9813             fix_ls_vgpr_init_bug(&ctx, startpgm);
9814
9815          split_arguments(&ctx, startpgm);
9816       }
9817
9818       /* In a merged VS+TCS HS, the VS implementation can be completely empty. */
9819       nir_function_impl *func = nir_shader_get_entrypoint(nir);
9820       bool empty_shader = nir_cf_list_is_empty_block(&func->body) &&
9821                           ((nir->info.stage == MESA_SHADER_VERTEX &&
9822                             (ctx.stage == vertex_tess_control_hs || ctx.stage == vertex_geometry_gs)) ||
9823                            (nir->info.stage == MESA_SHADER_TESS_EVAL &&
9824                             ctx.stage == tess_eval_geometry_gs));
9825
9826       bool check_merged_wave_info = ctx.tcs_in_out_eq ? i == 0 : (shader_count >= 2 && !empty_shader);
9827       bool endif_merged_wave_info = ctx.tcs_in_out_eq ? i == 1 : check_merged_wave_info;
9828       if (check_merged_wave_info) {
9829          Builder bld(ctx.program, ctx.block);
9830
9831          /* The s_bfm only cares about s0.u[5:0] so we don't need either s_bfe nor s_and here */
9832          Temp count = i == 0 ? get_arg(&ctx, args->merged_wave_info)
9833                              : bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc),
9834                                         get_arg(&ctx, args->merged_wave_info), Operand(i * 8u));
9835
9836          Temp mask = bld.sop2(aco_opcode::s_bfm_b64, bld.def(s2), count, Operand(0u));
9837          Temp cond;
9838
9839          if (ctx.program->wave_size == 64) {
9840             /* Special case for 64 active invocations, because 64 doesn't work with s_bfm */
9841             Temp active_64 = bld.sopc(aco_opcode::s_bitcmp1_b32, bld.def(s1, scc), count, Operand(6u /* log2(64) */));
9842             cond = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), mask, bld.scc(active_64));
9843          } else {
9844             /* We use s_bfm_b64 (not _b32) which works with 32, but we need to extract the lower half of the register */
9845             cond = emit_extract_vector(&ctx, mask, 0, bld.lm);
9846          }
9847
9848          begin_divergent_if_then(&ctx, &ic_merged_wave_info, cond);
9849       }
9850
9851       if (i) {
9852          Builder bld(ctx.program, ctx.block);
9853
9854          bld.barrier(aco_opcode::p_memory_barrier_shared);
9855          bld.sopp(aco_opcode::s_barrier);
9856
9857          if (ctx.stage == vertex_geometry_gs || ctx.stage == tess_eval_geometry_gs) {
9858             ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
9859          }
9860       } else if (ctx.stage == geometry_gs)
9861          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
9862
9863       if (ctx.stage == fragment_fs)
9864          handle_bc_optimize(&ctx);
9865
9866       visit_cf_list(&ctx, &func->body);
9867
9868       if (ctx.program->info->so.num_outputs && (ctx.stage == vertex_vs || ctx.stage == tess_eval_vs))
9869          emit_streamout(&ctx, 0);
9870
9871       if (ctx.stage == vertex_vs || ctx.stage == tess_eval_vs) {
9872          create_vs_exports(&ctx);
9873       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
9874          Builder bld(ctx.program, ctx.block);
9875          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
9876          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
9877       } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
9878          write_tcs_tess_factors(&ctx);
9879       }
9880
9881       if (ctx.stage == fragment_fs)
9882          create_fs_exports(&ctx);
9883
9884       if (endif_merged_wave_info) {
9885          begin_divergent_if_else(&ctx, &ic_merged_wave_info);
9886          end_divergent_if(&ctx, &ic_merged_wave_info);
9887       }
9888
9889       ralloc_free(ctx.divergent_vals);
9890
9891       if (i == 0 && ctx.stage == vertex_tess_control_hs && ctx.tcs_in_out_eq) {
9892          /* Outputs of the previous stage are inputs to the next stage */
9893          ctx.inputs = ctx.outputs;
9894          ctx.outputs = shader_io_state();
9895       }
9896    }
9897
9898    program->config->float_mode = program->blocks[0].fp_mode.val;
9899
9900    append_logical_end(ctx.block);
9901    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
9902    Builder bld(ctx.program, ctx.block);
9903    if (ctx.program->wb_smem_l1_on_end)
9904       bld.smem(aco_opcode::s_dcache_wb, false);
9905    bld.sopp(aco_opcode::s_endpgm);
9906
9907    cleanup_cfg(program);
9908 }
9909
9910 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
9911                            ac_shader_config* config,
9912                            struct radv_shader_args *args)
9913 {
9914    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
9915
9916    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
9917    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
9918    program->next_fp_mode.must_flush_denorms32 = false;
9919    program->next_fp_mode.must_flush_denorms16_64 = false;
9920    program->next_fp_mode.care_about_round32 = false;
9921    program->next_fp_mode.care_about_round16_64 = false;
9922    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9923    program->next_fp_mode.denorm32 = 0;
9924    program->next_fp_mode.round32 = fp_round_ne;
9925    program->next_fp_mode.round16_64 = fp_round_ne;
9926    ctx.block->fp_mode = program->next_fp_mode;
9927
9928    add_startpgm(&ctx);
9929    append_logical_start(ctx.block);
9930
9931    Builder bld(ctx.program, ctx.block);
9932
9933    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
9934
9935    Operand stream_id(0u);
9936    if (args->shader_info->so.num_outputs)
9937       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9938                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
9939
9940    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9941
9942    std::stack<Block> endif_blocks;
9943
9944    for (unsigned stream = 0; stream < 4; stream++) {
9945       if (stream_id.isConstant() && stream != stream_id.constantValue())
9946          continue;
9947
9948       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9949       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9950          continue;
9951
9952       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9953
9954       unsigned BB_if_idx = ctx.block->index;
9955       Block BB_endif = Block();
9956       if (!stream_id.isConstant()) {
9957          /* begin IF */
9958          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9959          append_logical_end(ctx.block);
9960          ctx.block->kind |= block_kind_uniform;
9961          bld.branch(aco_opcode::p_cbranch_z, cond);
9962
9963          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9964
9965          ctx.block = ctx.program->create_and_insert_block();
9966          add_edge(BB_if_idx, ctx.block);
9967          bld.reset(ctx.block);
9968          append_logical_start(ctx.block);
9969       }
9970
9971       unsigned offset = 0;
9972       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9973          if (args->shader_info->gs.output_streams[i] != stream)
9974             continue;
9975
9976          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9977          unsigned length = util_last_bit(output_usage_mask);
9978          for (unsigned j = 0; j < length; ++j) {
9979             if (!(output_usage_mask & (1 << j)))
9980                continue;
9981
9982             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9983             Temp voffset = vtx_offset;
9984             if (const_offset >= 4096u) {
9985                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9986                const_offset %= 4096u;
9987             }
9988
9989             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9990             mubuf->definitions[0] = bld.def(v1);
9991             mubuf->operands[0] = Operand(gsvs_ring);
9992             mubuf->operands[1] = Operand(voffset);
9993             mubuf->operands[2] = Operand(0u);
9994             mubuf->offen = true;
9995             mubuf->offset = const_offset;
9996             mubuf->glc = true;
9997             mubuf->slc = true;
9998             mubuf->dlc = args->options->chip_class >= GFX10;
9999             mubuf->barrier = barrier_none;
10000             mubuf->can_reorder = true;
10001
10002             ctx.outputs.mask[i] |= 1 << j;
10003             ctx.outputs.temps[i * 4u + j] = mubuf->definitions[0].getTemp();
10004
10005             bld.insert(std::move(mubuf));
10006
10007             offset++;
10008          }
10009       }
10010
10011       if (args->shader_info->so.num_outputs) {
10012          emit_streamout(&ctx, stream);
10013          bld.reset(ctx.block);
10014       }
10015
10016       if (stream == 0) {
10017          create_vs_exports(&ctx);
10018          ctx.block->kind |= block_kind_export_end;
10019       }
10020
10021       if (!stream_id.isConstant()) {
10022          append_logical_end(ctx.block);
10023
10024          /* branch from then block to endif block */
10025          bld.branch(aco_opcode::p_branch);
10026          add_edge(ctx.block->index, &BB_endif);
10027          ctx.block->kind |= block_kind_uniform;
10028
10029          /* emit else block */
10030          ctx.block = ctx.program->create_and_insert_block();
10031          add_edge(BB_if_idx, ctx.block);
10032          bld.reset(ctx.block);
10033          append_logical_start(ctx.block);
10034
10035          endif_blocks.push(std::move(BB_endif));
10036       }
10037    }
10038
10039    while (!endif_blocks.empty()) {
10040       Block BB_endif = std::move(endif_blocks.top());
10041       endif_blocks.pop();
10042
10043       Block *BB_else = ctx.block;
10044
10045       append_logical_end(BB_else);
10046       /* branch from else block to endif block */
10047       bld.branch(aco_opcode::p_branch);
10048       add_edge(BB_else->index, &BB_endif);
10049       BB_else->kind |= block_kind_uniform;
10050
10051       /** emit endif merge block */
10052       ctx.block = program->insert_block(std::move(BB_endif));
10053       bld.reset(ctx.block);
10054       append_logical_start(ctx.block);
10055    }
10056
10057    program->config->float_mode = program->blocks[0].fp_mode.val;
10058
10059    append_logical_end(ctx.block);
10060    ctx.block->kind |= block_kind_uniform;
10061    bld.sopp(aco_opcode::s_endpgm);
10062
10063    cleanup_cfg(program);
10064 }
10065 }