src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_old;
  83
  84    unsigned BB_if_idx;
  85    unsigned invert_idx;
  86    bool then_branch_divergent;
  87    Block BB_invert;
  88    Block BB_endif;
  89 };
  90
  91 static void visit_cf_list(struct isel_context *ctx,
  92                           struct exec_list *list);
  93
  94 static void add_logical_edge(unsigned pred_idx, Block *succ)
  95 {
  96    succ->logical_preds.emplace_back(pred_idx);
  97 }
  98
  99
 100 static void add_linear_edge(unsigned pred_idx, Block *succ)
 101 {
 102    succ->linear_preds.emplace_back(pred_idx);
 103 }
 104
 105 static void add_edge(unsigned pred_idx, Block *succ)
 106 {
 107    add_logical_edge(pred_idx, succ);
 108    add_linear_edge(pred_idx, succ);
 109 }
 110
 111 static void append_logical_start(Block *b)
 112 {
 113    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 114 }
 115
 116 static void append_logical_end(Block *b)
 117 {
 118    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 119 }
 120
 121 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 122 {
 123    assert(ctx->allocated[def->index].id());
 124    return ctx->allocated[def->index];
 125 }
 126
 127 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 128                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 129 {
 130    Builder bld(ctx->program, ctx->block);
 131    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 132    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 133
 134    if (ctx->program->wave_size == 32) {
 135       return thread_id_lo;
 136    } else {
 137       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 138       return thread_id_hi;
 139    }
 140 }
 141
 142 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 143 {
 144    Builder bld(ctx->program, ctx->block);
 145
 146    if (!dst.id())
 147       dst = bld.tmp(src.regClass());
 148
 149    assert(src.size() == dst.size());
 150
 151    if (ctx->stage != fragment_fs) {
 152       if (!dst.id())
 153          return src;
 154
 155       bld.copy(Definition(dst), src);
 156       return dst;
 157    }
 158
 159    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 160    ctx->program->needs_wqm |= program_needs_wqm;
 161    return dst;
 162 }
 163
 164 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 165 {
 166    if (index.regClass() == s1)
 167       return bld.readlane(bld.def(s1), data, index);
 168
 169    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 170
 171    /* Currently not implemented on GFX6-7 */
 172    assert(ctx->options->chip_class >= GFX8);
 173
 174    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 175       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 176    }
 177
 178    /* GFX10, wave64 mode:
 179     * The bpermute instruction is limited to half-wave operation, which means that it can't
 180     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 181     * emulate it here.
 182     */
 183    if (!ctx->has_gfx10_wave64_bpermute) {
 184       ctx->has_gfx10_wave64_bpermute = true;
 185       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 186       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 187    }
 188
 189    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 190    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 191    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 192    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 193
 194    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 195                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 196 }
 197
 198 Temp as_vgpr(isel_context *ctx, Temp val)
 199 {
 200    if (val.type() == RegType::sgpr) {
 201       Builder bld(ctx->program, ctx->block);
 202       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 203    }
 204    assert(val.type() == RegType::vgpr);
 205    return val;
 206 }
 207
 208 //assumes a != 0xffffffff
 209 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 210 {
 211    assert(b != 0);
 212    Builder bld(ctx->program, ctx->block);
 213
 214    if (util_is_power_of_two_or_zero(b)) {
 215       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 216       return;
 217    }
 218
 219    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 220
 221    assert(info.multiplier <= 0xffffffff);
 222
 223    bool pre_shift = info.pre_shift != 0;
 224    bool increment = info.increment != 0;
 225    bool multiply = true;
 226    bool post_shift = info.post_shift != 0;
 227
 228    if (!pre_shift && !increment && !multiply && !post_shift) {
 229       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 230       return;
 231    }
 232
 233    Temp pre_shift_dst = a;
 234    if (pre_shift) {
 235       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 237    }
 238
 239    Temp increment_dst = pre_shift_dst;
 240    if (increment) {
 241       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 242       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 243    }
 244
 245    Temp multiply_dst = increment_dst;
 246    if (multiply) {
 247       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 248       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 249                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 250    }
 251
 252    if (post_shift) {
 253       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 254    }
 255 }
 256
 257 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 258 {
 259    Builder bld(ctx->program, ctx->block);
 260    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 261 }
 262
 263
 264 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 265 {
 266    /* no need to extract the whole vector */
 267    if (src.regClass() == dst_rc) {
 268       assert(idx == 0);
 269       return src;
 270    }
 271    assert(src.size() > idx);
 272    Builder bld(ctx->program, ctx->block);
 273    auto it = ctx->allocated_vec.find(src.id());
 274    /* the size check needs to be early because elements other than 0 may be garbage */
 275    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 276       if (it->second[idx].regClass() == dst_rc) {
 277          return it->second[idx];
 278       } else {
 279          assert(dst_rc.size() == it->second[idx].regClass().size());
 280          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 281          return bld.copy(bld.def(dst_rc), it->second[idx]);
 282       }
 283    }
 284
 285    if (src.size() == dst_rc.size()) {
 286       assert(idx == 0);
 287       return bld.copy(bld.def(dst_rc), src);
 288    } else {
 289       Temp dst = bld.tmp(dst_rc);
 290       emit_extract_vector(ctx, src, idx, dst);
 291       return dst;
 292    }
 293 }
 294
 295 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 296 {
 297    if (num_components == 1)
 298       return;
 299    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 300       return;
 301    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 302    split->operands[0] = Operand(vec_src);
 303    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 304    for (unsigned i = 0; i < num_components; i++) {
 305       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 306       split->definitions[i] = Definition(elems[i]);
 307    }
 308    ctx->block->instructions.emplace_back(std::move(split));
 309    ctx->allocated_vec.emplace(vec_src.id(), elems);
 310 }
 311
 312 /* This vector expansion uses a mask to determine which elements in the new vector
 313  * come from the original vector. The other elements are undefined. */
 314 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 315 {
 316    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 317
 318    if (vec_src == dst)
 319       return;
 320
 321    Builder bld(ctx->program, ctx->block);
 322    if (num_components == 1) {
 323       if (dst.type() == RegType::sgpr)
 324          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 325       else
 326          bld.copy(Definition(dst), vec_src);
 327       return;
 328    }
 329
 330    unsigned component_size = dst.size() / num_components;
 331    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 332
 333    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 334    vec->definitions[0] = Definition(dst);
 335    unsigned k = 0;
 336    for (unsigned i = 0; i < num_components; i++) {
 337       if (mask & (1 << i)) {
 338          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 339          if (dst.type() == RegType::sgpr)
 340             src = bld.as_uniform(src);
 341          vec->operands[i] = Operand(src);
 342       } else {
 343          vec->operands[i] = Operand(0u);
 344       }
 345       elems[i] = vec->operands[i].getTemp();
 346    }
 347    ctx->block->instructions.emplace_back(std::move(vec));
 348    ctx->allocated_vec.emplace(dst.id(), elems);
 349 }
 350
 351 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 352 {
 353    Builder bld(ctx->program, ctx->block);
 354    if (!dst.id())
 355       dst = bld.tmp(bld.lm);
 356
 357    assert(val.regClass() == s1);
 358    assert(dst.regClass() == bld.lm);
 359
 360    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 361 }
 362
 363 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 364 {
 365    Builder bld(ctx->program, ctx->block);
 366    if (!dst.id())
 367       dst = bld.tmp(s1);
 368
 369    assert(val.regClass() == bld.lm);
 370    assert(dst.regClass() == s1);
 371
 372    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 373    Temp tmp = bld.tmp(s1);
 374    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 375    return emit_wqm(ctx, tmp, dst);
 376 }
 377
 378 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 379 {
 380    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 381       return get_ssa_temp(ctx, src.src.ssa);
 382
 383    if (src.src.ssa->num_components == size) {
 384       bool identity_swizzle = true;
 385       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 386          if (src.swizzle[i] != i)
 387             identity_swizzle = false;
 388       }
 389       if (identity_swizzle)
 390          return get_ssa_temp(ctx, src.src.ssa);
 391    }
 392
 393    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 394    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 395    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 396    assert(vec.size() % elem_size == 0);
 397
 398    RegClass elem_rc = RegClass(vec.type(), elem_size);
 399    if (size == 1) {
 400       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 401    } else {
 402       assert(size <= 4);
 403       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 404       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 405       for (unsigned i = 0; i < size; ++i) {
 406          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 407          vec_instr->operands[i] = Operand{elems[i]};
 408       }
 409       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 410       vec_instr->definitions[0] = Definition(dst);
 411       ctx->block->instructions.emplace_back(std::move(vec_instr));
 412       ctx->allocated_vec.emplace(dst.id(), elems);
 413       return dst;
 414    }
 415 }
 416
 417 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 418 {
 419    if (ptr.size() == 2)
 420       return ptr;
 421    Builder bld(ctx->program, ctx->block);
 422    if (ptr.type() == RegType::vgpr)
 423       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 424    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 425                      ptr, Operand((unsigned)ctx->options->address32_hi));
 426 }
 427
 428 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 429 {
 430    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 431    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 432    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 433    sop2->definitions[0] = Definition(dst);
 434    if (writes_scc)
 435       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 436    ctx->block->instructions.emplace_back(std::move(sop2));
 437 }
 438
 439 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 440                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 441 {
 442    Builder bld(ctx->program, ctx->block);
 443    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 444    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 445    if (src1.type() == RegType::sgpr) {
 446       if (commutative && src0.type() == RegType::vgpr) {
 447          Temp t = src0;
 448          src0 = src1;
 449          src1 = t;
 450       } else if (src0.type() == RegType::vgpr &&
 451                  op != aco_opcode::v_madmk_f32 &&
 452                  op != aco_opcode::v_madak_f32 &&
 453                  op != aco_opcode::v_madmk_f16 &&
 454                  op != aco_opcode::v_madak_f16) {
 455          /* If the instruction is not commutative, we emit a VOP3A instruction */
 456          bld.vop2_e64(op, Definition(dst), src0, src1);
 457          return;
 458       } else {
 459          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 460       }
 461    }
 462
 463    if (flush_denorms && ctx->program->chip_class < GFX9) {
 464       assert(dst.size() == 1);
 465       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 466       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 467    } else {
 468       bld.vop2(op, Definition(dst), src0, src1);
 469    }
 470 }
 471
 472 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 473                             bool flush_denorms = false)
 474 {
 475    Temp src0 = get_alu_src(ctx, instr->src[0]);
 476    Temp src1 = get_alu_src(ctx, instr->src[1]);
 477    Temp src2 = get_alu_src(ctx, instr->src[2]);
 478
 479    /* ensure that the instruction has at most 1 sgpr operand
 480     * The optimizer will inline constants for us */
 481    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 482       src0 = as_vgpr(ctx, src0);
 483    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 484       src1 = as_vgpr(ctx, src1);
 485    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 486       src2 = as_vgpr(ctx, src2);
 487
 488    Builder bld(ctx->program, ctx->block);
 489    if (flush_denorms && ctx->program->chip_class < GFX9) {
 490       assert(dst.size() == 1);
 491       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 492       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 493    } else {
 494       bld.vop3(op, Definition(dst), src0, src1, src2);
 495    }
 496 }
 497
 498 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 499 {
 500    Builder bld(ctx->program, ctx->block);
 501    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 502 }
 503
 504 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 505 {
 506    Temp src0 = get_alu_src(ctx, instr->src[0]);
 507    Temp src1 = get_alu_src(ctx, instr->src[1]);
 508    assert(src0.size() == src1.size());
 509
 510    aco_ptr<Instruction> vopc;
 511    if (src1.type() == RegType::sgpr) {
 512       if (src0.type() == RegType::vgpr) {
 513          /* to swap the operands, we might also have to change the opcode */
 514          switch (op) {
 515             case aco_opcode::v_cmp_lt_f32:
 516                op = aco_opcode::v_cmp_gt_f32;
 517                break;
 518             case aco_opcode::v_cmp_ge_f32:
 519                op = aco_opcode::v_cmp_le_f32;
 520                break;
 521             case aco_opcode::v_cmp_lt_i32:
 522                op = aco_opcode::v_cmp_gt_i32;
 523                break;
 524             case aco_opcode::v_cmp_ge_i32:
 525                op = aco_opcode::v_cmp_le_i32;
 526                break;
 527             case aco_opcode::v_cmp_lt_u32:
 528                op = aco_opcode::v_cmp_gt_u32;
 529                break;
 530             case aco_opcode::v_cmp_ge_u32:
 531                op = aco_opcode::v_cmp_le_u32;
 532                break;
 533             case aco_opcode::v_cmp_lt_f64:
 534                op = aco_opcode::v_cmp_gt_f64;
 535                break;
 536             case aco_opcode::v_cmp_ge_f64:
 537                op = aco_opcode::v_cmp_le_f64;
 538                break;
 539             case aco_opcode::v_cmp_lt_i64:
 540                op = aco_opcode::v_cmp_gt_i64;
 541                break;
 542             case aco_opcode::v_cmp_ge_i64:
 543                op = aco_opcode::v_cmp_le_i64;
 544                break;
 545             case aco_opcode::v_cmp_lt_u64:
 546                op = aco_opcode::v_cmp_gt_u64;
 547                break;
 548             case aco_opcode::v_cmp_ge_u64:
 549                op = aco_opcode::v_cmp_le_u64;
 550                break;
 551             default: /* eq and ne are commutative */
 552                break;
 553          }
 554          Temp t = src0;
 555          src0 = src1;
 556          src1 = t;
 557       } else {
 558          src1 = as_vgpr(ctx, src1);
 559       }
 560    }
 561
 562    Builder bld(ctx->program, ctx->block);
 563    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 564 }
 565
 566 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 567 {
 568    Temp src0 = get_alu_src(ctx, instr->src[0]);
 569    Temp src1 = get_alu_src(ctx, instr->src[1]);
 570    Builder bld(ctx->program, ctx->block);
 571
 572    assert(dst.regClass() == bld.lm);
 573    assert(src0.type() == RegType::sgpr);
 574    assert(src1.type() == RegType::sgpr);
 575    assert(src0.regClass() == src1.regClass());
 576
 577    /* Emit the SALU comparison instruction */
 578    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 579    /* Turn the result into a per-lane bool */
 580    bool_to_vector_condition(ctx, cmp, dst);
 581 }
 582
 583 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 584                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 585 {
 586    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 587    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 588    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 589    bool use_valu = s_op == aco_opcode::num_opcodes ||
 590                    divergent_vals ||
 591                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 592                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 593    aco_opcode op = use_valu ? v_op : s_op;
 594    assert(op != aco_opcode::num_opcodes);
 595    assert(dst.regClass() == ctx->program->lane_mask);
 596
 597    if (use_valu)
 598       emit_vopc_instruction(ctx, instr, op, dst);
 599    else
 600       emit_sopc_instruction(ctx, instr, op, dst);
 601 }
 602
 603 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 604 {
 605    Builder bld(ctx->program, ctx->block);
 606    Temp src0 = get_alu_src(ctx, instr->src[0]);
 607    Temp src1 = get_alu_src(ctx, instr->src[1]);
 608
 609    assert(dst.regClass() == bld.lm);
 610    assert(src0.regClass() == bld.lm);
 611    assert(src1.regClass() == bld.lm);
 612
 613    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 614 }
 615
 616 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 617 {
 618    Builder bld(ctx->program, ctx->block);
 619    Temp cond = get_alu_src(ctx, instr->src[0]);
 620    Temp then = get_alu_src(ctx, instr->src[1]);
 621    Temp els = get_alu_src(ctx, instr->src[2]);
 622
 623    assert(cond.regClass() == bld.lm);
 624
 625    if (dst.type() == RegType::vgpr) {
 626       aco_ptr<Instruction> bcsel;
 627       if (dst.size() == 1) {
 628          then = as_vgpr(ctx, then);
 629          els = as_vgpr(ctx, els);
 630
 631          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 632       } else if (dst.size() == 2) {
 633          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 634          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 635          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 637
 638          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 639          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 640
 641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 642       } else {
 643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 644          nir_print_instr(&instr->instr, stderr);
 645          fprintf(stderr, "\n");
 646       }
 647       return;
 648    }
 649
 650    if (instr->dest.dest.ssa.bit_size == 1) {
 651       assert(dst.regClass() == bld.lm);
 652       assert(then.regClass() == bld.lm);
 653       assert(els.regClass() == bld.lm);
 654    }
 655
 656    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 657       if (dst.regClass() == s1 || dst.regClass() == s2) {
 658          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 659          assert(dst.size() == then.size());
 660          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 661          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 662       } else {
 663          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 664          nir_print_instr(&instr->instr, stderr);
 665          fprintf(stderr, "\n");
 666       }
 667       return;
 668    }
 669
 670    /* divergent boolean bcsel
 671     * this implements bcsel on bools: dst = s0 ? s1 : s2
 672     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 673    assert(instr->dest.dest.ssa.bit_size == 1);
 674
 675    if (cond.id() != then.id())
 676       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 677
 678    if (cond.id() == els.id())
 679       bld.sop1(Builder::s_mov, Definition(dst), then);
 680    else
 681       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 682                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 683 }
 684
 685 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 686                     aco_opcode op, uint32_t undo)
 687 {
 688    /* multiply by 16777216 to handle denormals */
 689    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 690                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 691    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 692    scaled = bld.vop1(op, bld.def(v1), scaled);
 693    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 694
 695    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 696
 697    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 698 }
 699
 700 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 701 {
 702    if (ctx->block->fp_mode.denorm32 == 0) {
 703       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 704       return;
 705    }
 706
 707    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 708 }
 709
 710 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 711 {
 712    if (ctx->block->fp_mode.denorm32 == 0) {
 713       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 714       return;
 715    }
 716
 717    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 718 }
 719
 720 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 721 {
 722    if (ctx->block->fp_mode.denorm32 == 0) {
 723       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 724       return;
 725    }
 726
 727    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 728 }
 729
 730 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 731 {
 732    if (ctx->block->fp_mode.denorm32 == 0) {
 733       bld.vop1(aco_opcode::v_log_f32, dst, val);
 734       return;
 735    }
 736
 737    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 738 }
 739
 740 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 741 {
 742    if (ctx->options->chip_class >= GFX7)
 743       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 744
 745    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 746    /* TODO: create more efficient code! */
 747    if (val.type() == RegType::sgpr)
 748       val = as_vgpr(ctx, val);
 749
 750    /* Split the input value. */
 751    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 752    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 753
 754    /* Extract the exponent and compute the unbiased value. */
 755    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 756
 757    /* Extract the fractional part. */
 758    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 759    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 760
 761    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 762    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 763
 764    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 765    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 766    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 767    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 768    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 769
 770    /* Get the sign bit. */
 771    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 772
 773    /* Decide the operation to apply depending on the unbiased exponent. */
 774    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 775    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 776    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 777    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 778    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 779    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 780
 781    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 782 }
 783
 784 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 785 {
 786    if (ctx->options->chip_class >= GFX7)
 787       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 788
 789    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 790    Temp src0 = as_vgpr(ctx, val);
 791
 792    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 793    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 794
 795    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 796    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 797    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 798
 799    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 800    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 801    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 803
 804    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 805    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 806
 807    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 808
 809    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 810    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 811
 812    return add->definitions[0].getTemp();
 813 }
 814
 815 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 816 {
 817    if (!instr->dest.dest.is_ssa) {
 818       fprintf(stderr, "nir alu dst not in ssa: ");
 819       nir_print_instr(&instr->instr, stderr);
 820       fprintf(stderr, "\n");
 821       abort();
 822    }
 823    Builder bld(ctx->program, ctx->block);
 824    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 825    switch(instr->op) {
 826    case nir_op_vec2:
 827    case nir_op_vec3:
 828    case nir_op_vec4: {
 829       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 830       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 831       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 832          elems[i] = get_alu_src(ctx, instr->src[i]);
 833          vec->operands[i] = Operand{elems[i]};
 834       }
 835       vec->definitions[0] = Definition(dst);
 836       ctx->block->instructions.emplace_back(std::move(vec));
 837       ctx->allocated_vec.emplace(dst.id(), elems);
 838       break;
 839    }
 840    case nir_op_mov: {
 841       Temp src = get_alu_src(ctx, instr->src[0]);
 842       aco_ptr<Instruction> mov;
 843       if (dst.type() == RegType::sgpr) {
 844          if (src.type() == RegType::vgpr)
 845             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 846          else if (src.regClass() == s1)
 847             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 848          else if (src.regClass() == s2)
 849             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 850          else
 851             unreachable("wrong src register class for nir_op_imov");
 852       } else if (dst.regClass() == v1) {
 853          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 854       } else if (dst.regClass() == v2) {
 855          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 856       } else {
 857          nir_print_instr(&instr->instr, stderr);
 858          unreachable("Should have been lowered to scalar.");
 859       }
 860       break;
 861    }
 862    case nir_op_inot: {
 863       Temp src = get_alu_src(ctx, instr->src[0]);
 864       if (instr->dest.dest.ssa.bit_size == 1) {
 865          assert(src.regClass() == bld.lm);
 866          assert(dst.regClass() == bld.lm);
 867          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 868          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 869          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 870       } else if (dst.regClass() == v1) {
 871          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 872       } else if (dst.type() == RegType::sgpr) {
 873          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 874          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_ineg: {
 883       Temp src = get_alu_src(ctx, instr->src[0]);
 884       if (dst.regClass() == v1) {
 885          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 886       } else if (dst.regClass() == s1) {
 887          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 888       } else if (dst.size() == 2) {
 889          Temp src0 = bld.tmp(dst.type(), 1);
 890          Temp src1 = bld.tmp(dst.type(), 1);
 891          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 892
 893          if (dst.regClass() == s2) {
 894             Temp carry = bld.tmp(s1);
 895             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 896             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 897             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 898          } else {
 899             Temp lower = bld.tmp(v1);
 900             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 901             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 902             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 903          }
 904       } else {
 905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 906          nir_print_instr(&instr->instr, stderr);
 907          fprintf(stderr, "\n");
 908       }
 909       break;
 910    }
 911    case nir_op_iabs: {
 912       if (dst.regClass() == s1) {
 913          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 914       } else if (dst.regClass() == v1) {
 915          Temp src = get_alu_src(ctx, instr->src[0]);
 916          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 917       } else {
 918          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 919          nir_print_instr(&instr->instr, stderr);
 920          fprintf(stderr, "\n");
 921       }
 922       break;
 923    }
 924    case nir_op_isign: {
 925       Temp src = get_alu_src(ctx, instr->src[0]);
 926       if (dst.regClass() == s1) {
 927          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 928          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 929          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 930       } else if (dst.regClass() == s2) {
 931          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 932          Temp neqz;
 933          if (ctx->program->chip_class >= GFX8)
 934             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 935          else
 936             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 937          /* SCC gets zero-extended to 64 bit */
 938          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 939       } else if (dst.regClass() == v1) {
 940          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 941          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 942          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 943       } else if (dst.regClass() == v2) {
 944          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 945          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 946          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 947          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 948          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 949          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 950       } else {
 951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 952          nir_print_instr(&instr->instr, stderr);
 953          fprintf(stderr, "\n");
 954       }
 955       break;
 956    }
 957    case nir_op_imax: {
 958       if (dst.regClass() == v1) {
 959          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 960       } else if (dst.regClass() == s1) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 962       } else {
 963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 964          nir_print_instr(&instr->instr, stderr);
 965          fprintf(stderr, "\n");
 966       }
 967       break;
 968    }
 969    case nir_op_umax: {
 970       if (dst.regClass() == v1) {
 971          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 972       } else if (dst.regClass() == s1) {
 973          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 974       } else {
 975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 976          nir_print_instr(&instr->instr, stderr);
 977          fprintf(stderr, "\n");
 978       }
 979       break;
 980    }
 981    case nir_op_imin: {
 982       if (dst.regClass() == v1) {
 983          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 984       } else if (dst.regClass() == s1) {
 985          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 986       } else {
 987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 988          nir_print_instr(&instr->instr, stderr);
 989          fprintf(stderr, "\n");
 990       }
 991       break;
 992    }
 993    case nir_op_umin: {
 994       if (dst.regClass() == v1) {
 995          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 996       } else if (dst.regClass() == s1) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_ior: {
1006       if (instr->dest.dest.ssa.bit_size == 1) {
1007          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1008       } else if (dst.regClass() == v1) {
1009          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1010       } else if (dst.regClass() == s1) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1012       } else if (dst.regClass() == s2) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1014       } else {
1015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1016          nir_print_instr(&instr->instr, stderr);
1017          fprintf(stderr, "\n");
1018       }
1019       break;
1020    }
1021    case nir_op_iand: {
1022       if (instr->dest.dest.ssa.bit_size == 1) {
1023          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1024       } else if (dst.regClass() == v1) {
1025          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1026       } else if (dst.regClass() == s1) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1028       } else if (dst.regClass() == s2) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1030       } else {
1031          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1032          nir_print_instr(&instr->instr, stderr);
1033          fprintf(stderr, "\n");
1034       }
1035       break;
1036    }
1037    case nir_op_ixor: {
1038       if (instr->dest.dest.ssa.bit_size == 1) {
1039          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1040       } else if (dst.regClass() == v1) {
1041          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1042       } else if (dst.regClass() == s1) {
1043          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1044       } else if (dst.regClass() == s2) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1046       } else {
1047          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1048          nir_print_instr(&instr->instr, stderr);
1049          fprintf(stderr, "\n");
1050       }
1051       break;
1052    }
1053    case nir_op_ushr: {
1054       if (dst.regClass() == v1) {
1055          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1056       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1057          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1058                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1059       } else if (dst.regClass() == v2) {
1060          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1061                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1062       } else if (dst.regClass() == s2) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1064       } else if (dst.regClass() == s1) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1066       } else {
1067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1068          nir_print_instr(&instr->instr, stderr);
1069          fprintf(stderr, "\n");
1070       }
1071       break;
1072    }
1073    case nir_op_ishl: {
1074       if (dst.regClass() == v1) {
1075          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1076       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1077          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1078                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1079       } else if (dst.regClass() == v2) {
1080          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1081                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1082       } else if (dst.regClass() == s1) {
1083          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1084       } else if (dst.regClass() == s2) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_ishr: {
1094       if (dst.regClass() == v1) {
1095          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1096       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1097          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1098                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1099       } else if (dst.regClass() == v2) {
1100          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1101                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1102       } else if (dst.regClass() == s1) {
1103          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1104       } else if (dst.regClass() == s2) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1106       } else {
1107          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1108          nir_print_instr(&instr->instr, stderr);
1109          fprintf(stderr, "\n");
1110       }
1111       break;
1112    }
1113    case nir_op_find_lsb: {
1114       Temp src = get_alu_src(ctx, instr->src[0]);
1115       if (src.regClass() == s1) {
1116          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1117       } else if (src.regClass() == v1) {
1118          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1119       } else if (src.regClass() == s2) {
1120          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1121       } else {
1122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1123          nir_print_instr(&instr->instr, stderr);
1124          fprintf(stderr, "\n");
1125       }
1126       break;
1127    }
1128    case nir_op_ufind_msb:
1129    case nir_op_ifind_msb: {
1130       Temp src = get_alu_src(ctx, instr->src[0]);
1131       if (src.regClass() == s1 || src.regClass() == s2) {
1132          aco_opcode op = src.regClass() == s2 ?
1133                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1134                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1135          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1136
1137          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1138                                         Operand(src.size() * 32u - 1u), msb_rev);
1139          Temp msb = sub.def(0).getTemp();
1140          Temp carry = sub.def(1).getTemp();
1141
1142          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1143       } else if (src.regClass() == v1) {
1144          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1145          Temp msb_rev = bld.tmp(v1);
1146          emit_vop1_instruction(ctx, instr, op, msb_rev);
1147          Temp msb = bld.tmp(v1);
1148          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1149          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1150       } else {
1151          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1152          nir_print_instr(&instr->instr, stderr);
1153          fprintf(stderr, "\n");
1154       }
1155       break;
1156    }
1157    case nir_op_bitfield_reverse: {
1158       if (dst.regClass() == s1) {
1159          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1160       } else if (dst.regClass() == v1) {
1161          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else {
1163          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1164          nir_print_instr(&instr->instr, stderr);
1165          fprintf(stderr, "\n");
1166       }
1167       break;
1168    }
1169    case nir_op_iadd: {
1170       if (dst.regClass() == s1) {
1171          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1172          break;
1173       }
1174
1175       Temp src0 = get_alu_src(ctx, instr->src[0]);
1176       Temp src1 = get_alu_src(ctx, instr->src[1]);
1177       if (dst.regClass() == v1) {
1178          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1179          break;
1180       }
1181
1182       assert(src0.size() == 2 && src1.size() == 2);
1183       Temp src00 = bld.tmp(src0.type(), 1);
1184       Temp src01 = bld.tmp(dst.type(), 1);
1185       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1186       Temp src10 = bld.tmp(src1.type(), 1);
1187       Temp src11 = bld.tmp(dst.type(), 1);
1188       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1189
1190       if (dst.regClass() == s2) {
1191          Temp carry = bld.tmp(s1);
1192          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1193          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1195       } else if (dst.regClass() == v2) {
1196          Temp dst0 = bld.tmp(v1);
1197          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1198          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1199          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1200       } else {
1201          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1202          nir_print_instr(&instr->instr, stderr);
1203          fprintf(stderr, "\n");
1204       }
1205       break;
1206    }
1207    case nir_op_uadd_sat: {
1208       Temp src0 = get_alu_src(ctx, instr->src[0]);
1209       Temp src1 = get_alu_src(ctx, instr->src[1]);
1210       if (dst.regClass() == s1) {
1211          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1212          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1213                   src0, src1);
1214          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1215       } else if (dst.regClass() == v1) {
1216          if (ctx->options->chip_class >= GFX9) {
1217             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1218             add->operands[0] = Operand(src0);
1219             add->operands[1] = Operand(src1);
1220             add->definitions[0] = Definition(dst);
1221             add->clamp = 1;
1222             ctx->block->instructions.emplace_back(std::move(add));
1223          } else {
1224             if (src1.regClass() != v1)
1225                std::swap(src0, src1);
1226             assert(src1.regClass() == v1);
1227             Temp tmp = bld.tmp(v1);
1228             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1229             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1230          }
1231       } else {
1232          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1233          nir_print_instr(&instr->instr, stderr);
1234          fprintf(stderr, "\n");
1235       }
1236       break;
1237    }
1238    case nir_op_uadd_carry: {
1239       Temp src0 = get_alu_src(ctx, instr->src[0]);
1240       Temp src1 = get_alu_src(ctx, instr->src[1]);
1241       if (dst.regClass() == s1) {
1242          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1243          break;
1244       }
1245       if (dst.regClass() == v1) {
1246          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1247          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1248          break;
1249       }
1250
1251       Temp src00 = bld.tmp(src0.type(), 1);
1252       Temp src01 = bld.tmp(dst.type(), 1);
1253       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1254       Temp src10 = bld.tmp(src1.type(), 1);
1255       Temp src11 = bld.tmp(dst.type(), 1);
1256       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1257       if (dst.regClass() == s2) {
1258          Temp carry = bld.tmp(s1);
1259          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1260          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1261          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1262       } else if (dst.regClass() == v2) {
1263          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1264          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1265          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1266          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1267       } else {
1268          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1269          nir_print_instr(&instr->instr, stderr);
1270          fprintf(stderr, "\n");
1271       }
1272       break;
1273    }
1274    case nir_op_isub: {
1275       if (dst.regClass() == s1) {
1276          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1277          break;
1278       }
1279
1280       Temp src0 = get_alu_src(ctx, instr->src[0]);
1281       Temp src1 = get_alu_src(ctx, instr->src[1]);
1282       if (dst.regClass() == v1) {
1283          bld.vsub32(Definition(dst), src0, src1);
1284          break;
1285       }
1286
1287       Temp src00 = bld.tmp(src0.type(), 1);
1288       Temp src01 = bld.tmp(dst.type(), 1);
1289       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1290       Temp src10 = bld.tmp(src1.type(), 1);
1291       Temp src11 = bld.tmp(dst.type(), 1);
1292       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1293       if (dst.regClass() == s2) {
1294          Temp carry = bld.tmp(s1);
1295          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1296          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1297          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1298       } else if (dst.regClass() == v2) {
1299          Temp lower = bld.tmp(v1);
1300          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1301          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1302          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1303       } else {
1304          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1305          nir_print_instr(&instr->instr, stderr);
1306          fprintf(stderr, "\n");
1307       }
1308       break;
1309    }
1310    case nir_op_usub_borrow: {
1311       Temp src0 = get_alu_src(ctx, instr->src[0]);
1312       Temp src1 = get_alu_src(ctx, instr->src[1]);
1313       if (dst.regClass() == s1) {
1314          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1315          break;
1316       } else if (dst.regClass() == v1) {
1317          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1318          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1319          break;
1320       }
1321
1322       Temp src00 = bld.tmp(src0.type(), 1);
1323       Temp src01 = bld.tmp(dst.type(), 1);
1324       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1325       Temp src10 = bld.tmp(src1.type(), 1);
1326       Temp src11 = bld.tmp(dst.type(), 1);
1327       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1328       if (dst.regClass() == s2) {
1329          Temp borrow = bld.tmp(s1);
1330          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1331          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1332          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1333       } else if (dst.regClass() == v2) {
1334          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1335          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1336          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1337          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_imul: {
1346       if (dst.regClass() == v1) {
1347          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1349       } else if (dst.regClass() == s1) {
1350          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_umul_high: {
1359       if (dst.regClass() == v1) {
1360          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1361       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1362          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1) {
1364          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1365                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1366          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_imul_high: {
1375       if (dst.regClass() == v1) {
1376          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1377       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1378          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1) {
1380          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1381                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1382          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1383       } else {
1384          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1385          nir_print_instr(&instr->instr, stderr);
1386          fprintf(stderr, "\n");
1387       }
1388       break;
1389    }
1390    case nir_op_fmul: {
1391       if (dst.size() == 1) {
1392          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1393       } else if (dst.size() == 2) {
1394          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1395                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1396       } else {
1397          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1398          nir_print_instr(&instr->instr, stderr);
1399          fprintf(stderr, "\n");
1400       }
1401       break;
1402    }
1403    case nir_op_fadd: {
1404       if (dst.size() == 1) {
1405          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1406       } else if (dst.size() == 2) {
1407          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1408                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1409       } else {
1410          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1411          nir_print_instr(&instr->instr, stderr);
1412          fprintf(stderr, "\n");
1413       }
1414       break;
1415    }
1416    case nir_op_fsub: {
1417       Temp src0 = get_alu_src(ctx, instr->src[0]);
1418       Temp src1 = get_alu_src(ctx, instr->src[1]);
1419       if (dst.size() == 1) {
1420          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1421             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1422          else
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1424       } else if (dst.size() == 2) {
1425          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1426                                      get_alu_src(ctx, instr->src[0]),
1427                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1428          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1429          sub->neg[1] = true;
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_fmax: {
1438       if (dst.size() == 1) {
1439          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1440       } else if (dst.size() == 2) {
1441          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1442             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1443                                 get_alu_src(ctx, instr->src[0]),
1444                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1445             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1446          } else {
1447             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1448                      get_alu_src(ctx, instr->src[0]),
1449                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1450          }
1451       } else {
1452          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1453          nir_print_instr(&instr->instr, stderr);
1454          fprintf(stderr, "\n");
1455       }
1456       break;
1457    }
1458    case nir_op_fmin: {
1459       if (dst.size() == 1) {
1460          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1461       } else if (dst.size() == 2) {
1462          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1463             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1464                                 get_alu_src(ctx, instr->src[0]),
1465                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1466             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1467          } else {
1468             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1469                      get_alu_src(ctx, instr->src[0]),
1470                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1471          }
1472       } else {
1473          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1474          nir_print_instr(&instr->instr, stderr);
1475          fprintf(stderr, "\n");
1476       }
1477       break;
1478    }
1479    case nir_op_fmax3: {
1480       if (dst.size() == 1) {
1481          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1482       } else {
1483          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1484          nir_print_instr(&instr->instr, stderr);
1485          fprintf(stderr, "\n");
1486       }
1487       break;
1488    }
1489    case nir_op_fmin3: {
1490       if (dst.size() == 1) {
1491          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fmed3: {
1500       if (dst.size() == 1) {
1501          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1502       } else {
1503          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1504          nir_print_instr(&instr->instr, stderr);
1505          fprintf(stderr, "\n");
1506       }
1507       break;
1508    }
1509    case nir_op_umax3: {
1510       if (dst.size() == 1) {
1511          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1512       } else {
1513          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1514          nir_print_instr(&instr->instr, stderr);
1515          fprintf(stderr, "\n");
1516       }
1517       break;
1518    }
1519    case nir_op_umin3: {
1520       if (dst.size() == 1) {
1521          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1522       } else {
1523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1524          nir_print_instr(&instr->instr, stderr);
1525          fprintf(stderr, "\n");
1526       }
1527       break;
1528    }
1529    case nir_op_umed3: {
1530       if (dst.size() == 1) {
1531          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_imax3: {
1540       if (dst.size() == 1) {
1541          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_imin3: {
1550       if (dst.size() == 1) {
1551          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_imed3: {
1560       if (dst.size() == 1) {
1561          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1562       } else {
1563          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1564          nir_print_instr(&instr->instr, stderr);
1565          fprintf(stderr, "\n");
1566       }
1567       break;
1568    }
1569    case nir_op_cube_face_coord: {
1570       Temp in = get_alu_src(ctx, instr->src[0], 3);
1571       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1572                       emit_extract_vector(ctx, in, 1, v1),
1573                       emit_extract_vector(ctx, in, 2, v1) };
1574       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1575       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1576       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1577       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1578       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1579       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1580       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1581       break;
1582    }
1583    case nir_op_cube_face_index: {
1584       Temp in = get_alu_src(ctx, instr->src[0], 3);
1585       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1586                       emit_extract_vector(ctx, in, 1, v1),
1587                       emit_extract_vector(ctx, in, 2, v1) };
1588       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1589       break;
1590    }
1591    case nir_op_bcsel: {
1592       emit_bcsel(ctx, instr, dst);
1593       break;
1594    }
1595    case nir_op_frsq: {
1596       if (dst.size() == 1) {
1597          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1598       } else if (dst.size() == 2) {
1599          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1600       } else {
1601          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1602          nir_print_instr(&instr->instr, stderr);
1603          fprintf(stderr, "\n");
1604       }
1605       break;
1606    }
1607    case nir_op_fneg: {
1608       Temp src = get_alu_src(ctx, instr->src[0]);
1609       if (dst.size() == 1) {
1610          if (ctx->block->fp_mode.must_flush_denorms32)
1611             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1612          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1613       } else if (dst.size() == 2) {
1614          if (ctx->block->fp_mode.must_flush_denorms16_64)
1615             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1616          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1617          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1618          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1619          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1620       } else {
1621          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1622          nir_print_instr(&instr->instr, stderr);
1623          fprintf(stderr, "\n");
1624       }
1625       break;
1626    }
1627    case nir_op_fabs: {
1628       Temp src = get_alu_src(ctx, instr->src[0]);
1629       if (dst.size() == 1) {
1630          if (ctx->block->fp_mode.must_flush_denorms32)
1631             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1632          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1633       } else if (dst.size() == 2) {
1634          if (ctx->block->fp_mode.must_flush_denorms16_64)
1635             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1636          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1637          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1638          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1640       } else {
1641          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1642          nir_print_instr(&instr->instr, stderr);
1643          fprintf(stderr, "\n");
1644       }
1645       break;
1646    }
1647    case nir_op_fsat: {
1648       Temp src = get_alu_src(ctx, instr->src[0]);
1649       if (dst.size() == 1) {
1650          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1651          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1652          // TODO: confirm that this holds under any circumstances
1653       } else if (dst.size() == 2) {
1654          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1655          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1656          vop3->clamp = true;
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_flog2: {
1665       if (dst.size() == 1) {
1666          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_frcp: {
1675       if (dst.size() == 1) {
1676          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1677       } else if (dst.size() == 2) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1679       } else {
1680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1681          nir_print_instr(&instr->instr, stderr);
1682          fprintf(stderr, "\n");
1683       }
1684       break;
1685    }
1686    case nir_op_fexp2: {
1687       if (dst.size() == 1) {
1688          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1689       } else {
1690          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1691          nir_print_instr(&instr->instr, stderr);
1692          fprintf(stderr, "\n");
1693       }
1694       break;
1695    }
1696    case nir_op_fsqrt: {
1697       if (dst.size() == 1) {
1698          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1699       } else if (dst.size() == 2) {
1700          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1701       } else {
1702          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1703          nir_print_instr(&instr->instr, stderr);
1704          fprintf(stderr, "\n");
1705       }
1706       break;
1707    }
1708    case nir_op_ffract: {
1709       if (dst.size() == 1) {
1710          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1711       } else if (dst.size() == 2) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1713       } else {
1714          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1715          nir_print_instr(&instr->instr, stderr);
1716          fprintf(stderr, "\n");
1717       }
1718       break;
1719    }
1720    case nir_op_ffloor: {
1721       if (dst.size() == 1) {
1722          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1723       } else if (dst.size() == 2) {
1724          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1725       } else {
1726          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1727          nir_print_instr(&instr->instr, stderr);
1728          fprintf(stderr, "\n");
1729       }
1730       break;
1731    }
1732    case nir_op_fceil: {
1733       if (dst.size() == 1) {
1734          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1735       } else if (dst.size() == 2) {
1736          if (ctx->options->chip_class >= GFX7) {
1737             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1738          } else {
1739             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1740             Temp src0 = get_alu_src(ctx, instr->src[0]);
1741
1742             /* trunc = trunc(src0)
1743              * if (src0 > 0.0 && src0 != trunc)
1744              *    trunc += 1.0
1745              */
1746             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1747             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1748             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1749             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1750             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1751             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1752             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1753          }
1754       } else {
1755          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1756          nir_print_instr(&instr->instr, stderr);
1757          fprintf(stderr, "\n");
1758       }
1759       break;
1760    }
1761    case nir_op_ftrunc: {
1762       if (dst.size() == 1) {
1763          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1764       } else if (dst.size() == 2) {
1765          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1766       } else {
1767          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1768          nir_print_instr(&instr->instr, stderr);
1769          fprintf(stderr, "\n");
1770       }
1771       break;
1772    }
1773    case nir_op_fround_even: {
1774       if (dst.size() == 1) {
1775          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1776       } else if (dst.size() == 2) {
1777          if (ctx->options->chip_class >= GFX7) {
1778             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1779          } else {
1780             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1781             Temp src0 = get_alu_src(ctx, instr->src[0]);
1782
1783             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1784             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1785
1786             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1787             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1788             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1789             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1790             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1791             tmp = sub->definitions[0].getTemp();
1792
1793             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1794             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1795             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1796             Temp cond = vop3->definitions[0].getTemp();
1797
1798             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1799             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1800             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1801             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1802
1803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1804          }
1805       } else {
1806          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1807          nir_print_instr(&instr->instr, stderr);
1808          fprintf(stderr, "\n");
1809       }
1810       break;
1811    }
1812    case nir_op_fsin:
1813    case nir_op_fcos: {
1814       Temp src = get_alu_src(ctx, instr->src[0]);
1815       aco_ptr<Instruction> norm;
1816       if (dst.size() == 1) {
1817          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1818          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1819
1820          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1821          if (ctx->options->chip_class < GFX9)
1822             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1823
1824          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1825          bld.vop1(opcode, Definition(dst), tmp);
1826       } else {
1827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1828          nir_print_instr(&instr->instr, stderr);
1829          fprintf(stderr, "\n");
1830       }
1831       break;
1832    }
1833    case nir_op_ldexp: {
1834       if (dst.size() == 1) {
1835          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1836                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1837                   get_alu_src(ctx, instr->src[1]));
1838       } else if (dst.size() == 2) {
1839          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1840                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1841                   get_alu_src(ctx, instr->src[1]));
1842       } else {
1843          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1844          nir_print_instr(&instr->instr, stderr);
1845          fprintf(stderr, "\n");
1846       }
1847       break;
1848    }
1849    case nir_op_frexp_sig: {
1850       if (dst.size() == 1) {
1851          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1852                   get_alu_src(ctx, instr->src[0]));
1853       } else if (dst.size() == 2) {
1854          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1855                   get_alu_src(ctx, instr->src[0]));
1856       } else {
1857          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1858          nir_print_instr(&instr->instr, stderr);
1859          fprintf(stderr, "\n");
1860       }
1861       break;
1862    }
1863    case nir_op_frexp_exp: {
1864       if (instr->src[0].src.ssa->bit_size == 32) {
1865          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1866                   get_alu_src(ctx, instr->src[0]));
1867       } else if (instr->src[0].src.ssa->bit_size == 64) {
1868          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1869                   get_alu_src(ctx, instr->src[0]));
1870       } else {
1871          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1872          nir_print_instr(&instr->instr, stderr);
1873          fprintf(stderr, "\n");
1874       }
1875       break;
1876    }
1877    case nir_op_fsign: {
1878       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1879       if (dst.size() == 1) {
1880          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1881          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1882          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1884       } else if (dst.size() == 2) {
1885          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1886          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1887          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1888
1889          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1890          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1891          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1892
1893          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1894       } else {
1895          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1896          nir_print_instr(&instr->instr, stderr);
1897          fprintf(stderr, "\n");
1898       }
1899       break;
1900    }
1901    case nir_op_f2f32: {
1902       if (instr->src[0].src.ssa->bit_size == 64) {
1903          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_f2f64: {
1912       if (instr->src[0].src.ssa->bit_size == 32) {
1913          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1914       } else {
1915          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1916          nir_print_instr(&instr->instr, stderr);
1917          fprintf(stderr, "\n");
1918       }
1919       break;
1920    }
1921    case nir_op_i2f32: {
1922       assert(dst.size() == 1);
1923       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1924       break;
1925    }
1926    case nir_op_i2f64: {
1927       if (instr->src[0].src.ssa->bit_size == 32) {
1928          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1929       } else if (instr->src[0].src.ssa->bit_size == 64) {
1930          Temp src = get_alu_src(ctx, instr->src[0]);
1931          RegClass rc = RegClass(src.type(), 1);
1932          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1933          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1934          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1935          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1936          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1937          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1938
1939       } else {
1940          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1941          nir_print_instr(&instr->instr, stderr);
1942          fprintf(stderr, "\n");
1943       }
1944       break;
1945    }
1946    case nir_op_u2f32: {
1947       assert(dst.size() == 1);
1948       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1949       break;
1950    }
1951    case nir_op_u2f64: {
1952       if (instr->src[0].src.ssa->bit_size == 32) {
1953          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1954       } else if (instr->src[0].src.ssa->bit_size == 64) {
1955          Temp src = get_alu_src(ctx, instr->src[0]);
1956          RegClass rc = RegClass(src.type(), 1);
1957          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1958          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1959          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1960          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1961          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1962          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_f2i32: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32) {
1973          if (dst.type() == RegType::vgpr)
1974             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1975          else
1976             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1977                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1978
1979       } else if (instr->src[0].src.ssa->bit_size == 64) {
1980          if (dst.type() == RegType::vgpr)
1981             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1982          else
1983             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1984                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1985
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_f2u32: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       if (instr->src[0].src.ssa->bit_size == 32) {
1996          if (dst.type() == RegType::vgpr)
1997             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1998          else
1999             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2000                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2001
2002       } else if (instr->src[0].src.ssa->bit_size == 64) {
2003          if (dst.type() == RegType::vgpr)
2004             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2005          else
2006             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2007                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2008
2009       } else {
2010          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2011          nir_print_instr(&instr->instr, stderr);
2012          fprintf(stderr, "\n");
2013       }
2014       break;
2015    }
2016    case nir_op_f2i64: {
2017       Temp src = get_alu_src(ctx, instr->src[0]);
2018       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2019          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2020          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2021          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2022          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2023          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2024          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2025          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2026          Temp new_exponent = bld.tmp(v1);
2027          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2028          if (ctx->program->chip_class >= GFX8)
2029             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2030          else
2031             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2032          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2033          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2034          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2035          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2036          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2037          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2038          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2039          Temp new_lower = bld.tmp(v1);
2040          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2041          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2042          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2043
2044       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2045          if (src.type() == RegType::vgpr)
2046             src = bld.as_uniform(src);
2047          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2048          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2049          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2050          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2051          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2052          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2053          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2054          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2055          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2056          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2057          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2058          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2059          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2060          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2061          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2062          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2063          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2064          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2065          Temp borrow = bld.tmp(s1);
2066          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2067          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2068          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2069
2070       } else if (instr->src[0].src.ssa->bit_size == 64) {
2071          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2072          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2073          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2074          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2075          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2076          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2077          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2078          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2079          if (dst.type() == RegType::sgpr) {
2080             lower = bld.as_uniform(lower);
2081             upper = bld.as_uniform(upper);
2082          }
2083          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2084
2085       } else {
2086          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2087          nir_print_instr(&instr->instr, stderr);
2088          fprintf(stderr, "\n");
2089       }
2090       break;
2091    }
2092    case nir_op_f2u64: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2095          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2096          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2097          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2098          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2099          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2100          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2101          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2102          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2103          Temp new_exponent = bld.tmp(v1);
2104          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2105          if (ctx->program->chip_class >= GFX8)
2106             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2107          else
2108             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2109          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2110          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2111          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2112          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2114          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2116
2117       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2118          if (src.type() == RegType::vgpr)
2119             src = bld.as_uniform(src);
2120          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2121          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2122          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2123          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2124          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2125          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2126          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2127          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2128          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2129          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2130          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2131          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2132          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2133          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2134          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2135          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2136          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2137          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2138
2139       } else if (instr->src[0].src.ssa->bit_size == 64) {
2140          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2141          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2142          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2143          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2144          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2145          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2146          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2147          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2148          if (dst.type() == RegType::sgpr) {
2149             lower = bld.as_uniform(lower);
2150             upper = bld.as_uniform(upper);
2151          }
2152          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2153
2154       } else {
2155          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2156          nir_print_instr(&instr->instr, stderr);
2157          fprintf(stderr, "\n");
2158       }
2159       break;
2160    }
2161    case nir_op_b2f32: {
2162       Temp src = get_alu_src(ctx, instr->src[0]);
2163       assert(src.regClass() == bld.lm);
2164
2165       if (dst.regClass() == s1) {
2166          src = bool_to_scalar_condition(ctx, src);
2167          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2168       } else if (dst.regClass() == v1) {
2169          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2170       } else {
2171          unreachable("Wrong destination register class for nir_op_b2f32.");
2172       }
2173       break;
2174    }
2175    case nir_op_b2f64: {
2176       Temp src = get_alu_src(ctx, instr->src[0]);
2177       assert(src.regClass() == bld.lm);
2178
2179       if (dst.regClass() == s2) {
2180          src = bool_to_scalar_condition(ctx, src);
2181          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2182       } else if (dst.regClass() == v2) {
2183          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2184          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2185          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2186       } else {
2187          unreachable("Wrong destination register class for nir_op_b2f64.");
2188       }
2189       break;
2190    }
2191    case nir_op_i2i32: {
2192       Temp src = get_alu_src(ctx, instr->src[0]);
2193       if (instr->src[0].src.ssa->bit_size == 64) {
2194          /* we can actually just say dst = src, as it would map the lower register */
2195          emit_extract_vector(ctx, src, 0, dst);
2196       } else {
2197          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2198          nir_print_instr(&instr->instr, stderr);
2199          fprintf(stderr, "\n");
2200       }
2201       break;
2202    }
2203    case nir_op_u2u32: {
2204       Temp src = get_alu_src(ctx, instr->src[0]);
2205       if (instr->src[0].src.ssa->bit_size == 16) {
2206          if (dst.regClass() == s1) {
2207             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2208          } else {
2209             // TODO: do better with SDWA
2210             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2211          }
2212       } else if (instr->src[0].src.ssa->bit_size == 64) {
2213          /* we can actually just say dst = src, as it would map the lower register */
2214          emit_extract_vector(ctx, src, 0, dst);
2215       } else {
2216          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2217          nir_print_instr(&instr->instr, stderr);
2218          fprintf(stderr, "\n");
2219       }
2220       break;
2221    }
2222    case nir_op_i2i64: {
2223       Temp src = get_alu_src(ctx, instr->src[0]);
2224       if (src.regClass() == s1) {
2225          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2226          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2227       } else if (src.regClass() == v1) {
2228          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2230       } else {
2231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2232          nir_print_instr(&instr->instr, stderr);
2233          fprintf(stderr, "\n");
2234       }
2235       break;
2236    }
2237    case nir_op_u2u64: {
2238       Temp src = get_alu_src(ctx, instr->src[0]);
2239       if (instr->src[0].src.ssa->bit_size == 32) {
2240          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2241       } else {
2242          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2243          nir_print_instr(&instr->instr, stderr);
2244          fprintf(stderr, "\n");
2245       }
2246       break;
2247    }
2248    case nir_op_b2i32: {
2249       Temp src = get_alu_src(ctx, instr->src[0]);
2250       assert(src.regClass() == bld.lm);
2251
2252       if (dst.regClass() == s1) {
2253          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2254          bool_to_scalar_condition(ctx, src, dst);
2255       } else if (dst.regClass() == v1) {
2256          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2257       } else {
2258          unreachable("Invalid register class for b2i32");
2259       }
2260       break;
2261    }
2262    case nir_op_i2b1: {
2263       Temp src = get_alu_src(ctx, instr->src[0]);
2264       assert(dst.regClass() == bld.lm);
2265
2266       if (src.type() == RegType::vgpr) {
2267          assert(src.regClass() == v1 || src.regClass() == v2);
2268          assert(dst.regClass() == bld.lm);
2269          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2270                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2271       } else {
2272          assert(src.regClass() == s1 || src.regClass() == s2);
2273          Temp tmp;
2274          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2275             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2276          } else {
2277             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2278                            bld.scc(bld.def(s1)), Operand(0u), src);
2279          }
2280          bool_to_vector_condition(ctx, tmp, dst);
2281       }
2282       break;
2283    }
2284    case nir_op_pack_64_2x32_split: {
2285       Temp src0 = get_alu_src(ctx, instr->src[0]);
2286       Temp src1 = get_alu_src(ctx, instr->src[1]);
2287
2288       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2289       break;
2290    }
2291    case nir_op_unpack_64_2x32_split_x:
2292       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2293       break;
2294    case nir_op_unpack_64_2x32_split_y:
2295       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2296       break;
2297    case nir_op_pack_half_2x16: {
2298       Temp src = get_alu_src(ctx, instr->src[0], 2);
2299
2300       if (dst.regClass() == v1) {
2301          Temp src0 = bld.tmp(v1);
2302          Temp src1 = bld.tmp(v1);
2303          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2304          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2305             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2306          else
2307             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2308                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2309                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2310       } else {
2311          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2312          nir_print_instr(&instr->instr, stderr);
2313          fprintf(stderr, "\n");
2314       }
2315       break;
2316    }
2317    case nir_op_unpack_half_2x16_split_x: {
2318       if (dst.regClass() == v1) {
2319          Builder bld(ctx->program, ctx->block);
2320          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_unpack_half_2x16_split_y: {
2329       if (dst.regClass() == v1) {
2330          Builder bld(ctx->program, ctx->block);
2331          /* TODO: use SDWA here */
2332          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2333                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2334       } else {
2335          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2336          nir_print_instr(&instr->instr, stderr);
2337          fprintf(stderr, "\n");
2338       }
2339       break;
2340    }
2341    case nir_op_fquantize2f16: {
2342       Temp src = get_alu_src(ctx, instr->src[0]);
2343       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2344       Temp f32, cmp_res;
2345
2346       if (ctx->program->chip_class >= GFX8) {
2347          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2348          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2349          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2350       } else {
2351          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2352           * so compare the result and flush to 0 if it's smaller.
2353           */
2354          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2355          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2356          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2357          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2358          cmp_res = vop3->definitions[0].getTemp();
2359       }
2360
2361       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2362          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2363          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2364       } else {
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2366       }
2367       break;
2368    }
2369    case nir_op_bfm: {
2370       Temp bits = get_alu_src(ctx, instr->src[0]);
2371       Temp offset = get_alu_src(ctx, instr->src[1]);
2372
2373       if (dst.regClass() == s1) {
2374          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2375       } else if (dst.regClass() == v1) {
2376          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2377       } else {
2378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2379          nir_print_instr(&instr->instr, stderr);
2380          fprintf(stderr, "\n");
2381       }
2382       break;
2383    }
2384    case nir_op_bitfield_select: {
2385       /* (mask & insert) | (~mask & base) */
2386       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2387       Temp insert = get_alu_src(ctx, instr->src[1]);
2388       Temp base = get_alu_src(ctx, instr->src[2]);
2389
2390       /* dst = (insert & bitmask) | (base & ~bitmask) */
2391       if (dst.regClass() == s1) {
2392          aco_ptr<Instruction> sop2;
2393          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2394          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2395          Operand lhs;
2396          if (const_insert && const_bitmask) {
2397             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2398          } else {
2399             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2400             lhs = Operand(insert);
2401          }
2402
2403          Operand rhs;
2404          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2405          if (const_base && const_bitmask) {
2406             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2407          } else {
2408             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2409             rhs = Operand(base);
2410          }
2411
2412          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2413
2414       } else if (dst.regClass() == v1) {
2415          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2416             base = as_vgpr(ctx, base);
2417          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2418             insert = as_vgpr(ctx, insert);
2419
2420          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2421
2422       } else {
2423          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2424          nir_print_instr(&instr->instr, stderr);
2425          fprintf(stderr, "\n");
2426       }
2427       break;
2428    }
2429    case nir_op_ubfe:
2430    case nir_op_ibfe: {
2431       Temp base = get_alu_src(ctx, instr->src[0]);
2432       Temp offset = get_alu_src(ctx, instr->src[1]);
2433       Temp bits = get_alu_src(ctx, instr->src[2]);
2434
2435       if (dst.type() == RegType::sgpr) {
2436          Operand extract;
2437          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2438          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2439          if (const_offset && const_bits) {
2440             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2441             extract = Operand(const_extract);
2442          } else {
2443             Operand width;
2444             if (const_bits) {
2445                width = Operand(const_bits->u32 << 16);
2446             } else {
2447                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2448             }
2449             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2450          }
2451
2452          aco_opcode opcode;
2453          if (dst.regClass() == s1) {
2454             if (instr->op == nir_op_ubfe)
2455                opcode = aco_opcode::s_bfe_u32;
2456             else
2457                opcode = aco_opcode::s_bfe_i32;
2458          } else if (dst.regClass() == s2) {
2459             if (instr->op == nir_op_ubfe)
2460                opcode = aco_opcode::s_bfe_u64;
2461             else
2462                opcode = aco_opcode::s_bfe_i64;
2463          } else {
2464             unreachable("Unsupported BFE bit size");
2465          }
2466
2467          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2468
2469       } else {
2470          aco_opcode opcode;
2471          if (dst.regClass() == v1) {
2472             if (instr->op == nir_op_ubfe)
2473                opcode = aco_opcode::v_bfe_u32;
2474             else
2475                opcode = aco_opcode::v_bfe_i32;
2476          } else {
2477             unreachable("Unsupported BFE bit size");
2478          }
2479
2480          emit_vop3a_instruction(ctx, instr, opcode, dst);
2481       }
2482       break;
2483    }
2484    case nir_op_bit_count: {
2485       Temp src = get_alu_src(ctx, instr->src[0]);
2486       if (src.regClass() == s1) {
2487          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2488       } else if (src.regClass() == v1) {
2489          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2490       } else if (src.regClass() == v2) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2492                   emit_extract_vector(ctx, src, 1, v1),
2493                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2494                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2495       } else if (src.regClass() == s2) {
2496          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2497       } else {
2498          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2499          nir_print_instr(&instr->instr, stderr);
2500          fprintf(stderr, "\n");
2501       }
2502       break;
2503    }
2504    case nir_op_flt: {
2505       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2506       break;
2507    }
2508    case nir_op_fge: {
2509       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2510       break;
2511    }
2512    case nir_op_feq: {
2513       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2514       break;
2515    }
2516    case nir_op_fne: {
2517       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2518       break;
2519    }
2520    case nir_op_ilt: {
2521       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2522       break;
2523    }
2524    case nir_op_ige: {
2525       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2526       break;
2527    }
2528    case nir_op_ieq: {
2529       if (instr->src[0].src.ssa->bit_size == 1)
2530          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2531       else
2532          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2533                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2534       break;
2535    }
2536    case nir_op_ine: {
2537       if (instr->src[0].src.ssa->bit_size == 1)
2538          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2539       else
2540          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2541                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2542       break;
2543    }
2544    case nir_op_ult: {
2545       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2546       break;
2547    }
2548    case nir_op_uge: {
2549       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2550       break;
2551    }
2552    case nir_op_fddx:
2553    case nir_op_fddy:
2554    case nir_op_fddx_fine:
2555    case nir_op_fddy_fine:
2556    case nir_op_fddx_coarse:
2557    case nir_op_fddy_coarse: {
2558       Temp src = get_alu_src(ctx, instr->src[0]);
2559       uint16_t dpp_ctrl1, dpp_ctrl2;
2560       if (instr->op == nir_op_fddx_fine) {
2561          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2562          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2563       } else if (instr->op == nir_op_fddy_fine) {
2564          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2565          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2566       } else {
2567          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2568          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2569             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2570          else
2571             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2572       }
2573
2574       Temp tmp;
2575       if (ctx->program->chip_class >= GFX8) {
2576          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2577          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2578       } else {
2579          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2580          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2581          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2582       }
2583       emit_wqm(ctx, tmp, dst, true);
2584       break;
2585    }
2586    default:
2587       fprintf(stderr, "Unknown NIR ALU instr: ");
2588       nir_print_instr(&instr->instr, stderr);
2589       fprintf(stderr, "\n");
2590    }
2591 }
2592
2593 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2594 {
2595    Temp dst = get_ssa_temp(ctx, &instr->def);
2596
2597    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2598    // which get truncated the lsb if double and msb if int
2599    // for now, we only use s_mov_b64 with 64bit inline constants
2600    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2601    assert(dst.type() == RegType::sgpr);
2602
2603    Builder bld(ctx->program, ctx->block);
2604
2605    if (instr->def.bit_size == 1) {
2606       assert(dst.regClass() == bld.lm);
2607       int val = instr->value[0].b ? -1 : 0;
2608       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2609       bld.sop1(Builder::s_mov, Definition(dst), op);
2610    } else if (dst.size() == 1) {
2611       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2612    } else {
2613       assert(dst.size() != 1);
2614       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2615       if (instr->def.bit_size == 64)
2616          for (unsigned i = 0; i < dst.size(); i++)
2617             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2618       else {
2619          for (unsigned i = 0; i < dst.size(); i++)
2620             vec->operands[i] = Operand{instr->value[i].u32};
2621       }
2622       vec->definitions[0] = Definition(dst);
2623       ctx->block->instructions.emplace_back(std::move(vec));
2624    }
2625 }
2626
2627 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2628 {
2629    uint32_t new_mask = 0;
2630    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2631       if (mask & (1u << i))
2632          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2633    return new_mask;
2634 }
2635
2636 Operand load_lds_size_m0(isel_context *ctx)
2637 {
2638    /* TODO: m0 does not need to be initialized on GFX9+ */
2639    Builder bld(ctx->program, ctx->block);
2640    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2641 }
2642
2643 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2644               Temp address, unsigned base_offset, unsigned align)
2645 {
2646    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2647
2648    Builder bld(ctx->program, ctx->block);
2649
2650    Operand m = load_lds_size_m0(ctx);
2651
2652    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2653    unsigned bytes_read = 0;
2654    unsigned result_size = 0;
2655    unsigned total_bytes = num_components * elem_size_bytes;
2656    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2657    bool large_ds_read = ctx->options->chip_class >= GFX7;
2658
2659    while (bytes_read < total_bytes) {
2660       unsigned todo = total_bytes - bytes_read;
2661       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2662       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2663
2664       aco_opcode op = aco_opcode::last_opcode;
2665       bool read2 = false;
2666       if (todo >= 16 && aligned16 && large_ds_read) {
2667          op = aco_opcode::ds_read_b128;
2668          todo = 16;
2669       } else if (todo >= 16 && aligned8) {
2670          op = aco_opcode::ds_read2_b64;
2671          read2 = true;
2672          todo = 16;
2673       } else if (todo >= 12 && aligned16 && large_ds_read) {
2674          op = aco_opcode::ds_read_b96;
2675          todo = 12;
2676       } else if (todo >= 8 && aligned8) {
2677          op = aco_opcode::ds_read_b64;
2678          todo = 8;
2679       } else if (todo >= 8) {
2680          op = aco_opcode::ds_read2_b32;
2681          read2 = true;
2682          todo = 8;
2683       } else if (todo >= 4) {
2684          op = aco_opcode::ds_read_b32;
2685          todo = 4;
2686       } else {
2687          assert(false);
2688       }
2689       assert(todo % elem_size_bytes == 0);
2690       unsigned num_elements = todo / elem_size_bytes;
2691       unsigned offset = base_offset + bytes_read;
2692       unsigned max_offset = read2 ? 1019 : 65535;
2693
2694       Temp address_offset = address;
2695       if (offset > max_offset) {
2696          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2697          offset = bytes_read;
2698       }
2699       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2700
2701       Temp res;
2702       if (num_components == 1 && dst.type() == RegType::vgpr)
2703          res = dst;
2704       else
2705          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2706
2707       if (read2)
2708          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2709       else
2710          res = bld.ds(op, Definition(res), address_offset, m, offset);
2711
2712       if (num_components == 1) {
2713          assert(todo == total_bytes);
2714          if (dst.type() == RegType::sgpr)
2715             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2716          return;
2717       }
2718
2719       if (dst.type() == RegType::sgpr) {
2720          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2721          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2722          res = new_res;
2723       }
2724
2725       if (num_elements == 1) {
2726          result[result_size++] = res;
2727       } else {
2728          assert(res != dst && res.size() % num_elements == 0);
2729          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2730          split->operands[0] = Operand(res);
2731          for (unsigned i = 0; i < num_elements; i++)
2732             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2733          ctx->block->instructions.emplace_back(std::move(split));
2734       }
2735
2736       bytes_read += todo;
2737    }
2738
2739    assert(result_size == num_components && result_size > 1);
2740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2741    for (unsigned i = 0; i < result_size; i++)
2742       vec->operands[i] = Operand(result[i]);
2743    vec->definitions[0] = Definition(dst);
2744    ctx->block->instructions.emplace_back(std::move(vec));
2745    ctx->allocated_vec.emplace(dst.id(), result);
2746 }
2747
2748 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2749 {
2750    if (start == 0 && size == data.size())
2751       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2752
2753    unsigned size_hint = 1;
2754    auto it = ctx->allocated_vec.find(data.id());
2755    if (it != ctx->allocated_vec.end())
2756       size_hint = it->second[0].size();
2757    if (size % size_hint || start % size_hint)
2758       size_hint = 1;
2759
2760    start /= size_hint;
2761    size /= size_hint;
2762
2763    Temp elems[size];
2764    for (unsigned i = 0; i < size; i++)
2765       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2766
2767    if (size == 1)
2768       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2769
2770    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2771    for (unsigned i = 0; i < size; i++)
2772       vec->operands[i] = Operand(elems[i]);
2773    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2774    vec->definitions[0] = Definition(res);
2775    ctx->block->instructions.emplace_back(std::move(vec));
2776    return res;
2777 }
2778
2779 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2780 {
2781    Builder bld(ctx->program, ctx->block);
2782    unsigned bytes_written = 0;
2783    bool large_ds_write = ctx->options->chip_class >= GFX7;
2784
2785    while (bytes_written < total_size * 4) {
2786       unsigned todo = total_size * 4 - bytes_written;
2787       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2788       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2789
2790       aco_opcode op = aco_opcode::last_opcode;
2791       bool write2 = false;
2792       unsigned size = 0;
2793       if (todo >= 16 && aligned16 && large_ds_write) {
2794          op = aco_opcode::ds_write_b128;
2795          size = 4;
2796       } else if (todo >= 16 && aligned8) {
2797          op = aco_opcode::ds_write2_b64;
2798          write2 = true;
2799          size = 4;
2800       } else if (todo >= 12 && aligned16 && large_ds_write) {
2801          op = aco_opcode::ds_write_b96;
2802          size = 3;
2803       } else if (todo >= 8 && aligned8) {
2804          op = aco_opcode::ds_write_b64;
2805          size = 2;
2806       } else if (todo >= 8) {
2807          op = aco_opcode::ds_write2_b32;
2808          write2 = true;
2809          size = 2;
2810       } else if (todo >= 4) {
2811          op = aco_opcode::ds_write_b32;
2812          size = 1;
2813       } else {
2814          assert(false);
2815       }
2816
2817       unsigned offset = offset0 + offset1 + bytes_written;
2818       unsigned max_offset = write2 ? 1020 : 65535;
2819       Temp address_offset = address;
2820       if (offset > max_offset) {
2821          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2822          offset = offset1 + bytes_written;
2823       }
2824       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2825
2826       if (write2) {
2827          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2828          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2829          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2830       } else {
2831          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2832          bld.ds(op, address_offset, val, m, offset);
2833       }
2834
2835       bytes_written += size * 4;
2836    }
2837 }
2838
2839 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2840                Temp address, unsigned base_offset, unsigned align)
2841 {
2842    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2843
2844    Operand m = load_lds_size_m0(ctx);
2845
2846    /* we need at most two stores for 32bit variables */
2847    int start[2], count[2];
2848    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2849    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2850    assert(wrmask == 0);
2851
2852    /* one combined store is sufficient */
2853    if (count[0] == count[1]) {
2854       Builder bld(ctx->program, ctx->block);
2855
2856       Temp address_offset = address;
2857       if ((base_offset >> 2) + start[1] > 255) {
2858          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2859          base_offset = 0;
2860       }
2861
2862       assert(count[0] == 1);
2863       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2864       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2865       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2866       base_offset = base_offset / elem_size_bytes;
2867       bld.ds(op, address_offset, val0, val1, m,
2868              base_offset + start[0], base_offset + start[1]);
2869       return;
2870    }
2871
2872    for (unsigned i = 0; i < 2; i++) {
2873       if (count[i] == 0)
2874          continue;
2875
2876       unsigned elem_size_words = elem_size_bytes / 4;
2877       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2878                       base_offset, start[i] * elem_size_bytes, align);
2879    }
2880    return;
2881 }
2882
2883 void visit_store_vsgs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2884 {
2885    unsigned write_mask = nir_intrinsic_write_mask(instr);
2886    unsigned component = nir_intrinsic_component(instr);
2887    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2888    unsigned idx = (nir_intrinsic_base(instr) + component) * 4u;
2889    Operand offset(s1);
2890    Builder bld(ctx->program, ctx->block);
2891
2892    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2893    if (off_instr->type != nir_instr_type_load_const)
2894       offset = bld.v_mul24_imm(bld.def(v1), get_ssa_temp(ctx, instr->src[1].ssa), 16u);
2895    else
2896       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 16u;
2897
2898    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
2899    if (ctx->stage == vertex_es) {
2900       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
2901
2902       Temp elems[NIR_MAX_VEC_COMPONENTS * 2];
2903       if (elem_size_bytes == 8) {
2904          for (unsigned i = 0; i < src.size() / 2; i++) {
2905             Temp elem = emit_extract_vector(ctx, src, i, v2);
2906             elems[i*2] = bld.tmp(v1);
2907             elems[i*2+1] = bld.tmp(v1);
2908             bld.pseudo(aco_opcode::p_split_vector, Definition(elems[i*2]), Definition(elems[i*2+1]), elem);
2909          }
2910          write_mask = widen_mask(write_mask, 2);
2911          elem_size_bytes /= 2u;
2912       } else {
2913          for (unsigned i = 0; i < src.size(); i++)
2914             elems[i] = emit_extract_vector(ctx, src, i, v1);
2915       }
2916
2917       while (write_mask) {
2918          unsigned index = u_bit_scan(&write_mask);
2919          unsigned offset = index * elem_size_bytes;
2920          Temp elem = emit_extract_vector(ctx, src, index, RegClass(RegType::vgpr, elem_size_bytes / 4));
2921
2922          Operand vaddr_offset(v1);
2923          unsigned const_offset = idx + offset;
2924          if (const_offset >= 4096u) {
2925             vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
2926             const_offset %= 4096u;
2927          }
2928
2929          aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
2930          mtbuf->operands[0] = vaddr_offset;
2931          mtbuf->operands[1] = Operand(esgs_ring);
2932          mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->es2gs_offset));
2933          mtbuf->operands[3] = Operand(elem);
2934          mtbuf->offen = !vaddr_offset.isUndefined();
2935          mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
2936          mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
2937          mtbuf->offset = const_offset;
2938          mtbuf->glc = true;
2939          mtbuf->slc = true;
2940          mtbuf->barrier = barrier_none;
2941          mtbuf->can_reorder = true;
2942          bld.insert(std::move(mtbuf));
2943       }
2944    } else {
2945       unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
2946
2947       Temp vertex_idx = emit_mbcnt(ctx, bld.def(v1));
2948       Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
2949       vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), vertex_idx,
2950                             bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
2951
2952       Temp lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
2953       if (!offset.isUndefined())
2954          lds_base = bld.vadd32(bld.def(v1), offset, lds_base);
2955
2956       unsigned align = 1 << (ffs(itemsize) - 1);
2957       if (idx)
2958          align = std::min(align, 1u << (ffs(idx) - 1));
2959
2960       store_lds(ctx, elem_size_bytes, src, write_mask, lds_base, idx, align);
2961    }
2962 }
2963
2964 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2965 {
2966    if (ctx->stage == vertex_vs ||
2967        ctx->stage == fragment_fs ||
2968        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
2969       unsigned write_mask = nir_intrinsic_write_mask(instr);
2970       unsigned component = nir_intrinsic_component(instr);
2971       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2972       unsigned idx = nir_intrinsic_base(instr) + component;
2973
2974       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2975       if (off_instr->type != nir_instr_type_load_const) {
2976          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2977          nir_print_instr(off_instr, stderr);
2978          fprintf(stderr, "\n");
2979       }
2980       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2981
2982       if (instr->src[0].ssa->bit_size == 64)
2983          write_mask = widen_mask(write_mask, 2);
2984
2985       for (unsigned i = 0; i < 8; ++i) {
2986          if (write_mask & (1 << i)) {
2987             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
2988             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2989          }
2990          idx++;
2991       }
2992    } else if (ctx->stage == vertex_es ||
2993               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
2994       visit_store_vsgs_output(ctx, instr);
2995    } else {
2996       unreachable("Shader stage not implemented");
2997    }
2998 }
2999
3000 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3001 {
3002    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3003    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3004
3005    Builder bld(ctx->program, ctx->block);
3006    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3007    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3008 }
3009
3010 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3011 {
3012    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3013    for (unsigned i = 0; i < num_components; i++)
3014       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3015    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3016       assert(num_components == 4);
3017       Builder bld(ctx->program, ctx->block);
3018       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3019    }
3020
3021    for (Operand& op : vec->operands)
3022       op = op.isUndefined() ? Operand(0u) : op;
3023
3024    vec->definitions[0] = Definition(dst);
3025    ctx->block->instructions.emplace_back(std::move(vec));
3026    emit_split_vector(ctx, dst, num_components);
3027    return;
3028 }
3029
3030 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3031 {
3032    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3033    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3034    unsigned idx = nir_intrinsic_base(instr);
3035    unsigned component = nir_intrinsic_component(instr);
3036    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3037
3038    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3039    if (offset) {
3040       assert(offset->u32 == 0);
3041    } else {
3042       /* the lower 15bit of the prim_mask contain the offset into LDS
3043        * while the upper bits contain the number of prims */
3044       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3045       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3046       Builder bld(ctx->program, ctx->block);
3047       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3048       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3049       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3050       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3051       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3052    }
3053
3054    if (instr->dest.ssa.num_components == 1) {
3055       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3056    } else {
3057       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3058       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3059       {
3060          Temp tmp = {ctx->program->allocateId(), v1};
3061          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3062          vec->operands[i] = Operand(tmp);
3063       }
3064       vec->definitions[0] = Definition(dst);
3065       ctx->block->instructions.emplace_back(std::move(vec));
3066    }
3067 }
3068
3069 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3070                              unsigned offset, unsigned stride, unsigned channels)
3071 {
3072    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3073    if (vtx_info->chan_byte_size != 4 && channels == 3)
3074       return false;
3075    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3076           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3077 }
3078
3079 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3080                               unsigned offset, unsigned stride, unsigned *channels)
3081 {
3082    if (!vtx_info->chan_byte_size) {
3083       *channels = vtx_info->num_channels;
3084       return vtx_info->chan_format;
3085    }
3086
3087    unsigned num_channels = *channels;
3088    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3089       unsigned new_channels = num_channels + 1;
3090       /* first, assume more loads is worse and try using a larger data format */
3091       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3092          new_channels++;
3093          /* don't make the attribute potentially out-of-bounds */
3094          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3095             new_channels = 5;
3096       }
3097
3098       if (new_channels == 5) {
3099          /* then try decreasing load size (at the cost of more loads) */
3100          new_channels = *channels;
3101          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3102             new_channels--;
3103       }
3104
3105       if (new_channels < *channels)
3106          *channels = new_channels;
3107       num_channels = new_channels;
3108    }
3109
3110    switch (vtx_info->chan_format) {
3111    case V_008F0C_BUF_DATA_FORMAT_8:
3112       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3113                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3114    case V_008F0C_BUF_DATA_FORMAT_16:
3115       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3116                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3117    case V_008F0C_BUF_DATA_FORMAT_32:
3118       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3119                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3120    }
3121    unreachable("shouldn't reach here");
3122    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3123 }
3124
3125 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3126  * so we may need to fix it up. */
3127 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3128 {
3129    Builder bld(ctx->program, ctx->block);
3130
3131    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3132       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3133
3134    /* For the integer-like cases, do a natural sign extension.
3135     *
3136     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3137     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3138     * exponent.
3139     */
3140    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3141    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3142
3143    /* Convert back to the right type. */
3144    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3145       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3146       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3147       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3148    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3149       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3150    }
3151
3152    return alpha;
3153 }
3154
3155 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3156 {
3157    Builder bld(ctx->program, ctx->block);
3158    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3159    if (ctx->stage & sw_vs) {
3160
3161       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3162       if (off_instr->type != nir_instr_type_load_const) {
3163          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3164          nir_print_instr(off_instr, stderr);
3165          fprintf(stderr, "\n");
3166       }
3167       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3168
3169       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3170
3171       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3172       unsigned component = nir_intrinsic_component(instr);
3173       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3174       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3175       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3176       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3177
3178       unsigned dfmt = attrib_format & 0xf;
3179       unsigned nfmt = (attrib_format >> 4) & 0x7;
3180       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3181
3182       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3183       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3184       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3185       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3186       if (post_shuffle)
3187          num_channels = MAX2(num_channels, 3);
3188
3189       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3190       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3191
3192       Temp index;
3193       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3194          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3195          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3196          if (divisor) {
3197             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3198             if (divisor != 1) {
3199                Temp divided = bld.tmp(v1);
3200                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3201                index = bld.vadd32(bld.def(v1), start_instance, divided);
3202             } else {
3203                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3204             }
3205          } else {
3206             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3207          }
3208       } else {
3209          index = bld.vadd32(bld.def(v1),
3210                             get_arg(ctx, ctx->args->ac.base_vertex),
3211                             get_arg(ctx, ctx->args->ac.vertex_id));
3212       }
3213
3214       Temp channels[num_channels];
3215       unsigned channel_start = 0;
3216       bool direct_fetch = false;
3217
3218       /* skip unused channels at the start */
3219       if (vtx_info->chan_byte_size && !post_shuffle) {
3220          channel_start = ffs(mask) - 1;
3221          for (unsigned i = 0; i < channel_start; i++)
3222             channels[i] = Temp(0, s1);
3223       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3224          num_channels = 3 - (ffs(mask) - 1);
3225       }
3226
3227       /* load channels */
3228       while (channel_start < num_channels) {
3229          unsigned fetch_size = num_channels - channel_start;
3230          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3231
3232          /* use MUBUF when possible to avoid possible alignment issues */
3233          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3234          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3235                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3236                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3237                           vtx_info->chan_byte_size == 4;
3238          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3239          if (!use_mubuf)
3240             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3241
3242          Temp fetch_index = index;
3243          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3244             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3245             fetch_offset = fetch_offset % attrib_stride;
3246          }
3247
3248          Operand soffset(0u);
3249          if (fetch_offset >= 4096) {
3250             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3251             fetch_offset %= 4096;
3252          }
3253
3254          aco_opcode opcode;
3255          switch (fetch_size) {
3256          case 1:
3257             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3258             break;
3259          case 2:
3260             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3261             break;
3262          case 3:
3263             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3264             break;
3265          case 4:
3266             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3267             break;
3268          default:
3269             unreachable("Unimplemented load_input vector size");
3270          }
3271
3272          Temp fetch_dst;
3273          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3274              (alpha_adjust == RADV_ALPHA_ADJUST_NONE || num_channels <= 3)) {
3275             direct_fetch = true;
3276             fetch_dst = dst;
3277          } else {
3278             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3279          }
3280
3281          if (use_mubuf) {
3282             Instruction *mubuf = bld.mubuf(opcode,
3283                                            Definition(fetch_dst), fetch_index, list, soffset,
3284                                            fetch_offset, false, true).instr;
3285             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3286          } else {
3287             Instruction *mtbuf = bld.mtbuf(opcode,
3288                                            Definition(fetch_dst), fetch_index, list, soffset,
3289                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3290             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3291          }
3292
3293          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3294
3295          if (fetch_size == 1) {
3296             channels[channel_start] = fetch_dst;
3297          } else {
3298             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3299                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3300          }
3301
3302          channel_start += fetch_size;
3303       }
3304
3305       if (!direct_fetch) {
3306          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3307                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3308
3309          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3310          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3311          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3312
3313          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3314          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3315          unsigned num_temp = 0;
3316          for (unsigned i = 0; i < dst.size(); i++) {
3317             unsigned idx = i + component;
3318             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3319                Temp channel = channels[swizzle[idx]];
3320                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3321                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3322                vec->operands[i] = Operand(channel);
3323
3324                num_temp++;
3325                elems[i] = channel;
3326             } else if (is_float && idx == 3) {
3327                vec->operands[i] = Operand(0x3f800000u);
3328             } else if (!is_float && idx == 3) {
3329                vec->operands[i] = Operand(1u);
3330             } else {
3331                vec->operands[i] = Operand(0u);
3332             }
3333          }
3334          vec->definitions[0] = Definition(dst);
3335          ctx->block->instructions.emplace_back(std::move(vec));
3336          emit_split_vector(ctx, dst, dst.size());
3337
3338          if (num_temp == dst.size())
3339             ctx->allocated_vec.emplace(dst.id(), elems);
3340       }
3341    } else if (ctx->stage == fragment_fs) {
3342       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
3343       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
3344       if (off_instr->type != nir_instr_type_load_const ||
3345           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3346          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3347          nir_print_instr(off_instr, stderr);
3348          fprintf(stderr, "\n");
3349       }
3350
3351       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3352       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
3353       if (offset) {
3354          assert(offset->u32 == 0);
3355       } else {
3356          /* the lower 15bit of the prim_mask contain the offset into LDS
3357           * while the upper bits contain the number of prims */
3358          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
3359          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3360          Builder bld(ctx->program, ctx->block);
3361          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3362          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3363          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3364          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3365          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3366       }
3367
3368       unsigned idx = nir_intrinsic_base(instr);
3369       unsigned component = nir_intrinsic_component(instr);
3370       unsigned vertex_id = 2; /* P0 */
3371
3372       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
3373          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
3374          switch (src0->u32) {
3375          case 0:
3376             vertex_id = 2; /* P0 */
3377             break;
3378          case 1:
3379             vertex_id = 0; /* P10 */
3380             break;
3381          case 2:
3382             vertex_id = 1; /* P20 */
3383             break;
3384          default:
3385             unreachable("invalid vertex index");
3386          }
3387       }
3388
3389       if (dst.size() == 1) {
3390          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
3391       } else {
3392          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3393          for (unsigned i = 0; i < dst.size(); i++)
3394             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
3395          vec->definitions[0] = Definition(dst);
3396          bld.insert(std::move(vec));
3397       }
3398
3399    } else {
3400       unreachable("Shader stage not implemented");
3401    }
3402 }
3403
3404 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3405 {
3406    assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
3407    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3408
3409    Builder bld(ctx->program, ctx->block);
3410    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3411
3412    Temp offset = Temp();
3413    if (instr->src[0].ssa->parent_instr->type != nir_instr_type_load_const) {
3414       /* better code could be created, but this case probably doesn't happen
3415        * much in practice */
3416       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
3417       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3418          Temp elem;
3419          if (ctx->stage == vertex_geometry_gs) {
3420             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3421             if (i % 2u)
3422                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3423          } else {
3424             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3425          }
3426          if (offset.id()) {
3427             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(s2)),
3428                                  Operand(i), indirect_vertex);
3429             offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), offset, elem, cond);
3430          } else {
3431             offset = elem;
3432          }
3433       }
3434       if (ctx->stage == vertex_geometry_gs)
3435          offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), offset);
3436    } else {
3437       unsigned vertex = nir_src_as_uint(instr->src[0]);
3438       if (ctx->stage == vertex_geometry_gs)
3439          offset = bld.vop3(
3440          aco_opcode::v_bfe_u32, bld.def(v1), get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3441             Operand((vertex % 2u) * 16u), Operand(16u));
3442       else
3443          offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3444    }
3445
3446    unsigned const_offset = nir_intrinsic_base(instr);
3447    const_offset += nir_intrinsic_component(instr);
3448
3449    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3450    if (off_instr->type != nir_instr_type_load_const) {
3451       Temp indirect_offset = get_ssa_temp(ctx, instr->src[1].ssa);
3452       offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u),
3453                         bld.vadd32(bld.def(v1), indirect_offset, offset));
3454    } else {
3455       const_offset += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3456    }
3457    const_offset *= 4u;
3458
3459    offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), offset);
3460
3461    unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3462
3463    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3464    if (ctx->stage == geometry_gs) {
3465       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3466
3467       const_offset *= ctx->program->wave_size;
3468
3469       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3470       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3471          aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1)};
3472       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++) {
3473          Temp subelems[2];
3474          for (unsigned j = 0; j < elem_size_bytes / 4; j++) {
3475             Operand soffset(0u);
3476             if (const_offset >= 4096u)
3477                soffset = bld.copy(bld.def(s1), Operand(const_offset / 4096u * 4096u));
3478
3479             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
3480             mubuf->definitions[0] = bld.def(v1);
3481             subelems[j] = mubuf->definitions[0].getTemp();
3482             mubuf->operands[0] = Operand(offset);
3483             mubuf->operands[1] = Operand(esgs_ring);
3484             mubuf->operands[2] = Operand(soffset);
3485             mubuf->offen = true;
3486             mubuf->offset = const_offset % 4096u;
3487             mubuf->glc = true;
3488             mubuf->dlc = ctx->options->chip_class >= GFX10;
3489             mubuf->barrier = barrier_none;
3490             mubuf->can_reorder = true;
3491             bld.insert(std::move(mubuf));
3492
3493             const_offset += ctx->program->wave_size * 4u;
3494          }
3495
3496          if (elem_size_bytes == 4)
3497             elems[i] = subelems[0];
3498          else
3499             elems[i] = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), subelems[0], subelems[1]);
3500          vec->operands[i] = Operand(elems[i]);
3501       }
3502       vec->definitions[0] = Definition(dst);
3503       ctx->block->instructions.emplace_back(std::move(vec));
3504       ctx->allocated_vec.emplace(dst.id(), elems);
3505    } else {
3506       unsigned align = 16; /* alignment of indirect offset */
3507       align = std::min(align, 1u << (ffs(itemsize) - 1));
3508       if (const_offset)
3509          align = std::min(align, 1u << (ffs(const_offset) - 1));
3510
3511       load_lds(ctx, elem_size_bytes, dst, offset, const_offset, align);
3512    }
3513 }
3514
3515 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3516 {
3517    if (ctx->program->info->need_indirect_descriptor_sets) {
3518       Builder bld(ctx->program, ctx->block);
3519       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3520       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3521       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3522    }
3523
3524    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3525 }
3526
3527
3528 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3529 {
3530    Builder bld(ctx->program, ctx->block);
3531    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3532    if (!ctx->divergent_vals[instr->dest.ssa.index])
3533       index = bld.as_uniform(index);
3534    unsigned desc_set = nir_intrinsic_desc_set(instr);
3535    unsigned binding = nir_intrinsic_binding(instr);
3536
3537    Temp desc_ptr;
3538    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3539    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3540    unsigned offset = layout->binding[binding].offset;
3541    unsigned stride;
3542    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3543        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3544       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3545       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3546       offset = pipeline_layout->push_constant_size + 16 * idx;
3547       stride = 16;
3548    } else {
3549       desc_ptr = load_desc_ptr(ctx, desc_set);
3550       stride = layout->binding[binding].size;
3551    }
3552
3553    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3554    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3555    if (stride != 1) {
3556       if (nir_const_index) {
3557          const_index = const_index * stride;
3558       } else if (index.type() == RegType::vgpr) {
3559          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3560          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3561       } else {
3562          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3563       }
3564    }
3565    if (offset) {
3566       if (nir_const_index) {
3567          const_index = const_index + offset;
3568       } else if (index.type() == RegType::vgpr) {
3569          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3570       } else {
3571          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3572       }
3573    }
3574
3575    if (nir_const_index && const_index == 0) {
3576       index = desc_ptr;
3577    } else if (index.type() == RegType::vgpr) {
3578       index = bld.vadd32(bld.def(v1),
3579                          nir_const_index ? Operand(const_index) : Operand(index),
3580                          Operand(desc_ptr));
3581    } else {
3582       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3583                        nir_const_index ? Operand(const_index) : Operand(index),
3584                        Operand(desc_ptr));
3585    }
3586
3587    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3588 }
3589
3590 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3591                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3592 {
3593    Builder bld(ctx->program, ctx->block);
3594
3595    unsigned num_bytes = dst.size() * 4;
3596    bool dlc = glc && ctx->options->chip_class >= GFX10;
3597
3598    aco_opcode op;
3599    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3600       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3601       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3602       unsigned const_offset = 0;
3603
3604       Temp lower = Temp();
3605       if (num_bytes > 16) {
3606          assert(num_components == 3 || num_components == 4);
3607          op = aco_opcode::buffer_load_dwordx4;
3608          lower = bld.tmp(v4);
3609          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3610          mubuf->definitions[0] = Definition(lower);
3611          mubuf->operands[0] = vaddr;
3612          mubuf->operands[1] = Operand(rsrc);
3613          mubuf->operands[2] = soffset;
3614          mubuf->offen = (offset.type() == RegType::vgpr);
3615          mubuf->glc = glc;
3616          mubuf->dlc = dlc;
3617          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3618          mubuf->can_reorder = readonly;
3619          bld.insert(std::move(mubuf));
3620          emit_split_vector(ctx, lower, 2);
3621          num_bytes -= 16;
3622          const_offset = 16;
3623       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3624          /* GFX6 doesn't support loading vec3, expand to vec4. */
3625          num_bytes = 16;
3626       }
3627
3628       switch (num_bytes) {
3629          case 4:
3630             op = aco_opcode::buffer_load_dword;
3631             break;
3632          case 8:
3633             op = aco_opcode::buffer_load_dwordx2;
3634             break;
3635          case 12:
3636             assert(ctx->options->chip_class > GFX6);
3637             op = aco_opcode::buffer_load_dwordx3;
3638             break;
3639          case 16:
3640             op = aco_opcode::buffer_load_dwordx4;
3641             break;
3642          default:
3643             unreachable("Load SSBO not implemented for this size.");
3644       }
3645       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3646       mubuf->operands[0] = vaddr;
3647       mubuf->operands[1] = Operand(rsrc);
3648       mubuf->operands[2] = soffset;
3649       mubuf->offen = (offset.type() == RegType::vgpr);
3650       mubuf->glc = glc;
3651       mubuf->dlc = dlc;
3652       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3653       mubuf->can_reorder = readonly;
3654       mubuf->offset = const_offset;
3655       aco_ptr<Instruction> instr = std::move(mubuf);
3656
3657       if (dst.size() > 4) {
3658          assert(lower != Temp());
3659          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3660          instr->definitions[0] = Definition(upper);
3661          bld.insert(std::move(instr));
3662          if (dst.size() == 8)
3663             emit_split_vector(ctx, upper, 2);
3664          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3665          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3666          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3667          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3668          if (dst.size() == 8)
3669             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3670       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3671          Temp vec = bld.tmp(v4);
3672          instr->definitions[0] = Definition(vec);
3673          bld.insert(std::move(instr));
3674          emit_split_vector(ctx, vec, 4);
3675
3676          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3677          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3678          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3679          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3680       }
3681
3682       if (dst.type() == RegType::sgpr) {
3683          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3684          instr->definitions[0] = Definition(vec);
3685          bld.insert(std::move(instr));
3686          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3687       } else {
3688          instr->definitions[0] = Definition(dst);
3689          bld.insert(std::move(instr));
3690          emit_split_vector(ctx, dst, num_components);
3691       }
3692    } else {
3693       switch (num_bytes) {
3694          case 4:
3695             op = aco_opcode::s_buffer_load_dword;
3696             break;
3697          case 8:
3698             op = aco_opcode::s_buffer_load_dwordx2;
3699             break;
3700          case 12:
3701          case 16:
3702             op = aco_opcode::s_buffer_load_dwordx4;
3703             break;
3704          case 24:
3705          case 32:
3706             op = aco_opcode::s_buffer_load_dwordx8;
3707             break;
3708          default:
3709             unreachable("Load SSBO not implemented for this size.");
3710       }
3711       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3712       load->operands[0] = Operand(rsrc);
3713       load->operands[1] = Operand(bld.as_uniform(offset));
3714       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3715       load->definitions[0] = Definition(dst);
3716       load->glc = glc;
3717       load->dlc = dlc;
3718       load->barrier = readonly ? barrier_none : barrier_buffer;
3719       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3720       assert(ctx->options->chip_class >= GFX8 || !glc);
3721
3722       /* trim vector */
3723       if (dst.size() == 3) {
3724          Temp vec = bld.tmp(s4);
3725          load->definitions[0] = Definition(vec);
3726          bld.insert(std::move(load));
3727          emit_split_vector(ctx, vec, 4);
3728
3729          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3730                     emit_extract_vector(ctx, vec, 0, s1),
3731                     emit_extract_vector(ctx, vec, 1, s1),
3732                     emit_extract_vector(ctx, vec, 2, s1));
3733       } else if (dst.size() == 6) {
3734          Temp vec = bld.tmp(s8);
3735          load->definitions[0] = Definition(vec);
3736          bld.insert(std::move(load));
3737          emit_split_vector(ctx, vec, 4);
3738
3739          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3740                     emit_extract_vector(ctx, vec, 0, s2),
3741                     emit_extract_vector(ctx, vec, 1, s2),
3742                     emit_extract_vector(ctx, vec, 2, s2));
3743       } else {
3744          bld.insert(std::move(load));
3745       }
3746       emit_split_vector(ctx, dst, num_components);
3747    }
3748 }
3749
3750 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3751 {
3752    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3753    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3754
3755    Builder bld(ctx->program, ctx->block);
3756
3757    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3758    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3759    unsigned binding = nir_intrinsic_binding(idx_instr);
3760    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3761
3762    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3763       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3764                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3765                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3766                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3767       if (ctx->options->chip_class >= GFX10) {
3768          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3769                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3770                       S_008F0C_RESOURCE_LEVEL(1);
3771       } else {
3772          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3773                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3774       }
3775       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3776                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3777                                      Operand(0xFFFFFFFFu),
3778                                      Operand(desc_type));
3779       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3780                         rsrc, upper_dwords);
3781    } else {
3782       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3783       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3784    }
3785
3786    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3787 }
3788
3789 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3790 {
3791    Builder bld(ctx->program, ctx->block);
3792    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3793
3794    unsigned offset = nir_intrinsic_base(instr);
3795    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3796    if (index_cv && instr->dest.ssa.bit_size == 32) {
3797
3798       unsigned count = instr->dest.ssa.num_components;
3799       unsigned start = (offset + index_cv->u32) / 4u;
3800       start -= ctx->args->ac.base_inline_push_consts;
3801       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3802          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3803          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3804          for (unsigned i = 0; i < count; ++i) {
3805             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3806             vec->operands[i] = Operand{elems[i]};
3807          }
3808          vec->definitions[0] = Definition(dst);
3809          ctx->block->instructions.emplace_back(std::move(vec));
3810          ctx->allocated_vec.emplace(dst.id(), elems);
3811          return;
3812       }
3813    }
3814
3815    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3816    if (offset != 0) // TODO check if index != 0 as well
3817       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3818    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3819    Temp vec = dst;
3820    bool trim = false;
3821    aco_opcode op;
3822
3823    switch (dst.size()) {
3824    case 1:
3825       op = aco_opcode::s_load_dword;
3826       break;
3827    case 2:
3828       op = aco_opcode::s_load_dwordx2;
3829       break;
3830    case 3:
3831       vec = bld.tmp(s4);
3832       trim = true;
3833    case 4:
3834       op = aco_opcode::s_load_dwordx4;
3835       break;
3836    case 6:
3837       vec = bld.tmp(s8);
3838       trim = true;
3839    case 8:
3840       op = aco_opcode::s_load_dwordx8;
3841       break;
3842    default:
3843       unreachable("unimplemented or forbidden load_push_constant.");
3844    }
3845
3846    bld.smem(op, Definition(vec), ptr, index);
3847
3848    if (trim) {
3849       emit_split_vector(ctx, vec, 4);
3850       RegClass rc = dst.size() == 3 ? s1 : s2;
3851       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3852                  emit_extract_vector(ctx, vec, 0, rc),
3853                  emit_extract_vector(ctx, vec, 1, rc),
3854                  emit_extract_vector(ctx, vec, 2, rc));
3855
3856    }
3857    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3858 }
3859
3860 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3861 {
3862    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3863
3864    Builder bld(ctx->program, ctx->block);
3865
3866    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3867                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3868                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3869                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3870    if (ctx->options->chip_class >= GFX10) {
3871       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3872                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3873                    S_008F0C_RESOURCE_LEVEL(1);
3874    } else {
3875       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3876                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3877    }
3878
3879    unsigned base = nir_intrinsic_base(instr);
3880    unsigned range = nir_intrinsic_range(instr);
3881
3882    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3883    if (base && offset.type() == RegType::sgpr)
3884       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3885    else if (base && offset.type() == RegType::vgpr)
3886       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3887
3888    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3889                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3890                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3891                           Operand(desc_type));
3892
3893    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3894 }
3895
3896 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3897 {
3898    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3899       ctx->cf_info.exec_potentially_empty = true;
3900
3901    ctx->program->needs_exact = true;
3902
3903    // TODO: optimize uniform conditions
3904    Builder bld(ctx->program, ctx->block);
3905    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3906    assert(src.regClass() == bld.lm);
3907    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3908    bld.pseudo(aco_opcode::p_discard_if, src);
3909    ctx->block->kind |= block_kind_uses_discard_if;
3910    return;
3911 }
3912
3913 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3914 {
3915    Builder bld(ctx->program, ctx->block);
3916
3917    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3918       ctx->cf_info.exec_potentially_empty = true;
3919
3920    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3921                     ctx->cf_info.parent_loop.has_divergent_continue;
3922
3923    if (ctx->block->loop_nest_depth &&
3924        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3925       /* we handle discards the same way as jump instructions */
3926       append_logical_end(ctx->block);
3927
3928       /* in loops, discard behaves like break */
3929       Block *linear_target = ctx->cf_info.parent_loop.exit;
3930       ctx->block->kind |= block_kind_discard;
3931
3932       if (!divergent) {
3933          /* uniform discard - loop ends here */
3934          assert(nir_instr_is_last(&instr->instr));
3935          ctx->block->kind |= block_kind_uniform;
3936          ctx->cf_info.has_branch = true;
3937          bld.branch(aco_opcode::p_branch);
3938          add_linear_edge(ctx->block->index, linear_target);
3939          return;
3940       }
3941
3942       /* we add a break right behind the discard() instructions */
3943       ctx->block->kind |= block_kind_break;
3944       unsigned idx = ctx->block->index;
3945
3946       /* remove critical edges from linear CFG */
3947       bld.branch(aco_opcode::p_branch);
3948       Block* break_block = ctx->program->create_and_insert_block();
3949       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3950       break_block->kind |= block_kind_uniform;
3951       add_linear_edge(idx, break_block);
3952       add_linear_edge(break_block->index, linear_target);
3953       bld.reset(break_block);
3954       bld.branch(aco_opcode::p_branch);
3955
3956       Block* continue_block = ctx->program->create_and_insert_block();
3957       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3958       add_linear_edge(idx, continue_block);
3959       append_logical_start(continue_block);
3960       ctx->block = continue_block;
3961
3962       return;
3963    }
3964
3965    /* it can currently happen that NIR doesn't remove the unreachable code */
3966    if (!nir_instr_is_last(&instr->instr)) {
3967       ctx->program->needs_exact = true;
3968       /* save exec somewhere temporarily so that it doesn't get
3969        * overwritten before the discard from outer exec masks */
3970       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3971       bld.pseudo(aco_opcode::p_discard_if, cond);
3972       ctx->block->kind |= block_kind_uses_discard_if;
3973       return;
3974    }
3975
3976    /* This condition is incorrect for uniformly branched discards in a loop
3977     * predicated by a divergent condition, but the above code catches that case
3978     * and the discard would end up turning into a discard_if.
3979     * For example:
3980     * if (divergent) {
3981     *    while (...) {
3982     *       if (uniform) {
3983     *          discard;
3984     *       }
3985     *    }
3986     * }
3987     */
3988    if (!ctx->cf_info.parent_if.is_divergent) {
3989       /* program just ends here */
3990       ctx->block->kind |= block_kind_uniform;
3991       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3992               0 /* enabled mask */, 9 /* dest */,
3993               false /* compressed */, true/* done */, true /* valid mask */);
3994       bld.sopp(aco_opcode::s_endpgm);
3995       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3996    } else {
3997       ctx->block->kind |= block_kind_discard;
3998       /* branch and linear edge is added by visit_if() */
3999    }
4000 }
4001
4002 enum aco_descriptor_type {
4003    ACO_DESC_IMAGE,
4004    ACO_DESC_FMASK,
4005    ACO_DESC_SAMPLER,
4006    ACO_DESC_BUFFER,
4007    ACO_DESC_PLANE_0,
4008    ACO_DESC_PLANE_1,
4009    ACO_DESC_PLANE_2,
4010 };
4011
4012 static bool
4013 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
4014    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
4015       return false;
4016    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
4017    return dim == ac_image_cube ||
4018           dim == ac_image_1darray ||
4019           dim == ac_image_2darray ||
4020           dim == ac_image_2darraymsaa;
4021 }
4022
4023 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4024                       enum aco_descriptor_type desc_type,
4025                       const nir_tex_instr *tex_instr, bool image, bool write)
4026 {
4027 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4028    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4029    if (it != ctx->tex_desc.end())
4030       return it->second;
4031 */
4032    Temp index = Temp();
4033    bool index_set = false;
4034    unsigned constant_index = 0;
4035    unsigned descriptor_set;
4036    unsigned base_index;
4037    Builder bld(ctx->program, ctx->block);
4038
4039    if (!deref_instr) {
4040       assert(tex_instr && !image);
4041       descriptor_set = 0;
4042       base_index = tex_instr->sampler_index;
4043    } else {
4044       while(deref_instr->deref_type != nir_deref_type_var) {
4045          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4046          if (!array_size)
4047             array_size = 1;
4048
4049          assert(deref_instr->deref_type == nir_deref_type_array);
4050          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4051          if (const_value) {
4052             constant_index += array_size * const_value->u32;
4053          } else {
4054             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4055             if (indirect.type() == RegType::vgpr)
4056                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4057
4058             if (array_size != 1)
4059                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4060
4061             if (!index_set) {
4062                index = indirect;
4063                index_set = true;
4064             } else {
4065                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4066             }
4067          }
4068
4069          deref_instr = nir_src_as_deref(deref_instr->parent);
4070       }
4071       descriptor_set = deref_instr->var->data.descriptor_set;
4072       base_index = deref_instr->var->data.binding;
4073    }
4074
4075    Temp list = load_desc_ptr(ctx, descriptor_set);
4076    list = convert_pointer_to_64_bit(ctx, list);
4077
4078    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4079    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4080    unsigned offset = binding->offset;
4081    unsigned stride = binding->size;
4082    aco_opcode opcode;
4083    RegClass type;
4084
4085    assert(base_index < layout->binding_count);
4086
4087    switch (desc_type) {
4088    case ACO_DESC_IMAGE:
4089       type = s8;
4090       opcode = aco_opcode::s_load_dwordx8;
4091       break;
4092    case ACO_DESC_FMASK:
4093       type = s8;
4094       opcode = aco_opcode::s_load_dwordx8;
4095       offset += 32;
4096       break;
4097    case ACO_DESC_SAMPLER:
4098       type = s4;
4099       opcode = aco_opcode::s_load_dwordx4;
4100       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4101          offset += radv_combined_image_descriptor_sampler_offset(binding);
4102       break;
4103    case ACO_DESC_BUFFER:
4104       type = s4;
4105       opcode = aco_opcode::s_load_dwordx4;
4106       break;
4107    case ACO_DESC_PLANE_0:
4108    case ACO_DESC_PLANE_1:
4109       type = s8;
4110       opcode = aco_opcode::s_load_dwordx8;
4111       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4112       break;
4113    case ACO_DESC_PLANE_2:
4114       type = s4;
4115       opcode = aco_opcode::s_load_dwordx4;
4116       offset += 64;
4117       break;
4118    default:
4119       unreachable("invalid desc_type\n");
4120    }
4121
4122    offset += constant_index * stride;
4123
4124    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4125       (!index_set || binding->immutable_samplers_equal)) {
4126       if (binding->immutable_samplers_equal)
4127          constant_index = 0;
4128
4129       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4130       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4131                         Operand(samplers[constant_index * 4 + 0]),
4132                         Operand(samplers[constant_index * 4 + 1]),
4133                         Operand(samplers[constant_index * 4 + 2]),
4134                         Operand(samplers[constant_index * 4 + 3]));
4135    }
4136
4137    Operand off;
4138    if (!index_set) {
4139       off = bld.copy(bld.def(s1), Operand(offset));
4140    } else {
4141       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4142                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4143    }
4144
4145    Temp res = bld.smem(opcode, bld.def(type), list, off);
4146
4147    if (desc_type == ACO_DESC_PLANE_2) {
4148       Temp components[8];
4149       for (unsigned i = 0; i < 8; i++)
4150          components[i] = bld.tmp(s1);
4151       bld.pseudo(aco_opcode::p_split_vector,
4152                  Definition(components[0]),
4153                  Definition(components[1]),
4154                  Definition(components[2]),
4155                  Definition(components[3]),
4156                  res);
4157
4158       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4159       bld.pseudo(aco_opcode::p_split_vector,
4160                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4161                  Definition(components[4]),
4162                  Definition(components[5]),
4163                  Definition(components[6]),
4164                  Definition(components[7]),
4165                  desc2);
4166
4167       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4168                        components[0], components[1], components[2], components[3],
4169                        components[4], components[5], components[6], components[7]);
4170    }
4171
4172    return res;
4173 }
4174
4175 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4176 {
4177    switch (dim) {
4178    case GLSL_SAMPLER_DIM_BUF:
4179       return 1;
4180    case GLSL_SAMPLER_DIM_1D:
4181       return array ? 2 : 1;
4182    case GLSL_SAMPLER_DIM_2D:
4183       return array ? 3 : 2;
4184    case GLSL_SAMPLER_DIM_MS:
4185       return array ? 4 : 3;
4186    case GLSL_SAMPLER_DIM_3D:
4187    case GLSL_SAMPLER_DIM_CUBE:
4188       return 3;
4189    case GLSL_SAMPLER_DIM_RECT:
4190    case GLSL_SAMPLER_DIM_SUBPASS:
4191       return 2;
4192    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4193       return 3;
4194    default:
4195       break;
4196    }
4197    return 0;
4198 }
4199
4200
4201 /* Adjust the sample index according to FMASK.
4202  *
4203  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4204  * which is the identity mapping. Each nibble says which physical sample
4205  * should be fetched to get that sample.
4206  *
4207  * For example, 0x11111100 means there are only 2 samples stored and
4208  * the second sample covers 3/4 of the pixel. When reading samples 0
4209  * and 1, return physical sample 0 (determined by the first two 0s
4210  * in FMASK), otherwise return physical sample 1.
4211  *
4212  * The sample index should be adjusted as follows:
4213  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4214  */
4215 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4216 {
4217    Builder bld(ctx->program, ctx->block);
4218    Temp fmask = bld.tmp(v1);
4219    unsigned dim = ctx->options->chip_class >= GFX10
4220                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4221                   : 0;
4222
4223    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4224    load->operands[0] = Operand(coords);
4225    load->operands[1] = Operand(fmask_desc_ptr);
4226    load->definitions[0] = Definition(fmask);
4227    load->glc = false;
4228    load->dlc = false;
4229    load->dmask = 0x1;
4230    load->unrm = true;
4231    load->da = da;
4232    load->dim = dim;
4233    load->can_reorder = true; /* fmask images shouldn't be modified */
4234    ctx->block->instructions.emplace_back(std::move(load));
4235
4236    Operand sample_index4;
4237    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4238       sample_index4 = Operand(sample_index.constantValue() << 2);
4239    } else if (sample_index.regClass() == s1) {
4240       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4241    } else {
4242       assert(sample_index.regClass() == v1);
4243       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4244    }
4245
4246    Temp final_sample;
4247    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4248       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4249    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4250       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4251    else
4252       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4253
4254    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4255     * resource descriptor is 0 (invalid),
4256     */
4257    Temp compare = bld.tmp(bld.lm);
4258    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4259                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4260
4261    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4262
4263    /* Replace the MSAA sample index. */
4264    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4265 }
4266
4267 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4268 {
4269
4270    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4271    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4272    bool is_array = glsl_sampler_type_is_array(type);
4273    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4274    assert(!add_frag_pos && "Input attachments should be lowered.");
4275    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4276    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4277    int count = image_type_to_components_count(dim, is_array);
4278    std::vector<Operand> coords(count);
4279
4280    if (is_ms) {
4281       Operand sample_index;
4282       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4283       if (sample_cv)
4284          sample_index = Operand(sample_cv->u32);
4285       else
4286          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4287
4288       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4289          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4290          for (unsigned i = 0; i < vec->operands.size(); i++)
4291             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4292          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4293          vec->definitions[0] = Definition(fmask_load_address);
4294          ctx->block->instructions.emplace_back(std::move(vec));
4295
4296          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4297          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4298       }
4299       count--;
4300       coords[count] = sample_index;
4301    }
4302
4303    if (count == 1 && !gfx9_1d)
4304       return emit_extract_vector(ctx, src0, 0, v1);
4305
4306    if (gfx9_1d) {
4307       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4308       coords.resize(coords.size() + 1);
4309       coords[1] = Operand((uint32_t) 0);
4310       if (is_array)
4311          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4312    } else {
4313       for (int i = 0; i < count; i++)
4314          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4315    }
4316
4317    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4318        instr->intrinsic == nir_intrinsic_image_deref_store) {
4319       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4320       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4321
4322       if (!level_zero)
4323          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4324    }
4325
4326    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4327    for (unsigned i = 0; i < coords.size(); i++)
4328       vec->operands[i] = coords[i];
4329    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4330    vec->definitions[0] = Definition(res);
4331    ctx->block->instructions.emplace_back(std::move(vec));
4332    return res;
4333 }
4334
4335
4336 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4337 {
4338    Builder bld(ctx->program, ctx->block);
4339    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4340    const struct glsl_type *type = glsl_without_array(var->type);
4341    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4342    bool is_array = glsl_sampler_type_is_array(type);
4343    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4344
4345    if (dim == GLSL_SAMPLER_DIM_BUF) {
4346       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4347       unsigned num_channels = util_last_bit(mask);
4348       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4349       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4350
4351       aco_opcode opcode;
4352       switch (num_channels) {
4353       case 1:
4354          opcode = aco_opcode::buffer_load_format_x;
4355          break;
4356       case 2:
4357          opcode = aco_opcode::buffer_load_format_xy;
4358          break;
4359       case 3:
4360          opcode = aco_opcode::buffer_load_format_xyz;
4361          break;
4362       case 4:
4363          opcode = aco_opcode::buffer_load_format_xyzw;
4364          break;
4365       default:
4366          unreachable(">4 channel buffer image load");
4367       }
4368       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4369       load->operands[0] = Operand(vindex);
4370       load->operands[1] = Operand(rsrc);
4371       load->operands[2] = Operand((uint32_t) 0);
4372       Temp tmp;
4373       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4374          tmp = dst;
4375       else
4376          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4377       load->definitions[0] = Definition(tmp);
4378       load->idxen = true;
4379       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4380       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4381       load->barrier = barrier_image;
4382       ctx->block->instructions.emplace_back(std::move(load));
4383
4384       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4385       return;
4386    }
4387
4388    Temp coords = get_image_coords(ctx, instr, type);
4389    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4390
4391    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4392    unsigned num_components = util_bitcount(dmask);
4393    Temp tmp;
4394    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4395       tmp = dst;
4396    else
4397       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4398
4399    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4400    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4401
4402    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4403    load->operands[0] = Operand(coords);
4404    load->operands[1] = Operand(resource);
4405    load->definitions[0] = Definition(tmp);
4406    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4407    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4408    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4409    load->dmask = dmask;
4410    load->unrm = true;
4411    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4412    load->barrier = barrier_image;
4413    ctx->block->instructions.emplace_back(std::move(load));
4414
4415    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4416    return;
4417 }
4418
4419 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4420 {
4421    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4422    const struct glsl_type *type = glsl_without_array(var->type);
4423    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4424    bool is_array = glsl_sampler_type_is_array(type);
4425    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4426
4427    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4428
4429    if (dim == GLSL_SAMPLER_DIM_BUF) {
4430       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4431       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4432       aco_opcode opcode;
4433       switch (data.size()) {
4434       case 1:
4435          opcode = aco_opcode::buffer_store_format_x;
4436          break;
4437       case 2:
4438          opcode = aco_opcode::buffer_store_format_xy;
4439          break;
4440       case 3:
4441          opcode = aco_opcode::buffer_store_format_xyz;
4442          break;
4443       case 4:
4444          opcode = aco_opcode::buffer_store_format_xyzw;
4445          break;
4446       default:
4447          unreachable(">4 channel buffer image store");
4448       }
4449       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4450       store->operands[0] = Operand(vindex);
4451       store->operands[1] = Operand(rsrc);
4452       store->operands[2] = Operand((uint32_t) 0);
4453       store->operands[3] = Operand(data);
4454       store->idxen = true;
4455       store->glc = glc;
4456       store->dlc = false;
4457       store->disable_wqm = true;
4458       store->barrier = barrier_image;
4459       ctx->program->needs_exact = true;
4460       ctx->block->instructions.emplace_back(std::move(store));
4461       return;
4462    }
4463
4464    assert(data.type() == RegType::vgpr);
4465    Temp coords = get_image_coords(ctx, instr, type);
4466    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4467
4468    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4469    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4470
4471    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4472    store->operands[0] = Operand(coords);
4473    store->operands[1] = Operand(resource);
4474    store->operands[2] = Operand(s4);
4475    store->operands[3] = Operand(data);
4476    store->glc = glc;
4477    store->dlc = false;
4478    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4479    store->dmask = (1 << data.size()) - 1;
4480    store->unrm = true;
4481    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4482    store->disable_wqm = true;
4483    store->barrier = barrier_image;
4484    ctx->program->needs_exact = true;
4485    ctx->block->instructions.emplace_back(std::move(store));
4486    return;
4487 }
4488
4489 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4490 {
4491    /* return the previous value if dest is ever used */
4492    bool return_previous = false;
4493    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4494       return_previous = true;
4495       break;
4496    }
4497    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4498       return_previous = true;
4499       break;
4500    }
4501
4502    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4503    const struct glsl_type *type = glsl_without_array(var->type);
4504    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4505    bool is_array = glsl_sampler_type_is_array(type);
4506    Builder bld(ctx->program, ctx->block);
4507
4508    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4509    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4510
4511    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4512       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4513
4514    aco_opcode buf_op, image_op;
4515    switch (instr->intrinsic) {
4516       case nir_intrinsic_image_deref_atomic_add:
4517          buf_op = aco_opcode::buffer_atomic_add;
4518          image_op = aco_opcode::image_atomic_add;
4519          break;
4520       case nir_intrinsic_image_deref_atomic_umin:
4521          buf_op = aco_opcode::buffer_atomic_umin;
4522          image_op = aco_opcode::image_atomic_umin;
4523          break;
4524       case nir_intrinsic_image_deref_atomic_imin:
4525          buf_op = aco_opcode::buffer_atomic_smin;
4526          image_op = aco_opcode::image_atomic_smin;
4527          break;
4528       case nir_intrinsic_image_deref_atomic_umax:
4529          buf_op = aco_opcode::buffer_atomic_umax;
4530          image_op = aco_opcode::image_atomic_umax;
4531          break;
4532       case nir_intrinsic_image_deref_atomic_imax:
4533          buf_op = aco_opcode::buffer_atomic_smax;
4534          image_op = aco_opcode::image_atomic_smax;
4535          break;
4536       case nir_intrinsic_image_deref_atomic_and:
4537          buf_op = aco_opcode::buffer_atomic_and;
4538          image_op = aco_opcode::image_atomic_and;
4539          break;
4540       case nir_intrinsic_image_deref_atomic_or:
4541          buf_op = aco_opcode::buffer_atomic_or;
4542          image_op = aco_opcode::image_atomic_or;
4543          break;
4544       case nir_intrinsic_image_deref_atomic_xor:
4545          buf_op = aco_opcode::buffer_atomic_xor;
4546          image_op = aco_opcode::image_atomic_xor;
4547          break;
4548       case nir_intrinsic_image_deref_atomic_exchange:
4549          buf_op = aco_opcode::buffer_atomic_swap;
4550          image_op = aco_opcode::image_atomic_swap;
4551          break;
4552       case nir_intrinsic_image_deref_atomic_comp_swap:
4553          buf_op = aco_opcode::buffer_atomic_cmpswap;
4554          image_op = aco_opcode::image_atomic_cmpswap;
4555          break;
4556       default:
4557          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4558    }
4559
4560    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4561
4562    if (dim == GLSL_SAMPLER_DIM_BUF) {
4563       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4564       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4565       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4566       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4567       mubuf->operands[0] = Operand(vindex);
4568       mubuf->operands[1] = Operand(resource);
4569       mubuf->operands[2] = Operand((uint32_t)0);
4570       mubuf->operands[3] = Operand(data);
4571       if (return_previous)
4572          mubuf->definitions[0] = Definition(dst);
4573       mubuf->offset = 0;
4574       mubuf->idxen = true;
4575       mubuf->glc = return_previous;
4576       mubuf->dlc = false; /* Not needed for atomics */
4577       mubuf->disable_wqm = true;
4578       mubuf->barrier = barrier_image;
4579       ctx->program->needs_exact = true;
4580       ctx->block->instructions.emplace_back(std::move(mubuf));
4581       return;
4582    }
4583
4584    Temp coords = get_image_coords(ctx, instr, type);
4585    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4586    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4587    mimg->operands[0] = Operand(coords);
4588    mimg->operands[1] = Operand(resource);
4589    mimg->operands[2] = Operand(s4); /* no sampler */
4590    mimg->operands[3] = Operand(data);
4591    if (return_previous)
4592       mimg->definitions[0] = Definition(dst);
4593    mimg->glc = return_previous;
4594    mimg->dlc = false; /* Not needed for atomics */
4595    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4596    mimg->dmask = (1 << data.size()) - 1;
4597    mimg->unrm = true;
4598    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4599    mimg->disable_wqm = true;
4600    mimg->barrier = barrier_image;
4601    ctx->program->needs_exact = true;
4602    ctx->block->instructions.emplace_back(std::move(mimg));
4603    return;
4604 }
4605
4606 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4607 {
4608    if (in_elements && ctx->options->chip_class == GFX8) {
4609       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4610       Builder bld(ctx->program, ctx->block);
4611
4612       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4613
4614       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4615       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4616
4617       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4618       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4619
4620       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4621       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4622
4623       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4624       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4625                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4626       if (dst.type() == RegType::vgpr)
4627          bld.copy(Definition(dst), shr_dst);
4628
4629       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4630    } else {
4631       emit_extract_vector(ctx, desc, 2, dst);
4632    }
4633 }
4634
4635 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4636 {
4637    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4638    const struct glsl_type *type = glsl_without_array(var->type);
4639    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4640    bool is_array = glsl_sampler_type_is_array(type);
4641    Builder bld(ctx->program, ctx->block);
4642
4643    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4644       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4645       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4646    }
4647
4648    /* LOD */
4649    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4650
4651    /* Resource */
4652    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4653
4654    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4655
4656    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4657    mimg->operands[0] = Operand(lod);
4658    mimg->operands[1] = Operand(resource);
4659    uint8_t& dmask = mimg->dmask;
4660    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4661    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4662    mimg->da = glsl_sampler_type_is_array(type);
4663    mimg->can_reorder = true;
4664    Definition& def = mimg->definitions[0];
4665    ctx->block->instructions.emplace_back(std::move(mimg));
4666
4667    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4668        glsl_sampler_type_is_array(type)) {
4669
4670       assert(instr->dest.ssa.num_components == 3);
4671       Temp tmp = {ctx->program->allocateId(), v3};
4672       def = Definition(tmp);
4673       emit_split_vector(ctx, tmp, 3);
4674
4675       /* divide 3rd value by 6 by multiplying with magic number */
4676       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4677       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4678
4679       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4680                  emit_extract_vector(ctx, tmp, 0, v1),
4681                  emit_extract_vector(ctx, tmp, 1, v1),
4682                  by_6);
4683
4684    } else if (ctx->options->chip_class == GFX9 &&
4685               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4686               glsl_sampler_type_is_array(type)) {
4687       assert(instr->dest.ssa.num_components == 2);
4688       def = Definition(dst);
4689       dmask = 0x5;
4690    } else {
4691       def = Definition(dst);
4692    }
4693
4694    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4695 }
4696
4697 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4698 {
4699    Builder bld(ctx->program, ctx->block);
4700    unsigned num_components = instr->num_components;
4701
4702    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4703    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4704    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4705
4706    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4707    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4708 }
4709
4710 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4711 {
4712    Builder bld(ctx->program, ctx->block);
4713    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4714    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4715    unsigned writemask = nir_intrinsic_write_mask(instr);
4716    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4717
4718    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4719    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4720
4721    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4722                ctx->options->chip_class >= GFX8;
4723    if (smem)
4724       offset = bld.as_uniform(offset);
4725    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4726
4727    while (writemask) {
4728       int start, count;
4729       u_bit_scan_consecutive_range(&writemask, &start, &count);
4730       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4731          /* GFX6 doesn't support storing vec3, split it. */
4732          writemask |= 1u << (start + 2);
4733          count = 2;
4734       }
4735       int num_bytes = count * elem_size_bytes;
4736
4737       if (num_bytes > 16) {
4738          assert(elem_size_bytes == 8);
4739          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4740          count = 2;
4741          num_bytes = 16;
4742       }
4743
4744       // TODO: check alignment of sub-dword stores
4745       // TODO: split 3 bytes. there is no store instruction for that
4746
4747       Temp write_data;
4748       if (count != instr->num_components) {
4749          emit_split_vector(ctx, data, instr->num_components);
4750          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4751          for (int i = 0; i < count; i++) {
4752             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4753             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4754          }
4755          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4756          vec->definitions[0] = Definition(write_data);
4757          ctx->block->instructions.emplace_back(std::move(vec));
4758       } else if (!smem && data.type() != RegType::vgpr) {
4759          assert(num_bytes % 4 == 0);
4760          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4761       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4762          assert(num_bytes % 4 == 0);
4763          write_data = bld.as_uniform(data);
4764       } else {
4765          write_data = data;
4766       }
4767
4768       aco_opcode vmem_op, smem_op;
4769       switch (num_bytes) {
4770          case 4:
4771             vmem_op = aco_opcode::buffer_store_dword;
4772             smem_op = aco_opcode::s_buffer_store_dword;
4773             break;
4774          case 8:
4775             vmem_op = aco_opcode::buffer_store_dwordx2;
4776             smem_op = aco_opcode::s_buffer_store_dwordx2;
4777             break;
4778          case 12:
4779             vmem_op = aco_opcode::buffer_store_dwordx3;
4780             smem_op = aco_opcode::last_opcode;
4781             assert(!smem && ctx->options->chip_class > GFX6);
4782             break;
4783          case 16:
4784             vmem_op = aco_opcode::buffer_store_dwordx4;
4785             smem_op = aco_opcode::s_buffer_store_dwordx4;
4786             break;
4787          default:
4788             unreachable("Store SSBO not implemented for this size.");
4789       }
4790       if (ctx->stage == fragment_fs)
4791          smem_op = aco_opcode::p_fs_buffer_store_smem;
4792
4793       if (smem) {
4794          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4795          store->operands[0] = Operand(rsrc);
4796          if (start) {
4797             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4798                                 offset, Operand(start * elem_size_bytes));
4799             store->operands[1] = Operand(off);
4800          } else {
4801             store->operands[1] = Operand(offset);
4802          }
4803          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4804             store->operands[1].setFixed(m0);
4805          store->operands[2] = Operand(write_data);
4806          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4807          store->dlc = false;
4808          store->disable_wqm = true;
4809          store->barrier = barrier_buffer;
4810          ctx->block->instructions.emplace_back(std::move(store));
4811          ctx->program->wb_smem_l1_on_end = true;
4812          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4813             ctx->block->kind |= block_kind_needs_lowering;
4814             ctx->program->needs_exact = true;
4815          }
4816       } else {
4817          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4818          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4819          store->operands[1] = Operand(rsrc);
4820          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4821          store->operands[3] = Operand(write_data);
4822          store->offset = start * elem_size_bytes;
4823          store->offen = (offset.type() == RegType::vgpr);
4824          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4825          store->dlc = false;
4826          store->disable_wqm = true;
4827          store->barrier = barrier_buffer;
4828          ctx->program->needs_exact = true;
4829          ctx->block->instructions.emplace_back(std::move(store));
4830       }
4831    }
4832 }
4833
4834 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4835 {
4836    /* return the previous value if dest is ever used */
4837    bool return_previous = false;
4838    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4839       return_previous = true;
4840       break;
4841    }
4842    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4843       return_previous = true;
4844       break;
4845    }
4846
4847    Builder bld(ctx->program, ctx->block);
4848    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4849
4850    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4851       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4852                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4853
4854    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4855    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4856    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4857
4858    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4859
4860    aco_opcode op32, op64;
4861    switch (instr->intrinsic) {
4862       case nir_intrinsic_ssbo_atomic_add:
4863          op32 = aco_opcode::buffer_atomic_add;
4864          op64 = aco_opcode::buffer_atomic_add_x2;
4865          break;
4866       case nir_intrinsic_ssbo_atomic_imin:
4867          op32 = aco_opcode::buffer_atomic_smin;
4868          op64 = aco_opcode::buffer_atomic_smin_x2;
4869          break;
4870       case nir_intrinsic_ssbo_atomic_umin:
4871          op32 = aco_opcode::buffer_atomic_umin;
4872          op64 = aco_opcode::buffer_atomic_umin_x2;
4873          break;
4874       case nir_intrinsic_ssbo_atomic_imax:
4875          op32 = aco_opcode::buffer_atomic_smax;
4876          op64 = aco_opcode::buffer_atomic_smax_x2;
4877          break;
4878       case nir_intrinsic_ssbo_atomic_umax:
4879          op32 = aco_opcode::buffer_atomic_umax;
4880          op64 = aco_opcode::buffer_atomic_umax_x2;
4881          break;
4882       case nir_intrinsic_ssbo_atomic_and:
4883          op32 = aco_opcode::buffer_atomic_and;
4884          op64 = aco_opcode::buffer_atomic_and_x2;
4885          break;
4886       case nir_intrinsic_ssbo_atomic_or:
4887          op32 = aco_opcode::buffer_atomic_or;
4888          op64 = aco_opcode::buffer_atomic_or_x2;
4889          break;
4890       case nir_intrinsic_ssbo_atomic_xor:
4891          op32 = aco_opcode::buffer_atomic_xor;
4892          op64 = aco_opcode::buffer_atomic_xor_x2;
4893          break;
4894       case nir_intrinsic_ssbo_atomic_exchange:
4895          op32 = aco_opcode::buffer_atomic_swap;
4896          op64 = aco_opcode::buffer_atomic_swap_x2;
4897          break;
4898       case nir_intrinsic_ssbo_atomic_comp_swap:
4899          op32 = aco_opcode::buffer_atomic_cmpswap;
4900          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4901          break;
4902       default:
4903          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4904    }
4905    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4906    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4907    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4908    mubuf->operands[1] = Operand(rsrc);
4909    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4910    mubuf->operands[3] = Operand(data);
4911    if (return_previous)
4912       mubuf->definitions[0] = Definition(dst);
4913    mubuf->offset = 0;
4914    mubuf->offen = (offset.type() == RegType::vgpr);
4915    mubuf->glc = return_previous;
4916    mubuf->dlc = false; /* Not needed for atomics */
4917    mubuf->disable_wqm = true;
4918    mubuf->barrier = barrier_buffer;
4919    ctx->program->needs_exact = true;
4920    ctx->block->instructions.emplace_back(std::move(mubuf));
4921 }
4922
4923 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4924
4925    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4926    Builder bld(ctx->program, ctx->block);
4927    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4928    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4929 }
4930
4931 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4932 {
4933    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4934                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4935
4936    if (addr.type() == RegType::vgpr)
4937       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4938    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4939 }
4940
4941 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4942 {
4943    Builder bld(ctx->program, ctx->block);
4944    unsigned num_components = instr->num_components;
4945    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4946
4947    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4948    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4949
4950    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4951    bool dlc = glc && ctx->options->chip_class >= GFX10;
4952    aco_opcode op;
4953    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4954       bool global = ctx->options->chip_class >= GFX9;
4955
4956       if (ctx->options->chip_class >= GFX7) {
4957          aco_opcode op;
4958          switch (num_bytes) {
4959          case 4:
4960             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4961             break;
4962          case 8:
4963             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4964             break;
4965          case 12:
4966             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4967             break;
4968          case 16:
4969             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4970             break;
4971          default:
4972             unreachable("load_global not implemented for this size.");
4973          }
4974
4975          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4976          flat->operands[0] = Operand(addr);
4977          flat->operands[1] = Operand(s1);
4978          flat->glc = glc;
4979          flat->dlc = dlc;
4980          flat->barrier = barrier_buffer;
4981
4982          if (dst.type() == RegType::sgpr) {
4983             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4984             flat->definitions[0] = Definition(vec);
4985             ctx->block->instructions.emplace_back(std::move(flat));
4986             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4987          } else {
4988             flat->definitions[0] = Definition(dst);
4989             ctx->block->instructions.emplace_back(std::move(flat));
4990          }
4991          emit_split_vector(ctx, dst, num_components);
4992       } else {
4993          assert(ctx->options->chip_class == GFX6);
4994
4995          /* GFX6 doesn't support loading vec3, expand to vec4. */
4996          num_bytes = num_bytes == 12 ? 16 : num_bytes;
4997
4998          aco_opcode op;
4999          switch (num_bytes) {
5000          case 4:
5001             op = aco_opcode::buffer_load_dword;
5002             break;
5003          case 8:
5004             op = aco_opcode::buffer_load_dwordx2;
5005             break;
5006          case 16:
5007             op = aco_opcode::buffer_load_dwordx4;
5008             break;
5009          default:
5010             unreachable("load_global not implemented for this size.");
5011          }
5012
5013          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5014
5015          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
5016          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5017          mubuf->operands[1] = Operand(rsrc);
5018          mubuf->operands[2] = Operand(0u);
5019          mubuf->glc = glc;
5020          mubuf->dlc = false;
5021          mubuf->offset = 0;
5022          mubuf->addr64 = addr.type() == RegType::vgpr;
5023          mubuf->disable_wqm = false;
5024          mubuf->barrier = barrier_buffer;
5025          aco_ptr<Instruction> instr = std::move(mubuf);
5026
5027          /* expand vector */
5028          if (dst.size() == 3) {
5029             Temp vec = bld.tmp(v4);
5030             instr->definitions[0] = Definition(vec);
5031             bld.insert(std::move(instr));
5032             emit_split_vector(ctx, vec, 4);
5033
5034             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5035             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5036             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5037             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5038          }
5039
5040          if (dst.type() == RegType::sgpr) {
5041             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5042             instr->definitions[0] = Definition(vec);
5043             bld.insert(std::move(instr));
5044             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5045             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5046          } else {
5047             instr->definitions[0] = Definition(dst);
5048             bld.insert(std::move(instr));
5049             emit_split_vector(ctx, dst, num_components);
5050          }
5051       }
5052    } else {
5053       switch (num_bytes) {
5054          case 4:
5055             op = aco_opcode::s_load_dword;
5056             break;
5057          case 8:
5058             op = aco_opcode::s_load_dwordx2;
5059             break;
5060          case 12:
5061          case 16:
5062             op = aco_opcode::s_load_dwordx4;
5063             break;
5064          default:
5065             unreachable("load_global not implemented for this size.");
5066       }
5067       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5068       load->operands[0] = Operand(addr);
5069       load->operands[1] = Operand(0u);
5070       load->definitions[0] = Definition(dst);
5071       load->glc = glc;
5072       load->dlc = dlc;
5073       load->barrier = barrier_buffer;
5074       assert(ctx->options->chip_class >= GFX8 || !glc);
5075
5076       if (dst.size() == 3) {
5077          /* trim vector */
5078          Temp vec = bld.tmp(s4);
5079          load->definitions[0] = Definition(vec);
5080          ctx->block->instructions.emplace_back(std::move(load));
5081          emit_split_vector(ctx, vec, 4);
5082
5083          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5084                     emit_extract_vector(ctx, vec, 0, s1),
5085                     emit_extract_vector(ctx, vec, 1, s1),
5086                     emit_extract_vector(ctx, vec, 2, s1));
5087       } else {
5088          ctx->block->instructions.emplace_back(std::move(load));
5089       }
5090    }
5091 }
5092
5093 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5094 {
5095    Builder bld(ctx->program, ctx->block);
5096    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5097
5098    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5099    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5100
5101    if (ctx->options->chip_class >= GFX7)
5102       addr = as_vgpr(ctx, addr);
5103
5104    unsigned writemask = nir_intrinsic_write_mask(instr);
5105    while (writemask) {
5106       int start, count;
5107       u_bit_scan_consecutive_range(&writemask, &start, &count);
5108       if (count == 3 && ctx->options->chip_class == GFX6) {
5109          /* GFX6 doesn't support storing vec3, split it. */
5110          writemask |= 1u << (start + 2);
5111          count = 2;
5112       }
5113       unsigned num_bytes = count * elem_size_bytes;
5114
5115       Temp write_data = data;
5116       if (count != instr->num_components) {
5117          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5118          for (int i = 0; i < count; i++)
5119             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5120          write_data = bld.tmp(RegType::vgpr, count);
5121          vec->definitions[0] = Definition(write_data);
5122          ctx->block->instructions.emplace_back(std::move(vec));
5123       }
5124
5125       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5126       unsigned offset = start * elem_size_bytes;
5127
5128       if (ctx->options->chip_class >= GFX7) {
5129          if (offset > 0 && ctx->options->chip_class < GFX9) {
5130             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5131             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5132             Temp carry = bld.tmp(bld.lm);
5133             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5134
5135             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5136                      Operand(offset), addr0);
5137             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5138                      Operand(0u), addr1,
5139                      carry).def(1).setHint(vcc);
5140
5141             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5142
5143             offset = 0;
5144          }
5145
5146          bool global = ctx->options->chip_class >= GFX9;
5147          aco_opcode op;
5148          switch (num_bytes) {
5149          case 4:
5150             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5151             break;
5152          case 8:
5153             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5154             break;
5155          case 12:
5156             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5157             break;
5158          case 16:
5159             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5160             break;
5161          default:
5162             unreachable("store_global not implemented for this size.");
5163          }
5164
5165          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5166          flat->operands[0] = Operand(addr);
5167          flat->operands[1] = Operand(s1);
5168          flat->operands[2] = Operand(data);
5169          flat->glc = glc;
5170          flat->dlc = false;
5171          flat->offset = offset;
5172          flat->disable_wqm = true;
5173          flat->barrier = barrier_buffer;
5174          ctx->program->needs_exact = true;
5175          ctx->block->instructions.emplace_back(std::move(flat));
5176       } else {
5177          assert(ctx->options->chip_class == GFX6);
5178
5179          aco_opcode op;
5180          switch (num_bytes) {
5181          case 4:
5182             op = aco_opcode::buffer_store_dword;
5183             break;
5184          case 8:
5185             op = aco_opcode::buffer_store_dwordx2;
5186             break;
5187          case 16:
5188             op = aco_opcode::buffer_store_dwordx4;
5189             break;
5190          default:
5191             unreachable("store_global not implemented for this size.");
5192          }
5193
5194          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5195
5196          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5197          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5198          mubuf->operands[1] = Operand(rsrc);
5199          mubuf->operands[2] = Operand(0u);
5200          mubuf->operands[3] = Operand(write_data);
5201          mubuf->glc = glc;
5202          mubuf->dlc = false;
5203          mubuf->offset = offset;
5204          mubuf->addr64 = addr.type() == RegType::vgpr;
5205          mubuf->disable_wqm = true;
5206          mubuf->barrier = barrier_buffer;
5207          ctx->program->needs_exact = true;
5208          ctx->block->instructions.emplace_back(std::move(mubuf));
5209       }
5210    }
5211 }
5212
5213 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5214 {
5215    /* return the previous value if dest is ever used */
5216    bool return_previous = false;
5217    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5218       return_previous = true;
5219       break;
5220    }
5221    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5222       return_previous = true;
5223       break;
5224    }
5225
5226    Builder bld(ctx->program, ctx->block);
5227    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5228    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5229
5230    if (ctx->options->chip_class >= GFX7)
5231       addr = as_vgpr(ctx, addr);
5232
5233    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5234       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5235                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5236
5237    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5238
5239    aco_opcode op32, op64;
5240
5241    if (ctx->options->chip_class >= GFX7) {
5242       bool global = ctx->options->chip_class >= GFX9;
5243       switch (instr->intrinsic) {
5244          case nir_intrinsic_global_atomic_add:
5245             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5246             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5247             break;
5248          case nir_intrinsic_global_atomic_imin:
5249             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5250             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5251             break;
5252          case nir_intrinsic_global_atomic_umin:
5253             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5254             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5255             break;
5256          case nir_intrinsic_global_atomic_imax:
5257             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5258             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5259             break;
5260          case nir_intrinsic_global_atomic_umax:
5261             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5262             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5263             break;
5264          case nir_intrinsic_global_atomic_and:
5265             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5266             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5267             break;
5268          case nir_intrinsic_global_atomic_or:
5269             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5270             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5271             break;
5272          case nir_intrinsic_global_atomic_xor:
5273             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5274             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5275             break;
5276          case nir_intrinsic_global_atomic_exchange:
5277             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5278             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5279             break;
5280          case nir_intrinsic_global_atomic_comp_swap:
5281             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5282             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5283             break;
5284          default:
5285             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5286       }
5287
5288       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5289       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5290       flat->operands[0] = Operand(addr);
5291       flat->operands[1] = Operand(s1);
5292       flat->operands[2] = Operand(data);
5293       if (return_previous)
5294          flat->definitions[0] = Definition(dst);
5295       flat->glc = return_previous;
5296       flat->dlc = false; /* Not needed for atomics */
5297       flat->offset = 0;
5298       flat->disable_wqm = true;
5299       flat->barrier = barrier_buffer;
5300       ctx->program->needs_exact = true;
5301       ctx->block->instructions.emplace_back(std::move(flat));
5302    } else {
5303       assert(ctx->options->chip_class == GFX6);
5304
5305       switch (instr->intrinsic) {
5306          case nir_intrinsic_global_atomic_add:
5307             op32 = aco_opcode::buffer_atomic_add;
5308             op64 = aco_opcode::buffer_atomic_add_x2;
5309             break;
5310          case nir_intrinsic_global_atomic_imin:
5311             op32 = aco_opcode::buffer_atomic_smin;
5312             op64 = aco_opcode::buffer_atomic_smin_x2;
5313             break;
5314          case nir_intrinsic_global_atomic_umin:
5315             op32 = aco_opcode::buffer_atomic_umin;
5316             op64 = aco_opcode::buffer_atomic_umin_x2;
5317             break;
5318          case nir_intrinsic_global_atomic_imax:
5319             op32 = aco_opcode::buffer_atomic_smax;
5320             op64 = aco_opcode::buffer_atomic_smax_x2;
5321             break;
5322          case nir_intrinsic_global_atomic_umax:
5323             op32 = aco_opcode::buffer_atomic_umax;
5324             op64 = aco_opcode::buffer_atomic_umax_x2;
5325             break;
5326          case nir_intrinsic_global_atomic_and:
5327             op32 = aco_opcode::buffer_atomic_and;
5328             op64 = aco_opcode::buffer_atomic_and_x2;
5329             break;
5330          case nir_intrinsic_global_atomic_or:
5331             op32 = aco_opcode::buffer_atomic_or;
5332             op64 = aco_opcode::buffer_atomic_or_x2;
5333             break;
5334          case nir_intrinsic_global_atomic_xor:
5335             op32 = aco_opcode::buffer_atomic_xor;
5336             op64 = aco_opcode::buffer_atomic_xor_x2;
5337             break;
5338          case nir_intrinsic_global_atomic_exchange:
5339             op32 = aco_opcode::buffer_atomic_swap;
5340             op64 = aco_opcode::buffer_atomic_swap_x2;
5341             break;
5342          case nir_intrinsic_global_atomic_comp_swap:
5343             op32 = aco_opcode::buffer_atomic_cmpswap;
5344             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5345             break;
5346          default:
5347             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5348       }
5349
5350       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5351
5352       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5353
5354       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5355       mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5356       mubuf->operands[1] = Operand(rsrc);
5357       mubuf->operands[2] = Operand(0u);
5358       mubuf->operands[3] = Operand(data);
5359       if (return_previous)
5360          mubuf->definitions[0] = Definition(dst);
5361       mubuf->glc = return_previous;
5362       mubuf->dlc = false;
5363       mubuf->offset = 0;
5364       mubuf->addr64 = addr.type() == RegType::vgpr;
5365       mubuf->disable_wqm = true;
5366       mubuf->barrier = barrier_buffer;
5367       ctx->program->needs_exact = true;
5368       ctx->block->instructions.emplace_back(std::move(mubuf));
5369    }
5370 }
5371
5372 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5373    Builder bld(ctx->program, ctx->block);
5374    switch(instr->intrinsic) {
5375       case nir_intrinsic_group_memory_barrier:
5376       case nir_intrinsic_memory_barrier:
5377          bld.barrier(aco_opcode::p_memory_barrier_common);
5378          break;
5379       case nir_intrinsic_memory_barrier_buffer:
5380          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5381          break;
5382       case nir_intrinsic_memory_barrier_image:
5383          bld.barrier(aco_opcode::p_memory_barrier_image);
5384          break;
5385       case nir_intrinsic_memory_barrier_shared:
5386          bld.barrier(aco_opcode::p_memory_barrier_shared);
5387          break;
5388       default:
5389          unreachable("Unimplemented memory barrier intrinsic");
5390          break;
5391    }
5392 }
5393
5394 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5395 {
5396    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5397    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5398    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5399    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5400    Builder bld(ctx->program, ctx->block);
5401
5402    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5403    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5404    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5405 }
5406
5407 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5408 {
5409    unsigned writemask = nir_intrinsic_write_mask(instr);
5410    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5411    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5412    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5413    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5414
5415    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5416    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5417 }
5418
5419 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5420 {
5421    unsigned offset = nir_intrinsic_base(instr);
5422    Operand m = load_lds_size_m0(ctx);
5423    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5424    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5425
5426    unsigned num_operands = 3;
5427    aco_opcode op32, op64, op32_rtn, op64_rtn;
5428    switch(instr->intrinsic) {
5429       case nir_intrinsic_shared_atomic_add:
5430          op32 = aco_opcode::ds_add_u32;
5431          op64 = aco_opcode::ds_add_u64;
5432          op32_rtn = aco_opcode::ds_add_rtn_u32;
5433          op64_rtn = aco_opcode::ds_add_rtn_u64;
5434          break;
5435       case nir_intrinsic_shared_atomic_imin:
5436          op32 = aco_opcode::ds_min_i32;
5437          op64 = aco_opcode::ds_min_i64;
5438          op32_rtn = aco_opcode::ds_min_rtn_i32;
5439          op64_rtn = aco_opcode::ds_min_rtn_i64;
5440          break;
5441       case nir_intrinsic_shared_atomic_umin:
5442          op32 = aco_opcode::ds_min_u32;
5443          op64 = aco_opcode::ds_min_u64;
5444          op32_rtn = aco_opcode::ds_min_rtn_u32;
5445          op64_rtn = aco_opcode::ds_min_rtn_u64;
5446          break;
5447       case nir_intrinsic_shared_atomic_imax:
5448          op32 = aco_opcode::ds_max_i32;
5449          op64 = aco_opcode::ds_max_i64;
5450          op32_rtn = aco_opcode::ds_max_rtn_i32;
5451          op64_rtn = aco_opcode::ds_max_rtn_i64;
5452          break;
5453       case nir_intrinsic_shared_atomic_umax:
5454          op32 = aco_opcode::ds_max_u32;
5455          op64 = aco_opcode::ds_max_u64;
5456          op32_rtn = aco_opcode::ds_max_rtn_u32;
5457          op64_rtn = aco_opcode::ds_max_rtn_u64;
5458          break;
5459       case nir_intrinsic_shared_atomic_and:
5460          op32 = aco_opcode::ds_and_b32;
5461          op64 = aco_opcode::ds_and_b64;
5462          op32_rtn = aco_opcode::ds_and_rtn_b32;
5463          op64_rtn = aco_opcode::ds_and_rtn_b64;
5464          break;
5465       case nir_intrinsic_shared_atomic_or:
5466          op32 = aco_opcode::ds_or_b32;
5467          op64 = aco_opcode::ds_or_b64;
5468          op32_rtn = aco_opcode::ds_or_rtn_b32;
5469          op64_rtn = aco_opcode::ds_or_rtn_b64;
5470          break;
5471       case nir_intrinsic_shared_atomic_xor:
5472          op32 = aco_opcode::ds_xor_b32;
5473          op64 = aco_opcode::ds_xor_b64;
5474          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5475          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5476          break;
5477       case nir_intrinsic_shared_atomic_exchange:
5478          op32 = aco_opcode::ds_write_b32;
5479          op64 = aco_opcode::ds_write_b64;
5480          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5481          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5482          break;
5483       case nir_intrinsic_shared_atomic_comp_swap:
5484          op32 = aco_opcode::ds_cmpst_b32;
5485          op64 = aco_opcode::ds_cmpst_b64;
5486          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5487          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5488          num_operands = 4;
5489          break;
5490       default:
5491          unreachable("Unhandled shared atomic intrinsic");
5492    }
5493
5494    /* return the previous value if dest is ever used */
5495    bool return_previous = false;
5496    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5497       return_previous = true;
5498       break;
5499    }
5500    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5501       return_previous = true;
5502       break;
5503    }
5504
5505    aco_opcode op;
5506    if (data.size() == 1) {
5507       assert(instr->dest.ssa.bit_size == 32);
5508       op = return_previous ? op32_rtn : op32;
5509    } else {
5510       assert(instr->dest.ssa.bit_size == 64);
5511       op = return_previous ? op64_rtn : op64;
5512    }
5513
5514    if (offset > 65535) {
5515       Builder bld(ctx->program, ctx->block);
5516       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5517       offset = 0;
5518    }
5519
5520    aco_ptr<DS_instruction> ds;
5521    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5522    ds->operands[0] = Operand(address);
5523    ds->operands[1] = Operand(data);
5524    if (num_operands == 4)
5525       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5526    ds->operands[num_operands - 1] = m;
5527    ds->offset0 = offset;
5528    if (return_previous)
5529       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5530    ctx->block->instructions.emplace_back(std::move(ds));
5531 }
5532
5533 Temp get_scratch_resource(isel_context *ctx)
5534 {
5535    Builder bld(ctx->program, ctx->block);
5536    Temp scratch_addr = ctx->program->private_segment_buffer;
5537    if (ctx->stage != compute_cs)
5538       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5539
5540    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5541                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5542
5543    if (ctx->program->chip_class >= GFX10) {
5544       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5545                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5546                    S_008F0C_RESOURCE_LEVEL(1);
5547    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5548       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5549                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5550    }
5551
5552    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5553    if (ctx->program->chip_class <= GFX8)
5554       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5555
5556    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5557 }
5558
5559 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5560    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5561    Builder bld(ctx->program, ctx->block);
5562    Temp rsrc = get_scratch_resource(ctx);
5563    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5564    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5565
5566    aco_opcode op;
5567    switch (dst.size()) {
5568       case 1:
5569          op = aco_opcode::buffer_load_dword;
5570          break;
5571       case 2:
5572          op = aco_opcode::buffer_load_dwordx2;
5573          break;
5574       case 3:
5575          op = aco_opcode::buffer_load_dwordx3;
5576          break;
5577       case 4:
5578          op = aco_opcode::buffer_load_dwordx4;
5579          break;
5580       case 6:
5581       case 8: {
5582          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5583          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5584                                 bld.def(v4), offset, rsrc,
5585                                 ctx->program->scratch_offset, 0, true);
5586          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5587                                                   aco_opcode::buffer_load_dwordx4,
5588                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5589                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5590          emit_split_vector(ctx, lower, 2);
5591          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5592          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5593          if (dst.size() == 8) {
5594             emit_split_vector(ctx, upper, 2);
5595             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5596             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5597          } else {
5598             elems[2] = upper;
5599          }
5600
5601          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5602                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5603          for (unsigned i = 0; i < dst.size() / 2; i++)
5604             vec->operands[i] = Operand(elems[i]);
5605          vec->definitions[0] = Definition(dst);
5606          bld.insert(std::move(vec));
5607          ctx->allocated_vec.emplace(dst.id(), elems);
5608          return;
5609       }
5610       default:
5611          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5612    }
5613
5614    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5615    emit_split_vector(ctx, dst, instr->num_components);
5616 }
5617
5618 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5619    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5620    Builder bld(ctx->program, ctx->block);
5621    Temp rsrc = get_scratch_resource(ctx);
5622    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5623    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5624
5625    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5626    unsigned writemask = nir_intrinsic_write_mask(instr);
5627
5628    while (writemask) {
5629       int start, count;
5630       u_bit_scan_consecutive_range(&writemask, &start, &count);
5631       int num_bytes = count * elem_size_bytes;
5632
5633       if (num_bytes > 16) {
5634          assert(elem_size_bytes == 8);
5635          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5636          count = 2;
5637          num_bytes = 16;
5638       }
5639
5640       // TODO: check alignment of sub-dword stores
5641       // TODO: split 3 bytes. there is no store instruction for that
5642
5643       Temp write_data;
5644       if (count != instr->num_components) {
5645          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5646          for (int i = 0; i < count; i++) {
5647             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5648             vec->operands[i] = Operand(elem);
5649          }
5650          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5651          vec->definitions[0] = Definition(write_data);
5652          ctx->block->instructions.emplace_back(std::move(vec));
5653       } else {
5654          write_data = data;
5655       }
5656
5657       aco_opcode op;
5658       switch (num_bytes) {
5659          case 4:
5660             op = aco_opcode::buffer_store_dword;
5661             break;
5662          case 8:
5663             op = aco_opcode::buffer_store_dwordx2;
5664             break;
5665          case 12:
5666             op = aco_opcode::buffer_store_dwordx3;
5667             break;
5668          case 16:
5669             op = aco_opcode::buffer_store_dwordx4;
5670             break;
5671          default:
5672             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5673       }
5674
5675       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5676    }
5677 }
5678
5679 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5680    uint8_t log2_ps_iter_samples;
5681    if (ctx->program->info->ps.force_persample) {
5682       log2_ps_iter_samples =
5683          util_logbase2(ctx->options->key.fs.num_samples);
5684    } else {
5685       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5686    }
5687
5688    /* The bit pattern matches that used by fixed function fragment
5689     * processing. */
5690    static const unsigned ps_iter_masks[] = {
5691       0xffff, /* not used */
5692       0x5555,
5693       0x1111,
5694       0x0101,
5695       0x0001,
5696    };
5697    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5698
5699    Builder bld(ctx->program, ctx->block);
5700
5701    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5702                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5703    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5704    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5705    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5706    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5707 }
5708
5709 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
5710    Builder bld(ctx->program, ctx->block);
5711
5712    unsigned stream = nir_intrinsic_stream_id(instr);
5713    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5714    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
5715    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
5716
5717    /* get GSVS ring */
5718    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
5719
5720    unsigned num_components =
5721       ctx->program->info->gs.num_stream_output_components[stream];
5722    assert(num_components);
5723
5724    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
5725    unsigned stream_offset = 0;
5726    for (unsigned i = 0; i < stream; i++) {
5727       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
5728       stream_offset += prev_stride * ctx->program->wave_size;
5729    }
5730
5731    /* Limit on the stride field for <= GFX7. */
5732    assert(stride < (1 << 14));
5733
5734    Temp gsvs_dwords[4];
5735    for (unsigned i = 0; i < 4; i++)
5736       gsvs_dwords[i] = bld.tmp(s1);
5737    bld.pseudo(aco_opcode::p_split_vector,
5738               Definition(gsvs_dwords[0]),
5739               Definition(gsvs_dwords[1]),
5740               Definition(gsvs_dwords[2]),
5741               Definition(gsvs_dwords[3]),
5742               gsvs_ring);
5743
5744    if (stream_offset) {
5745       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
5746
5747       Temp carry = bld.tmp(s1);
5748       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
5749       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
5750    }
5751
5752    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
5753    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
5754
5755    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5756                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
5757
5758    unsigned offset = 0;
5759    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
5760       if (ctx->program->info->gs.output_streams[i] != stream)
5761          continue;
5762
5763       for (unsigned j = 0; j < 4; j++) {
5764          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
5765             continue;
5766
5767          if (ctx->outputs.mask[i] & (1 << j)) {
5768             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
5769             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
5770             if (const_offset >= 4096u) {
5771                if (vaddr_offset.isUndefined())
5772                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
5773                else
5774                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
5775                const_offset %= 4096u;
5776             }
5777
5778             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
5779             mtbuf->operands[0] = vaddr_offset;
5780             mtbuf->operands[1] = Operand(gsvs_ring);
5781             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
5782             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
5783             mtbuf->offen = !vaddr_offset.isUndefined();
5784             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
5785             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
5786             mtbuf->offset = const_offset;
5787             mtbuf->glc = true;
5788             mtbuf->slc = true;
5789             mtbuf->barrier = barrier_gs_data;
5790             mtbuf->can_reorder = true;
5791             bld.insert(std::move(mtbuf));
5792          }
5793
5794          offset += ctx->shader->info.gs.vertices_out;
5795       }
5796
5797       /* outputs for the next vertex are undefined and keeping them around can
5798        * create invalid IR with control flow */
5799       ctx->outputs.mask[i] = 0;
5800    }
5801
5802    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
5803 }
5804
5805 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5806 {
5807    Builder bld(ctx->program, ctx->block);
5808
5809    if (cluster_size == 1) {
5810       return src;
5811    } if (op == nir_op_iand && cluster_size == 4) {
5812       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5813       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5814       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5815                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5816    } else if (op == nir_op_ior && cluster_size == 4) {
5817       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5818       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5819                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5820    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5821       //subgroupAnd(val) -> (exec & ~val) == 0
5822       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5823       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5824       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5825    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5826       //subgroupOr(val) -> (val & exec) != 0
5827       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5828       return bool_to_vector_condition(ctx, tmp);
5829    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5830       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5831       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5832       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5833       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5834       return bool_to_vector_condition(ctx, tmp);
5835    } else {
5836       //subgroupClustered{And,Or,Xor}(val, n) ->
5837       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5838       //cluster_offset = ~(n - 1) & lane_id
5839       //cluster_mask = ((1 << n) - 1)
5840       //subgroupClusteredAnd():
5841       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5842       //subgroupClusteredOr():
5843       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5844       //subgroupClusteredXor():
5845       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5846       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5847       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5848
5849       Temp tmp;
5850       if (op == nir_op_iand)
5851          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5852       else
5853          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5854
5855       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5856
5857       if (ctx->program->chip_class <= GFX7)
5858          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5859       else if (ctx->program->wave_size == 64)
5860          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5861       else
5862          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5863       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5864       if (cluster_mask != 0xffffffff)
5865          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5866
5867       Definition cmp_def = Definition();
5868       if (op == nir_op_iand) {
5869          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5870       } else if (op == nir_op_ior) {
5871          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5872       } else if (op == nir_op_ixor) {
5873          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5874                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5875          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5876       }
5877       cmp_def.setHint(vcc);
5878       return cmp_def.getTemp();
5879    }
5880 }
5881
5882 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5883 {
5884    Builder bld(ctx->program, ctx->block);
5885
5886    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5887    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5888    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5889    Temp tmp;
5890    if (op == nir_op_iand)
5891       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5892    else
5893       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5894
5895    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5896    Temp lo = lohi.def(0).getTemp();
5897    Temp hi = lohi.def(1).getTemp();
5898    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5899
5900    Definition cmp_def = Definition();
5901    if (op == nir_op_iand)
5902       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5903    else if (op == nir_op_ior)
5904       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5905    else if (op == nir_op_ixor)
5906       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5907                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5908    cmp_def.setHint(vcc);
5909    return cmp_def.getTemp();
5910 }
5911
5912 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5913 {
5914    Builder bld(ctx->program, ctx->block);
5915
5916    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5917    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5918    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5919    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5920    if (op == nir_op_iand)
5921       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5922    else if (op == nir_op_ior)
5923       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5924    else if (op == nir_op_ixor)
5925       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5926
5927    assert(false);
5928    return Temp();
5929 }
5930
5931 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5932 {
5933    Builder bld(ctx->program, ctx->block);
5934    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5935    if (src.regClass().type() == RegType::vgpr) {
5936       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5937    } else if (src.regClass() == s1) {
5938       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5939    } else if (src.regClass() == s2) {
5940       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5941    } else {
5942       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5943       nir_print_instr(&instr->instr, stderr);
5944       fprintf(stderr, "\n");
5945    }
5946 }
5947
5948 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5949 {
5950    Builder bld(ctx->program, ctx->block);
5951    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5952    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5953    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5954
5955    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5956    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5957    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5958    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5959
5960    /* Build DD X/Y */
5961    if (ctx->program->chip_class >= GFX8) {
5962       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5963       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5964       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5965       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5966       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5967       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5968    } else {
5969       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5970       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5971       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5972       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5973       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5974       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5975       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5976       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5977       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5978       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5979    }
5980
5981    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5982    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5983    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5984    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5985    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5986    Temp wqm1 = bld.tmp(v1);
5987    emit_wqm(ctx, tmp1, wqm1, true);
5988    Temp wqm2 = bld.tmp(v1);
5989    emit_wqm(ctx, tmp2, wqm2, true);
5990    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5991    return;
5992 }
5993
5994 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5995 {
5996    Builder bld(ctx->program, ctx->block);
5997    switch(instr->intrinsic) {
5998    case nir_intrinsic_load_barycentric_sample:
5999    case nir_intrinsic_load_barycentric_pixel:
6000    case nir_intrinsic_load_barycentric_centroid: {
6001       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
6002       Temp bary = Temp(0, s2);
6003       switch (mode) {
6004       case INTERP_MODE_SMOOTH:
6005       case INTERP_MODE_NONE:
6006          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6007             bary = get_arg(ctx, ctx->args->ac.persp_center);
6008          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6009             bary = ctx->persp_centroid;
6010          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6011             bary = get_arg(ctx, ctx->args->ac.persp_sample);
6012          break;
6013       case INTERP_MODE_NOPERSPECTIVE:
6014          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6015             bary = get_arg(ctx, ctx->args->ac.linear_center);
6016          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6017             bary = ctx->linear_centroid;
6018          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6019             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6020          break;
6021       default:
6022          break;
6023       }
6024       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6025       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6026       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6027       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6028                  Operand(p1), Operand(p2));
6029       emit_split_vector(ctx, dst, 2);
6030       break;
6031    }
6032    case nir_intrinsic_load_barycentric_model: {
6033       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
6034
6035       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6036       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
6037       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
6038       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
6039       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6040                  Operand(p1), Operand(p2), Operand(p3));
6041       emit_split_vector(ctx, dst, 3);
6042       break;
6043    }
6044    case nir_intrinsic_load_barycentric_at_sample: {
6045       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6046       switch (ctx->options->key.fs.num_samples) {
6047          case 2: sample_pos_offset += 1 << 3; break;
6048          case 4: sample_pos_offset += 3 << 3; break;
6049          case 8: sample_pos_offset += 7 << 3; break;
6050          default: break;
6051       }
6052       Temp sample_pos;
6053       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6054       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6055       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6056       if (addr.type() == RegType::sgpr) {
6057          Operand offset;
6058          if (const_addr) {
6059             sample_pos_offset += const_addr->u32 << 3;
6060             offset = Operand(sample_pos_offset);
6061          } else if (ctx->options->chip_class >= GFX9) {
6062             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6063          } else {
6064             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6065             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6066          }
6067
6068          Operand off = bld.copy(bld.def(s1), Operand(offset));
6069          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6070
6071       } else if (ctx->options->chip_class >= GFX9) {
6072          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6073          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6074       } else if (ctx->options->chip_class >= GFX7) {
6075          /* addr += private_segment_buffer + sample_pos_offset */
6076          Temp tmp0 = bld.tmp(s1);
6077          Temp tmp1 = bld.tmp(s1);
6078          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6079          Definition scc_tmp = bld.def(s1, scc);
6080          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6081          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6082          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6083          Temp pck0 = bld.tmp(v1);
6084          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6085          tmp1 = as_vgpr(ctx, tmp1);
6086          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6087          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6088
6089          /* sample_pos = flat_load_dwordx2 addr */
6090          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6091       } else {
6092          assert(ctx->options->chip_class == GFX6);
6093
6094          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6095                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6096          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6097
6098          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6099          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6100
6101          sample_pos = bld.tmp(v2);
6102
6103          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6104          load->definitions[0] = Definition(sample_pos);
6105          load->operands[0] = Operand(addr);
6106          load->operands[1] = Operand(rsrc);
6107          load->operands[2] = Operand(0u);
6108          load->offset = sample_pos_offset;
6109          load->offen = 0;
6110          load->addr64 = true;
6111          load->glc = false;
6112          load->dlc = false;
6113          load->disable_wqm = false;
6114          load->barrier = barrier_none;
6115          load->can_reorder = true;
6116          ctx->block->instructions.emplace_back(std::move(load));
6117       }
6118
6119       /* sample_pos -= 0.5 */
6120       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6121       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6122       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6123       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6124       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6125
6126       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6127       break;
6128    }
6129    case nir_intrinsic_load_barycentric_at_offset: {
6130       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6131       RegClass rc = RegClass(offset.type(), 1);
6132       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6133       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6134       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6135       break;
6136    }
6137    case nir_intrinsic_load_front_face: {
6138       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6139                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6140       break;
6141    }
6142    case nir_intrinsic_load_view_index:
6143    case nir_intrinsic_load_layer_id: {
6144       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & (sw_vs | sw_gs))) {
6145          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6146          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6147          break;
6148       }
6149
6150       unsigned idx = nir_intrinsic_base(instr);
6151       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6152                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6153       break;
6154    }
6155    case nir_intrinsic_load_frag_coord: {
6156       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6157       break;
6158    }
6159    case nir_intrinsic_load_sample_pos: {
6160       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6161       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6162       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6163                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6164                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6165       break;
6166    }
6167    case nir_intrinsic_load_interpolated_input:
6168       visit_load_interpolated_input(ctx, instr);
6169       break;
6170    case nir_intrinsic_store_output:
6171       visit_store_output(ctx, instr);
6172       break;
6173    case nir_intrinsic_load_input:
6174    case nir_intrinsic_load_input_vertex:
6175       visit_load_input(ctx, instr);
6176       break;
6177    case nir_intrinsic_load_per_vertex_input:
6178       visit_load_per_vertex_input(ctx, instr);
6179       break;
6180    case nir_intrinsic_load_ubo:
6181       visit_load_ubo(ctx, instr);
6182       break;
6183    case nir_intrinsic_load_push_constant:
6184       visit_load_push_constant(ctx, instr);
6185       break;
6186    case nir_intrinsic_load_constant:
6187       visit_load_constant(ctx, instr);
6188       break;
6189    case nir_intrinsic_vulkan_resource_index:
6190       visit_load_resource(ctx, instr);
6191       break;
6192    case nir_intrinsic_discard:
6193       visit_discard(ctx, instr);
6194       break;
6195    case nir_intrinsic_discard_if:
6196       visit_discard_if(ctx, instr);
6197       break;
6198    case nir_intrinsic_load_shared:
6199       visit_load_shared(ctx, instr);
6200       break;
6201    case nir_intrinsic_store_shared:
6202       visit_store_shared(ctx, instr);
6203       break;
6204    case nir_intrinsic_shared_atomic_add:
6205    case nir_intrinsic_shared_atomic_imin:
6206    case nir_intrinsic_shared_atomic_umin:
6207    case nir_intrinsic_shared_atomic_imax:
6208    case nir_intrinsic_shared_atomic_umax:
6209    case nir_intrinsic_shared_atomic_and:
6210    case nir_intrinsic_shared_atomic_or:
6211    case nir_intrinsic_shared_atomic_xor:
6212    case nir_intrinsic_shared_atomic_exchange:
6213    case nir_intrinsic_shared_atomic_comp_swap:
6214       visit_shared_atomic(ctx, instr);
6215       break;
6216    case nir_intrinsic_image_deref_load:
6217       visit_image_load(ctx, instr);
6218       break;
6219    case nir_intrinsic_image_deref_store:
6220       visit_image_store(ctx, instr);
6221       break;
6222    case nir_intrinsic_image_deref_atomic_add:
6223    case nir_intrinsic_image_deref_atomic_umin:
6224    case nir_intrinsic_image_deref_atomic_imin:
6225    case nir_intrinsic_image_deref_atomic_umax:
6226    case nir_intrinsic_image_deref_atomic_imax:
6227    case nir_intrinsic_image_deref_atomic_and:
6228    case nir_intrinsic_image_deref_atomic_or:
6229    case nir_intrinsic_image_deref_atomic_xor:
6230    case nir_intrinsic_image_deref_atomic_exchange:
6231    case nir_intrinsic_image_deref_atomic_comp_swap:
6232       visit_image_atomic(ctx, instr);
6233       break;
6234    case nir_intrinsic_image_deref_size:
6235       visit_image_size(ctx, instr);
6236       break;
6237    case nir_intrinsic_load_ssbo:
6238       visit_load_ssbo(ctx, instr);
6239       break;
6240    case nir_intrinsic_store_ssbo:
6241       visit_store_ssbo(ctx, instr);
6242       break;
6243    case nir_intrinsic_load_global:
6244       visit_load_global(ctx, instr);
6245       break;
6246    case nir_intrinsic_store_global:
6247       visit_store_global(ctx, instr);
6248       break;
6249    case nir_intrinsic_global_atomic_add:
6250    case nir_intrinsic_global_atomic_imin:
6251    case nir_intrinsic_global_atomic_umin:
6252    case nir_intrinsic_global_atomic_imax:
6253    case nir_intrinsic_global_atomic_umax:
6254    case nir_intrinsic_global_atomic_and:
6255    case nir_intrinsic_global_atomic_or:
6256    case nir_intrinsic_global_atomic_xor:
6257    case nir_intrinsic_global_atomic_exchange:
6258    case nir_intrinsic_global_atomic_comp_swap:
6259       visit_global_atomic(ctx, instr);
6260       break;
6261    case nir_intrinsic_ssbo_atomic_add:
6262    case nir_intrinsic_ssbo_atomic_imin:
6263    case nir_intrinsic_ssbo_atomic_umin:
6264    case nir_intrinsic_ssbo_atomic_imax:
6265    case nir_intrinsic_ssbo_atomic_umax:
6266    case nir_intrinsic_ssbo_atomic_and:
6267    case nir_intrinsic_ssbo_atomic_or:
6268    case nir_intrinsic_ssbo_atomic_xor:
6269    case nir_intrinsic_ssbo_atomic_exchange:
6270    case nir_intrinsic_ssbo_atomic_comp_swap:
6271       visit_atomic_ssbo(ctx, instr);
6272       break;
6273    case nir_intrinsic_load_scratch:
6274       visit_load_scratch(ctx, instr);
6275       break;
6276    case nir_intrinsic_store_scratch:
6277       visit_store_scratch(ctx, instr);
6278       break;
6279    case nir_intrinsic_get_buffer_size:
6280       visit_get_buffer_size(ctx, instr);
6281       break;
6282    case nir_intrinsic_control_barrier: {
6283       unsigned* bsize = ctx->program->info->cs.block_size;
6284       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6285       if (workgroup_size > ctx->program->wave_size)
6286          bld.sopp(aco_opcode::s_barrier);
6287       break;
6288    }
6289    case nir_intrinsic_group_memory_barrier:
6290    case nir_intrinsic_memory_barrier:
6291    case nir_intrinsic_memory_barrier_buffer:
6292    case nir_intrinsic_memory_barrier_image:
6293    case nir_intrinsic_memory_barrier_shared:
6294       emit_memory_barrier(ctx, instr);
6295       break;
6296    case nir_intrinsic_memory_barrier_tcs_patch:
6297       break;
6298    case nir_intrinsic_load_num_work_groups: {
6299       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6300       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6301       emit_split_vector(ctx, dst, 3);
6302       break;
6303    }
6304    case nir_intrinsic_load_local_invocation_id: {
6305       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6306       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6307       emit_split_vector(ctx, dst, 3);
6308       break;
6309    }
6310    case nir_intrinsic_load_work_group_id: {
6311       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6312       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6313       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6314                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6315                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6316                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6317       emit_split_vector(ctx, dst, 3);
6318       break;
6319    }
6320    case nir_intrinsic_load_local_invocation_index: {
6321       Temp id = emit_mbcnt(ctx, bld.def(v1));
6322
6323       /* The tg_size bits [6:11] contain the subgroup id,
6324        * we need this multiplied by the wave size, and then OR the thread id to it.
6325        */
6326       if (ctx->program->wave_size == 64) {
6327          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6328          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6329                                 get_arg(ctx, ctx->args->ac.tg_size));
6330          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6331       } else {
6332          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6333          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6334                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6335          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6336       }
6337       break;
6338    }
6339    case nir_intrinsic_load_subgroup_id: {
6340       if (ctx->stage == compute_cs) {
6341          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6342                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6343       } else {
6344          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6345       }
6346       break;
6347    }
6348    case nir_intrinsic_load_subgroup_invocation: {
6349       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6350       break;
6351    }
6352    case nir_intrinsic_load_num_subgroups: {
6353       if (ctx->stage == compute_cs)
6354          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6355                   get_arg(ctx, ctx->args->ac.tg_size));
6356       else
6357          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6358       break;
6359    }
6360    case nir_intrinsic_ballot: {
6361       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6362       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6363       Definition tmp = bld.def(dst.regClass());
6364       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6365       if (instr->src[0].ssa->bit_size == 1) {
6366          assert(src.regClass() == bld.lm);
6367          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6368       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6369          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6370       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6371          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6372       } else {
6373          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6374          nir_print_instr(&instr->instr, stderr);
6375          fprintf(stderr, "\n");
6376       }
6377       if (dst.size() != bld.lm.size()) {
6378          /* Wave32 with ballot size set to 64 */
6379          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6380       }
6381       emit_wqm(ctx, tmp.getTemp(), dst);
6382       break;
6383    }
6384    case nir_intrinsic_shuffle:
6385    case nir_intrinsic_read_invocation: {
6386       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6387       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6388          emit_uniform_subgroup(ctx, instr, src);
6389       } else {
6390          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6391          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6392             tid = bld.as_uniform(tid);
6393          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6394          if (src.regClass() == v1) {
6395             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6396          } else if (src.regClass() == v2) {
6397             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6398             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6399             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6400             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6401             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6402             emit_split_vector(ctx, dst, 2);
6403          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6404             assert(src.regClass() == bld.lm);
6405             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6406             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6407          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6408             assert(src.regClass() == bld.lm);
6409             Temp tmp;
6410             if (ctx->program->chip_class <= GFX7)
6411                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6412             else if (ctx->program->wave_size == 64)
6413                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6414             else
6415                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6416             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6417             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6418             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6419          } else {
6420             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6421             nir_print_instr(&instr->instr, stderr);
6422             fprintf(stderr, "\n");
6423          }
6424       }
6425       break;
6426    }
6427    case nir_intrinsic_load_sample_id: {
6428       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6429                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6430       break;
6431    }
6432    case nir_intrinsic_load_sample_mask_in: {
6433       visit_load_sample_mask_in(ctx, instr);
6434       break;
6435    }
6436    case nir_intrinsic_read_first_invocation: {
6437       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6438       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6439       if (src.regClass() == v1) {
6440          emit_wqm(ctx,
6441                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6442                   dst);
6443       } else if (src.regClass() == v2) {
6444          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6445          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6446          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6447          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6448          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6449          emit_split_vector(ctx, dst, 2);
6450       } else if (instr->dest.ssa.bit_size == 1) {
6451          assert(src.regClass() == bld.lm);
6452          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6453                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6454          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6455       } else if (src.regClass() == s1) {
6456          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6457       } else if (src.regClass() == s2) {
6458          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6459       } else {
6460          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6461          nir_print_instr(&instr->instr, stderr);
6462          fprintf(stderr, "\n");
6463       }
6464       break;
6465    }
6466    case nir_intrinsic_vote_all: {
6467       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6468       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6469       assert(src.regClass() == bld.lm);
6470       assert(dst.regClass() == bld.lm);
6471
6472       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6473       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6474       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6475       break;
6476    }
6477    case nir_intrinsic_vote_any: {
6478       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6479       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6480       assert(src.regClass() == bld.lm);
6481       assert(dst.regClass() == bld.lm);
6482
6483       Temp tmp = bool_to_scalar_condition(ctx, src);
6484       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6485       break;
6486    }
6487    case nir_intrinsic_reduce:
6488    case nir_intrinsic_inclusive_scan:
6489    case nir_intrinsic_exclusive_scan: {
6490       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6491       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6492       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6493       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6494          nir_intrinsic_cluster_size(instr) : 0;
6495       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6496
6497       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6498          emit_uniform_subgroup(ctx, instr, src);
6499       } else if (instr->dest.ssa.bit_size == 1) {
6500          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6501             op = nir_op_iand;
6502          else if (op == nir_op_iadd)
6503             op = nir_op_ixor;
6504          else if (op == nir_op_umax || op == nir_op_imax)
6505             op = nir_op_ior;
6506          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6507
6508          switch (instr->intrinsic) {
6509          case nir_intrinsic_reduce:
6510             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6511             break;
6512          case nir_intrinsic_exclusive_scan:
6513             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6514             break;
6515          case nir_intrinsic_inclusive_scan:
6516             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6517             break;
6518          default:
6519             assert(false);
6520          }
6521       } else if (cluster_size == 1) {
6522          bld.copy(Definition(dst), src);
6523       } else {
6524          src = as_vgpr(ctx, src);
6525
6526          ReduceOp reduce_op;
6527          switch (op) {
6528          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6529             CASE(iadd)
6530             CASE(imul)
6531             CASE(fadd)
6532             CASE(fmul)
6533             CASE(imin)
6534             CASE(umin)
6535             CASE(fmin)
6536             CASE(imax)
6537             CASE(umax)
6538             CASE(fmax)
6539             CASE(iand)
6540             CASE(ior)
6541             CASE(ixor)
6542             default:
6543                unreachable("unknown reduction op");
6544          #undef CASE
6545          }
6546
6547          aco_opcode aco_op;
6548          switch (instr->intrinsic) {
6549             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6550             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6551             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6552             default:
6553                unreachable("unknown reduce intrinsic");
6554          }
6555
6556          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6557          reduce->operands[0] = Operand(src);
6558          // filled in by aco_reduce_assign.cpp, used internally as part of the
6559          // reduce sequence
6560          assert(dst.size() == 1 || dst.size() == 2);
6561          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6562          reduce->operands[2] = Operand(v1.as_linear());
6563
6564          Temp tmp_dst = bld.tmp(dst.regClass());
6565          reduce->definitions[0] = Definition(tmp_dst);
6566          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6567          reduce->definitions[2] = Definition();
6568          reduce->definitions[3] = Definition(scc, s1);
6569          reduce->definitions[4] = Definition();
6570          reduce->reduce_op = reduce_op;
6571          reduce->cluster_size = cluster_size;
6572          ctx->block->instructions.emplace_back(std::move(reduce));
6573
6574          emit_wqm(ctx, tmp_dst, dst);
6575       }
6576       break;
6577    }
6578    case nir_intrinsic_quad_broadcast: {
6579       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6580       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6581          emit_uniform_subgroup(ctx, instr, src);
6582       } else {
6583          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6584          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6585          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6586
6587          if (instr->dest.ssa.bit_size == 1) {
6588             assert(src.regClass() == bld.lm);
6589             assert(dst.regClass() == bld.lm);
6590             uint32_t half_mask = 0x11111111u << lane;
6591             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6592             Temp tmp = bld.tmp(bld.lm);
6593             bld.sop1(Builder::s_wqm, Definition(tmp),
6594                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6595                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6596             emit_wqm(ctx, tmp, dst);
6597          } else if (instr->dest.ssa.bit_size == 32) {
6598             if (ctx->program->chip_class >= GFX8)
6599                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6600             else
6601                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6602          } else if (instr->dest.ssa.bit_size == 64) {
6603             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6604             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6605             if (ctx->program->chip_class >= GFX8) {
6606                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6607                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6608             } else {
6609                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6610                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6611             }
6612             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6613             emit_split_vector(ctx, dst, 2);
6614          } else {
6615             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6616             nir_print_instr(&instr->instr, stderr);
6617             fprintf(stderr, "\n");
6618          }
6619       }
6620       break;
6621    }
6622    case nir_intrinsic_quad_swap_horizontal:
6623    case nir_intrinsic_quad_swap_vertical:
6624    case nir_intrinsic_quad_swap_diagonal:
6625    case nir_intrinsic_quad_swizzle_amd: {
6626       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6627       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6628          emit_uniform_subgroup(ctx, instr, src);
6629          break;
6630       }
6631       uint16_t dpp_ctrl = 0;
6632       switch (instr->intrinsic) {
6633       case nir_intrinsic_quad_swap_horizontal:
6634          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6635          break;
6636       case nir_intrinsic_quad_swap_vertical:
6637          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6638          break;
6639       case nir_intrinsic_quad_swap_diagonal:
6640          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6641          break;
6642       case nir_intrinsic_quad_swizzle_amd:
6643          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6644          break;
6645       default:
6646          break;
6647       }
6648       if (ctx->program->chip_class < GFX8)
6649          dpp_ctrl |= (1 << 15);
6650
6651       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6652       if (instr->dest.ssa.bit_size == 1) {
6653          assert(src.regClass() == bld.lm);
6654          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6655          if (ctx->program->chip_class >= GFX8)
6656             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6657          else
6658             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6659          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6660          emit_wqm(ctx, tmp, dst);
6661       } else if (instr->dest.ssa.bit_size == 32) {
6662          Temp tmp;
6663          if (ctx->program->chip_class >= GFX8)
6664             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6665          else
6666             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6667          emit_wqm(ctx, tmp, dst);
6668       } else if (instr->dest.ssa.bit_size == 64) {
6669          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6670          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6671          if (ctx->program->chip_class >= GFX8) {
6672             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6673             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6674          } else {
6675             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6676             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6677          }
6678          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6679          emit_split_vector(ctx, dst, 2);
6680       } else {
6681          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6682          nir_print_instr(&instr->instr, stderr);
6683          fprintf(stderr, "\n");
6684       }
6685       break;
6686    }
6687    case nir_intrinsic_masked_swizzle_amd: {
6688       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6689       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6690          emit_uniform_subgroup(ctx, instr, src);
6691          break;
6692       }
6693       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6694       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6695       if (dst.regClass() == v1) {
6696          emit_wqm(ctx,
6697                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6698                   dst);
6699       } else if (dst.regClass() == v2) {
6700          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6701          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6702          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6703          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6704          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6705          emit_split_vector(ctx, dst, 2);
6706       } else {
6707          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6708          nir_print_instr(&instr->instr, stderr);
6709          fprintf(stderr, "\n");
6710       }
6711       break;
6712    }
6713    case nir_intrinsic_write_invocation_amd: {
6714       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6715       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6716       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6717       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6718       if (dst.regClass() == v1) {
6719          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6720          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6721       } else if (dst.regClass() == v2) {
6722          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6723          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6724          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6725          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6726          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6727          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6728          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6729          emit_split_vector(ctx, dst, 2);
6730       } else {
6731          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6732          nir_print_instr(&instr->instr, stderr);
6733          fprintf(stderr, "\n");
6734       }
6735       break;
6736    }
6737    case nir_intrinsic_mbcnt_amd: {
6738       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6739       RegClass rc = RegClass(src.type(), 1);
6740       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6741       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6742       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6743       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6744       emit_wqm(ctx, wqm_tmp, dst);
6745       break;
6746    }
6747    case nir_intrinsic_load_helper_invocation: {
6748       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6749       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6750       ctx->block->kind |= block_kind_needs_lowering;
6751       ctx->program->needs_exact = true;
6752       break;
6753    }
6754    case nir_intrinsic_is_helper_invocation: {
6755       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6756       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6757       ctx->block->kind |= block_kind_needs_lowering;
6758       ctx->program->needs_exact = true;
6759       break;
6760    }
6761    case nir_intrinsic_demote:
6762       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6763
6764       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6765          ctx->cf_info.exec_potentially_empty = true;
6766       ctx->block->kind |= block_kind_uses_demote;
6767       ctx->program->needs_exact = true;
6768       break;
6769    case nir_intrinsic_demote_if: {
6770       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6771       assert(src.regClass() == bld.lm);
6772       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6773       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6774
6775       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6776          ctx->cf_info.exec_potentially_empty = true;
6777       ctx->block->kind |= block_kind_uses_demote;
6778       ctx->program->needs_exact = true;
6779       break;
6780    }
6781    case nir_intrinsic_first_invocation: {
6782       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6783                get_ssa_temp(ctx, &instr->dest.ssa));
6784       break;
6785    }
6786    case nir_intrinsic_shader_clock:
6787       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6788       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6789       break;
6790    case nir_intrinsic_load_vertex_id_zero_base: {
6791       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6792       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6793       break;
6794    }
6795    case nir_intrinsic_load_first_vertex: {
6796       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6797       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6798       break;
6799    }
6800    case nir_intrinsic_load_base_instance: {
6801       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6802       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6803       break;
6804    }
6805    case nir_intrinsic_load_instance_id: {
6806       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6807       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6808       break;
6809    }
6810    case nir_intrinsic_load_draw_id: {
6811       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6812       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6813       break;
6814    }
6815    case nir_intrinsic_load_invocation_id: {
6816       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6817       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6818       if (ctx->options->chip_class >= GFX10)
6819          bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6820       else
6821          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6822       break;
6823    }
6824    case nir_intrinsic_load_primitive_id: {
6825       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6826       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6827       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
6828       break;
6829    }
6830    case nir_intrinsic_emit_vertex_with_counter: {
6831       visit_emit_vertex_with_counter(ctx, instr);
6832       break;
6833    }
6834    case nir_intrinsic_end_primitive_with_counter: {
6835       unsigned stream = nir_intrinsic_stream_id(instr);
6836       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
6837       break;
6838    }
6839    case nir_intrinsic_set_vertex_count: {
6840       /* unused, the HW keeps track of this for us */
6841       break;
6842    }
6843    default:
6844       fprintf(stderr, "Unimplemented intrinsic instr: ");
6845       nir_print_instr(&instr->instr, stderr);
6846       fprintf(stderr, "\n");
6847       abort();
6848
6849       break;
6850    }
6851 }
6852
6853
6854 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6855                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6856                     enum glsl_base_type *stype)
6857 {
6858    nir_deref_instr *texture_deref_instr = NULL;
6859    nir_deref_instr *sampler_deref_instr = NULL;
6860    int plane = -1;
6861
6862    for (unsigned i = 0; i < instr->num_srcs; i++) {
6863       switch (instr->src[i].src_type) {
6864       case nir_tex_src_texture_deref:
6865          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6866          break;
6867       case nir_tex_src_sampler_deref:
6868          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6869          break;
6870       case nir_tex_src_plane:
6871          plane = nir_src_as_int(instr->src[i].src);
6872          break;
6873       default:
6874          break;
6875       }
6876    }
6877
6878    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6879
6880    if (!sampler_deref_instr)
6881       sampler_deref_instr = texture_deref_instr;
6882
6883    if (plane >= 0) {
6884       assert(instr->op != nir_texop_txf_ms &&
6885              instr->op != nir_texop_samples_identical);
6886       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6887       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6888    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6889       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6890    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6891       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6892    } else {
6893       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6894    }
6895    if (samp_ptr) {
6896       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6897
6898       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6899          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6900          Builder bld(ctx->program, ctx->block);
6901
6902          /* to avoid unnecessary moves, we split and recombine sampler and image */
6903          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6904                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6905          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6906          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6907                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6908                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6909          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6910                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6911
6912          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6913          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6914                                img[0], img[1], img[2], img[3],
6915                                img[4], img[5], img[6], img[7]);
6916          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6917                                 samp[0], samp[1], samp[2], samp[3]);
6918       }
6919    }
6920    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6921                      instr->op == nir_texop_samples_identical))
6922       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6923 }
6924
6925 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6926                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6927 {
6928    Builder bld(ctx->program, ctx->block);
6929
6930    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6931    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6932    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6933
6934    Operand neg_one(0xbf800000u);
6935    Operand one(0x3f800000u);
6936    Operand two(0x40000000u);
6937    Operand four(0x40800000u);
6938
6939    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6940    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6941    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6942
6943    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6944    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6945    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6946    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6947
6948    // select sc
6949    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6950    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6951                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6952                        one, is_ma_y);
6953    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6954
6955    // select tc
6956    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6957    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6958    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6959
6960    // select ma
6961    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6962                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6963                   deriv_z, is_ma_z);
6964    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6965    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6966 }
6967
6968 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6969 {
6970    Builder bld(ctx->program, ctx->block);
6971    Temp coord_args[4], ma, tc, sc, id;
6972    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6973       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6974
6975    if (is_array) {
6976       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6977
6978       // see comment in ac_prepare_cube_coords()
6979       if (ctx->options->chip_class <= GFX8)
6980          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6981    }
6982
6983    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6984
6985    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6986    vop3a->operands[0] = Operand(ma);
6987    vop3a->abs[0] = true;
6988    Temp invma = bld.tmp(v1);
6989    vop3a->definitions[0] = Definition(invma);
6990    ctx->block->instructions.emplace_back(std::move(vop3a));
6991
6992    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6993    if (!is_deriv)
6994       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6995
6996    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6997    if (!is_deriv)
6998       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6999
7000    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
7001
7002    if (is_deriv) {
7003       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
7004       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
7005
7006       for (unsigned i = 0; i < 2; i++) {
7007          // see comment in ac_prepare_cube_coords()
7008          Temp deriv_ma;
7009          Temp deriv_sc, deriv_tc;
7010          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
7011                            &deriv_ma, &deriv_sc, &deriv_tc);
7012
7013          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
7014
7015          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7016                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
7017                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
7018          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7019                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
7020                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
7021          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
7022       }
7023
7024       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
7025       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
7026    }
7027
7028    if (is_array)
7029       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
7030    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
7031
7032 }
7033
7034 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
7035 {
7036    Temp coord_vec[3];
7037    for (unsigned i = 0; i < coords.size(); i++)
7038       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
7039
7040    Builder bld(ctx->program, ctx->block);
7041    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
7042
7043    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7044    for (unsigned i = 0; i < coords.size(); i++)
7045       vec->operands[i] = Operand(coord_vec[i]);
7046    Temp res = bld.tmp(RegType::vgpr, coords.size());
7047    vec->definitions[0] = Definition(res);
7048    ctx->block->instructions.emplace_back(std::move(vec));
7049    return res;
7050 }
7051
7052 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7053 {
7054    if (vec->parent_instr->type != nir_instr_type_alu)
7055       return;
7056    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7057    if (vec_instr->op != nir_op_vec(vec->num_components))
7058       return;
7059
7060    for (unsigned i = 0; i < vec->num_components; i++) {
7061       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7062               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7063    }
7064 }
7065
7066 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7067 {
7068    Builder bld(ctx->program, ctx->block);
7069    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7070         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7071    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
7072         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
7073    nir_const_value *sample_index_cv = NULL;
7074    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7075    enum glsl_base_type stype;
7076    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7077
7078    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7079                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7080    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7081                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7082
7083    for (unsigned i = 0; i < instr->num_srcs; i++) {
7084       switch (instr->src[i].src_type) {
7085       case nir_tex_src_coord:
7086          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
7087          break;
7088       case nir_tex_src_bias:
7089          if (instr->op == nir_texop_txb) {
7090             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7091             has_bias = true;
7092          }
7093          break;
7094       case nir_tex_src_lod: {
7095          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7096
7097          if (val && val->f32 <= 0.0) {
7098             level_zero = true;
7099          } else {
7100             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7101             has_lod = true;
7102          }
7103          break;
7104       }
7105       case nir_tex_src_comparator:
7106          if (instr->is_shadow) {
7107             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7108             has_compare = true;
7109          }
7110          break;
7111       case nir_tex_src_offset:
7112          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7113          get_const_vec(instr->src[i].src.ssa, const_offset);
7114          has_offset = true;
7115          break;
7116       case nir_tex_src_ddx:
7117          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7118          has_ddx = true;
7119          break;
7120       case nir_tex_src_ddy:
7121          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7122          has_ddy = true;
7123          break;
7124       case nir_tex_src_ms_index:
7125          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7126          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7127          has_sample_index = true;
7128          break;
7129       case nir_tex_src_texture_offset:
7130       case nir_tex_src_sampler_offset:
7131       default:
7132          break;
7133       }
7134    }
7135 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
7136    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7137       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7138
7139    if (instr->op == nir_texop_texture_samples) {
7140       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7141
7142       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7143       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7144       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7145       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7146
7147       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7148                samples, Operand(1u), bld.scc(is_msaa));
7149       return;
7150    }
7151
7152    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7153       aco_ptr<Instruction> tmp_instr;
7154       Temp acc, pack = Temp();
7155
7156       uint32_t pack_const = 0;
7157       for (unsigned i = 0; i < offset.size(); i++) {
7158          if (!const_offset[i])
7159             continue;
7160          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7161       }
7162
7163       if (offset.type() == RegType::sgpr) {
7164          for (unsigned i = 0; i < offset.size(); i++) {
7165             if (const_offset[i])
7166                continue;
7167
7168             acc = emit_extract_vector(ctx, offset, i, s1);
7169             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7170
7171             if (i) {
7172                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7173             }
7174
7175             if (pack == Temp()) {
7176                pack = acc;
7177             } else {
7178                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7179             }
7180          }
7181
7182          if (pack_const && pack != Temp())
7183             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7184       } else {
7185          for (unsigned i = 0; i < offset.size(); i++) {
7186             if (const_offset[i])
7187                continue;
7188
7189             acc = emit_extract_vector(ctx, offset, i, v1);
7190             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7191
7192             if (i) {
7193                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7194             }
7195
7196             if (pack == Temp()) {
7197                pack = acc;
7198             } else {
7199                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7200             }
7201          }
7202
7203          if (pack_const && pack != Temp())
7204             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7205       }
7206       if (pack_const && pack == Temp())
7207          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7208       else if (pack == Temp())
7209          has_offset = false;
7210       else
7211          offset = pack;
7212    }
7213
7214    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7215       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7216
7217    /* pack derivatives */
7218    if (has_ddx || has_ddy) {
7219       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7220          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
7221                              ddx, Operand(0u), ddy, Operand(0u));
7222       } else {
7223          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
7224       }
7225       has_derivs = true;
7226    }
7227
7228    if (instr->coord_components > 1 &&
7229        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7230        instr->is_array &&
7231        instr->op != nir_texop_txf)
7232       coords = apply_round_slice(ctx, coords, 1);
7233
7234    if (instr->coord_components > 2 &&
7235       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7236        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7237        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7238        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7239        instr->is_array &&
7240        instr->op != nir_texop_txf &&
7241        instr->op != nir_texop_txf_ms &&
7242        instr->op != nir_texop_fragment_fetch &&
7243        instr->op != nir_texop_fragment_mask_fetch)
7244       coords = apply_round_slice(ctx, coords, 2);
7245
7246    if (ctx->options->chip_class == GFX9 &&
7247        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7248        instr->op != nir_texop_lod && instr->coord_components) {
7249       assert(coords.size() > 0 && coords.size() < 3);
7250
7251       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
7252       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
7253       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
7254       if (coords.size() > 1)
7255          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
7256       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
7257       vec->definitions[0] = Definition(coords);
7258       ctx->block->instructions.emplace_back(std::move(vec));
7259    }
7260
7261    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7262
7263    if (instr->op == nir_texop_samples_identical)
7264       resource = fmask_ptr;
7265
7266    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7267              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7268             instr->op != nir_texop_txs &&
7269             instr->op != nir_texop_fragment_fetch &&
7270             instr->op != nir_texop_fragment_mask_fetch) {
7271       assert(has_sample_index);
7272       Operand op(sample_index);
7273       if (sample_index_cv)
7274          op = Operand(sample_index_cv->u32);
7275       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7276    }
7277
7278    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7279       Temp split_coords[coords.size()];
7280       emit_split_vector(ctx, coords, coords.size());
7281       for (unsigned i = 0; i < coords.size(); i++)
7282          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
7283
7284       unsigned i = 0;
7285       for (; i < std::min(offset.size(), instr->coord_components); i++) {
7286          Temp off = emit_extract_vector(ctx, offset, i, v1);
7287          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
7288       }
7289
7290       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7291       for (unsigned i = 0; i < coords.size(); i++)
7292          vec->operands[i] = Operand(split_coords[i]);
7293       coords = bld.tmp(coords.regClass());
7294       vec->definitions[0] = Definition(coords);
7295       ctx->block->instructions.emplace_back(std::move(vec));
7296
7297       has_offset = false;
7298    }
7299
7300    /* Build tex instruction */
7301    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7302    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7303                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7304                   : 0;
7305    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7306    Temp tmp_dst = dst;
7307
7308    /* gather4 selects the component by dmask and always returns vec4 */
7309    if (instr->op == nir_texop_tg4) {
7310       assert(instr->dest.ssa.num_components == 4);
7311       if (instr->is_shadow)
7312          dmask = 1;
7313       else
7314          dmask = 1 << instr->component;
7315       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7316          tmp_dst = bld.tmp(v4);
7317    } else if (instr->op == nir_texop_samples_identical) {
7318       tmp_dst = bld.tmp(v1);
7319    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7320       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7321    }
7322
7323    aco_ptr<MIMG_instruction> tex;
7324    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7325       if (!has_lod)
7326          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7327
7328       bool div_by_6 = instr->op == nir_texop_txs &&
7329                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7330                       instr->is_array &&
7331                       (dmask & (1 << 2));
7332       if (tmp_dst.id() == dst.id() && div_by_6)
7333          tmp_dst = bld.tmp(tmp_dst.regClass());
7334
7335       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7336       tex->operands[0] = Operand(as_vgpr(ctx,lod));
7337       tex->operands[1] = Operand(resource);
7338       if (ctx->options->chip_class == GFX9 &&
7339           instr->op == nir_texop_txs &&
7340           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7341           instr->is_array) {
7342          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7343       } else if (instr->op == nir_texop_query_levels) {
7344          tex->dmask = 1 << 3;
7345       } else {
7346          tex->dmask = dmask;
7347       }
7348       tex->da = da;
7349       tex->definitions[0] = Definition(tmp_dst);
7350       tex->dim = dim;
7351       tex->can_reorder = true;
7352       ctx->block->instructions.emplace_back(std::move(tex));
7353
7354       if (div_by_6) {
7355          /* divide 3rd value by 6 by multiplying with magic number */
7356          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7357          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7358          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7359          assert(instr->dest.ssa.num_components == 3);
7360          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7361          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7362                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7363                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7364                               by_6);
7365
7366       }
7367
7368       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7369       return;
7370    }
7371
7372    Temp tg4_compare_cube_wa64 = Temp();
7373
7374    if (tg4_integer_workarounds) {
7375       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7376       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7377       tex->operands[1] = Operand(resource);
7378       tex->dim = dim;
7379       tex->dmask = 0x3;
7380       tex->da = da;
7381       Temp size = bld.tmp(v2);
7382       tex->definitions[0] = Definition(size);
7383       tex->can_reorder = true;
7384       ctx->block->instructions.emplace_back(std::move(tex));
7385       emit_split_vector(ctx, size, size.size());
7386
7387       Temp half_texel[2];
7388       for (unsigned i = 0; i < 2; i++) {
7389          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7390          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7391          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7392          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7393       }
7394
7395       Temp orig_coords[2] = {
7396          emit_extract_vector(ctx, coords, 0, v1),
7397          emit_extract_vector(ctx, coords, 1, v1)};
7398       Temp new_coords[2] = {
7399          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
7400          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
7401       };
7402
7403       if (tg4_integer_cube_workaround) {
7404          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7405          Temp desc[resource.size()];
7406          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7407                                                                            Format::PSEUDO, 1, resource.size())};
7408          split->operands[0] = Operand(resource);
7409          for (unsigned i = 0; i < resource.size(); i++) {
7410             desc[i] = bld.tmp(s1);
7411             split->definitions[i] = Definition(desc[i]);
7412          }
7413          ctx->block->instructions.emplace_back(std::move(split));
7414
7415          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7416          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7417                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7418
7419          Temp nfmt;
7420          if (stype == GLSL_TYPE_UINT) {
7421             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7422                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7423                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7424                             bld.scc(compare_cube_wa));
7425          } else {
7426             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7427                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7428                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7429                             bld.scc(compare_cube_wa));
7430          }
7431          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7432          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7433
7434          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7435
7436          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7437                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7438          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7439
7440          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7441                                                                          Format::PSEUDO, resource.size(), 1)};
7442          for (unsigned i = 0; i < resource.size(); i++)
7443             vec->operands[i] = Operand(desc[i]);
7444          resource = bld.tmp(resource.regClass());
7445          vec->definitions[0] = Definition(resource);
7446          ctx->block->instructions.emplace_back(std::move(vec));
7447
7448          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7449                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
7450          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7451                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
7452       }
7453
7454       if (coords.size() == 3) {
7455          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
7456                              new_coords[0], new_coords[1],
7457                              emit_extract_vector(ctx, coords, 2, v1));
7458       } else {
7459          assert(coords.size() == 2);
7460          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
7461                              new_coords[0], new_coords[1]);
7462       }
7463    }
7464
7465    std::vector<Operand> args;
7466    if (has_offset)
7467       args.emplace_back(Operand(offset));
7468    if (has_bias)
7469       args.emplace_back(Operand(bias));
7470    if (has_compare)
7471       args.emplace_back(Operand(compare));
7472    if (has_derivs)
7473       args.emplace_back(Operand(derivs));
7474    args.emplace_back(Operand(coords));
7475    if (has_sample_index)
7476       args.emplace_back(Operand(sample_index));
7477    if (has_lod)
7478       args.emplace_back(lod);
7479
7480    Temp arg;
7481    if (args.size() > 1) {
7482       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7483       unsigned size = 0;
7484       for (unsigned i = 0; i < args.size(); i++) {
7485          size += args[i].size();
7486          vec->operands[i] = args[i];
7487       }
7488       RegClass rc = RegClass(RegType::vgpr, size);
7489       Temp tmp = bld.tmp(rc);
7490       vec->definitions[0] = Definition(tmp);
7491       ctx->block->instructions.emplace_back(std::move(vec));
7492       arg = tmp;
7493    } else {
7494       assert(args[0].isTemp());
7495       arg = as_vgpr(ctx, args[0].getTemp());
7496    }
7497
7498    /* we don't need the bias, sample index, compare value or offset to be
7499     * computed in WQM but if the p_create_vector copies the coordinates, then it
7500     * needs to be in WQM */
7501    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
7502        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7503        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7504       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7505
7506    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7507       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7508
7509       assert(coords.size() == 1);
7510       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7511       aco_opcode op;
7512       switch (last_bit) {
7513       case 1:
7514          op = aco_opcode::buffer_load_format_x; break;
7515       case 2:
7516          op = aco_opcode::buffer_load_format_xy; break;
7517       case 3:
7518          op = aco_opcode::buffer_load_format_xyz; break;
7519       case 4:
7520          op = aco_opcode::buffer_load_format_xyzw; break;
7521       default:
7522          unreachable("Tex instruction loads more than 4 components.");
7523       }
7524
7525       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7526       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7527          tmp_dst = dst;
7528       else
7529          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7530
7531       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7532       mubuf->operands[0] = Operand(coords);
7533       mubuf->operands[1] = Operand(resource);
7534       mubuf->operands[2] = Operand((uint32_t) 0);
7535       mubuf->definitions[0] = Definition(tmp_dst);
7536       mubuf->idxen = true;
7537       mubuf->can_reorder = true;
7538       ctx->block->instructions.emplace_back(std::move(mubuf));
7539
7540       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7541       return;
7542    }
7543
7544
7545    if (instr->op == nir_texop_txf ||
7546        instr->op == nir_texop_txf_ms ||
7547        instr->op == nir_texop_samples_identical ||
7548        instr->op == nir_texop_fragment_fetch ||
7549        instr->op == nir_texop_fragment_mask_fetch) {
7550       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7551       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
7552       tex->operands[0] = Operand(arg);
7553       tex->operands[1] = Operand(resource);
7554       tex->dim = dim;
7555       tex->dmask = dmask;
7556       tex->unrm = true;
7557       tex->da = da;
7558       tex->definitions[0] = Definition(tmp_dst);
7559       tex->can_reorder = true;
7560       ctx->block->instructions.emplace_back(std::move(tex));
7561
7562       if (instr->op == nir_texop_samples_identical) {
7563          assert(dmask == 1 && dst.regClass() == v1);
7564          assert(dst.id() != tmp_dst.id());
7565
7566          Temp tmp = bld.tmp(bld.lm);
7567          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7568          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7569
7570       } else {
7571          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7572       }
7573       return;
7574    }
7575
7576    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7577    aco_opcode opcode = aco_opcode::image_sample;
7578    if (has_offset) { /* image_sample_*_o */
7579       if (has_compare) {
7580          opcode = aco_opcode::image_sample_c_o;
7581          if (has_derivs)
7582             opcode = aco_opcode::image_sample_c_d_o;
7583          if (has_bias)
7584             opcode = aco_opcode::image_sample_c_b_o;
7585          if (level_zero)
7586             opcode = aco_opcode::image_sample_c_lz_o;
7587          if (has_lod)
7588             opcode = aco_opcode::image_sample_c_l_o;
7589       } else {
7590          opcode = aco_opcode::image_sample_o;
7591          if (has_derivs)
7592             opcode = aco_opcode::image_sample_d_o;
7593          if (has_bias)
7594             opcode = aco_opcode::image_sample_b_o;
7595          if (level_zero)
7596             opcode = aco_opcode::image_sample_lz_o;
7597          if (has_lod)
7598             opcode = aco_opcode::image_sample_l_o;
7599       }
7600    } else { /* no offset */
7601       if (has_compare) {
7602          opcode = aco_opcode::image_sample_c;
7603          if (has_derivs)
7604             opcode = aco_opcode::image_sample_c_d;
7605          if (has_bias)
7606             opcode = aco_opcode::image_sample_c_b;
7607          if (level_zero)
7608             opcode = aco_opcode::image_sample_c_lz;
7609          if (has_lod)
7610             opcode = aco_opcode::image_sample_c_l;
7611       } else {
7612          opcode = aco_opcode::image_sample;
7613          if (has_derivs)
7614             opcode = aco_opcode::image_sample_d;
7615          if (has_bias)
7616             opcode = aco_opcode::image_sample_b;
7617          if (level_zero)
7618             opcode = aco_opcode::image_sample_lz;
7619          if (has_lod)
7620             opcode = aco_opcode::image_sample_l;
7621       }
7622    }
7623
7624    if (instr->op == nir_texop_tg4) {
7625       if (has_offset) {
7626          opcode = aco_opcode::image_gather4_lz_o;
7627          if (has_compare)
7628             opcode = aco_opcode::image_gather4_c_lz_o;
7629       } else {
7630          opcode = aco_opcode::image_gather4_lz;
7631          if (has_compare)
7632             opcode = aco_opcode::image_gather4_c_lz;
7633       }
7634    } else if (instr->op == nir_texop_lod) {
7635       opcode = aco_opcode::image_get_lod;
7636    }
7637
7638    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7639    tex->operands[0] = Operand(arg);
7640    tex->operands[1] = Operand(resource);
7641    tex->operands[2] = Operand(sampler);
7642    tex->dim = dim;
7643    tex->dmask = dmask;
7644    tex->da = da;
7645    tex->definitions[0] = Definition(tmp_dst);
7646    tex->can_reorder = true;
7647    ctx->block->instructions.emplace_back(std::move(tex));
7648
7649    if (tg4_integer_cube_workaround) {
7650       assert(tmp_dst.id() != dst.id());
7651       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7652
7653       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7654       Temp val[4];
7655       for (unsigned i = 0; i < dst.size(); i++) {
7656          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7657          Temp cvt_val;
7658          if (stype == GLSL_TYPE_UINT)
7659             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7660          else
7661             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7662          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7663       }
7664       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7665       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7666                            val[0], val[1], val[2], val[3]);
7667    }
7668    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7669    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7670
7671 }
7672
7673
7674 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7675 {
7676    Temp tmp = get_ssa_temp(ctx, ssa);
7677    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7678       return Operand(tmp.regClass());
7679    else
7680       return Operand(tmp);
7681 }
7682
7683 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7684 {
7685    aco_ptr<Pseudo_instruction> phi;
7686    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7687    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7688
7689    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7690    logical |= ctx->block->kind & block_kind_merge;
7691    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7692
7693    /* we want a sorted list of sources, since the predecessor list is also sorted */
7694    std::map<unsigned, nir_ssa_def*> phi_src;
7695    nir_foreach_phi_src(src, instr)
7696       phi_src[src->pred->index] = src->src.ssa;
7697
7698    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7699    unsigned num_operands = 0;
7700    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7701    unsigned num_defined = 0;
7702    unsigned cur_pred_idx = 0;
7703    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7704       if (cur_pred_idx < preds.size()) {
7705          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7706          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7707          unsigned skipped = 0;
7708          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7709             skipped++;
7710          if (cur_pred_idx + skipped < preds.size()) {
7711             for (unsigned i = 0; i < skipped; i++)
7712                operands[num_operands++] = Operand(dst.regClass());
7713             cur_pred_idx += skipped;
7714          } else {
7715             continue;
7716          }
7717       }
7718       cur_pred_idx++;
7719       Operand op = get_phi_operand(ctx, src.second);
7720       operands[num_operands++] = op;
7721       num_defined += !op.isUndefined();
7722    }
7723    /* handle block_kind_continue_or_break at loop exit blocks */
7724    while (cur_pred_idx++ < preds.size())
7725       operands[num_operands++] = Operand(dst.regClass());
7726
7727    if (num_defined == 0) {
7728       Builder bld(ctx->program, ctx->block);
7729       if (dst.regClass() == s1) {
7730          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7731       } else if (dst.regClass() == v1) {
7732          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7733       } else {
7734          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7735          for (unsigned i = 0; i < dst.size(); i++)
7736             vec->operands[i] = Operand(0u);
7737          vec->definitions[0] = Definition(dst);
7738          ctx->block->instructions.emplace_back(std::move(vec));
7739       }
7740       return;
7741    }
7742
7743    /* we can use a linear phi in some cases if one src is undef */
7744    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7745       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7746
7747       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7748       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7749       assert(invert->kind & block_kind_invert);
7750
7751       unsigned then_block = invert->linear_preds[0];
7752
7753       Block* insert_block = NULL;
7754       for (unsigned i = 0; i < num_operands; i++) {
7755          Operand op = operands[i];
7756          if (op.isUndefined())
7757             continue;
7758          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7759          phi->operands[0] = op;
7760          break;
7761       }
7762       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7763       phi->operands[1] = Operand(dst.regClass());
7764       phi->definitions[0] = Definition(dst);
7765       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7766       return;
7767    }
7768
7769    /* try to scalarize vector phis */
7770    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7771       // TODO: scalarize linear phis on divergent ifs
7772       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7773       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7774       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7775          Operand src = operands[i];
7776          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7777             can_scalarize = false;
7778       }
7779       if (can_scalarize) {
7780          unsigned num_components = instr->dest.ssa.num_components;
7781          assert(dst.size() % num_components == 0);
7782          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7783
7784          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7785          for (unsigned k = 0; k < num_components; k++) {
7786             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7787             for (unsigned i = 0; i < num_operands; i++) {
7788                Operand src = operands[i];
7789                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7790             }
7791             Temp phi_dst = {ctx->program->allocateId(), rc};
7792             phi->definitions[0] = Definition(phi_dst);
7793             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7794             new_vec[k] = phi_dst;
7795             vec->operands[k] = Operand(phi_dst);
7796          }
7797          vec->definitions[0] = Definition(dst);
7798          ctx->block->instructions.emplace_back(std::move(vec));
7799          ctx->allocated_vec.emplace(dst.id(), new_vec);
7800          return;
7801       }
7802    }
7803
7804    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7805    for (unsigned i = 0; i < num_operands; i++)
7806       phi->operands[i] = operands[i];
7807    phi->definitions[0] = Definition(dst);
7808    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7809 }
7810
7811
7812 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7813 {
7814    Temp dst = get_ssa_temp(ctx, &instr->def);
7815
7816    assert(dst.type() == RegType::sgpr);
7817
7818    if (dst.size() == 1) {
7819       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7820    } else {
7821       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7822       for (unsigned i = 0; i < dst.size(); i++)
7823          vec->operands[i] = Operand(0u);
7824       vec->definitions[0] = Definition(dst);
7825       ctx->block->instructions.emplace_back(std::move(vec));
7826    }
7827 }
7828
7829 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7830 {
7831    Builder bld(ctx->program, ctx->block);
7832    Block *logical_target;
7833    append_logical_end(ctx->block);
7834    unsigned idx = ctx->block->index;
7835
7836    switch (instr->type) {
7837    case nir_jump_break:
7838       logical_target = ctx->cf_info.parent_loop.exit;
7839       add_logical_edge(idx, logical_target);
7840       ctx->block->kind |= block_kind_break;
7841
7842       if (!ctx->cf_info.parent_if.is_divergent &&
7843           !ctx->cf_info.parent_loop.has_divergent_continue) {
7844          /* uniform break - directly jump out of the loop */
7845          ctx->block->kind |= block_kind_uniform;
7846          ctx->cf_info.has_branch = true;
7847          bld.branch(aco_opcode::p_branch);
7848          add_linear_edge(idx, logical_target);
7849          return;
7850       }
7851       ctx->cf_info.parent_loop.has_divergent_branch = true;
7852       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7853       break;
7854    case nir_jump_continue:
7855       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7856       add_logical_edge(idx, logical_target);
7857       ctx->block->kind |= block_kind_continue;
7858
7859       if (ctx->cf_info.parent_if.is_divergent) {
7860          /* for potential uniform breaks after this continue,
7861             we must ensure that they are handled correctly */
7862          ctx->cf_info.parent_loop.has_divergent_continue = true;
7863          ctx->cf_info.parent_loop.has_divergent_branch = true;
7864          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7865       } else {
7866          /* uniform continue - directly jump to the loop header */
7867          ctx->block->kind |= block_kind_uniform;
7868          ctx->cf_info.has_branch = true;
7869          bld.branch(aco_opcode::p_branch);
7870          add_linear_edge(idx, logical_target);
7871          return;
7872       }
7873       break;
7874    default:
7875       fprintf(stderr, "Unknown NIR jump instr: ");
7876       nir_print_instr(&instr->instr, stderr);
7877       fprintf(stderr, "\n");
7878       abort();
7879    }
7880
7881    /* remove critical edges from linear CFG */
7882    bld.branch(aco_opcode::p_branch);
7883    Block* break_block = ctx->program->create_and_insert_block();
7884    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7885    break_block->kind |= block_kind_uniform;
7886    add_linear_edge(idx, break_block);
7887    /* the loop_header pointer might be invalidated by this point */
7888    if (instr->type == nir_jump_continue)
7889       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7890    add_linear_edge(break_block->index, logical_target);
7891    bld.reset(break_block);
7892    bld.branch(aco_opcode::p_branch);
7893
7894    Block* continue_block = ctx->program->create_and_insert_block();
7895    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7896    add_linear_edge(idx, continue_block);
7897    append_logical_start(continue_block);
7898    ctx->block = continue_block;
7899    return;
7900 }
7901
7902 void visit_block(isel_context *ctx, nir_block *block)
7903 {
7904    nir_foreach_instr(instr, block) {
7905       switch (instr->type) {
7906       case nir_instr_type_alu:
7907          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7908          break;
7909       case nir_instr_type_load_const:
7910          visit_load_const(ctx, nir_instr_as_load_const(instr));
7911          break;
7912       case nir_instr_type_intrinsic:
7913          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7914          break;
7915       case nir_instr_type_tex:
7916          visit_tex(ctx, nir_instr_as_tex(instr));
7917          break;
7918       case nir_instr_type_phi:
7919          visit_phi(ctx, nir_instr_as_phi(instr));
7920          break;
7921       case nir_instr_type_ssa_undef:
7922          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7923          break;
7924       case nir_instr_type_deref:
7925          break;
7926       case nir_instr_type_jump:
7927          visit_jump(ctx, nir_instr_as_jump(instr));
7928          break;
7929       default:
7930          fprintf(stderr, "Unknown NIR instr type: ");
7931          nir_print_instr(instr, stderr);
7932          fprintf(stderr, "\n");
7933          //abort();
7934       }
7935    }
7936
7937    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7938       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7939 }
7940
7941
7942
7943 static void visit_loop(isel_context *ctx, nir_loop *loop)
7944 {
7945    append_logical_end(ctx->block);
7946    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7947    Builder bld(ctx->program, ctx->block);
7948    bld.branch(aco_opcode::p_branch);
7949    unsigned loop_preheader_idx = ctx->block->index;
7950
7951    Block loop_exit = Block();
7952    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7953    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7954
7955    Block* loop_header = ctx->program->create_and_insert_block();
7956    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7957    loop_header->kind |= block_kind_loop_header;
7958    add_edge(loop_preheader_idx, loop_header);
7959    ctx->block = loop_header;
7960
7961    /* emit loop body */
7962    unsigned loop_header_idx = loop_header->index;
7963    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7964    append_logical_start(ctx->block);
7965    visit_cf_list(ctx, &loop->body);
7966
7967    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7968    if (!ctx->cf_info.has_branch) {
7969       append_logical_end(ctx->block);
7970       if (ctx->cf_info.exec_potentially_empty) {
7971          /* Discards can result in code running with an empty exec mask.
7972           * This would result in divergent breaks not ever being taken. As a
7973           * workaround, break the loop when the loop mask is empty instead of
7974           * always continuing. */
7975          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7976          unsigned block_idx = ctx->block->index;
7977
7978          /* create helper blocks to avoid critical edges */
7979          Block *break_block = ctx->program->create_and_insert_block();
7980          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7981          break_block->kind = block_kind_uniform;
7982          bld.reset(break_block);
7983          bld.branch(aco_opcode::p_branch);
7984          add_linear_edge(block_idx, break_block);
7985          add_linear_edge(break_block->index, &loop_exit);
7986
7987          Block *continue_block = ctx->program->create_and_insert_block();
7988          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7989          continue_block->kind = block_kind_uniform;
7990          bld.reset(continue_block);
7991          bld.branch(aco_opcode::p_branch);
7992          add_linear_edge(block_idx, continue_block);
7993          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7994
7995          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7996          ctx->block = &ctx->program->blocks[block_idx];
7997       } else {
7998          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7999          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8000             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8001          else
8002             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8003       }
8004
8005       bld.reset(ctx->block);
8006       bld.branch(aco_opcode::p_branch);
8007    }
8008
8009    /* fixup phis in loop header from unreachable blocks */
8010    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
8011       bool linear = ctx->cf_info.has_branch;
8012       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
8013       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
8014          if ((logical && instr->opcode == aco_opcode::p_phi) ||
8015              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
8016             /* the last operand should be the one that needs to be removed */
8017             instr->operands.pop_back();
8018          } else if (!is_phi(instr)) {
8019             break;
8020          }
8021       }
8022    }
8023
8024    ctx->cf_info.has_branch = false;
8025
8026    // TODO: if the loop has not a single exit, we must add one °°
8027    /* emit loop successor block */
8028    ctx->block = ctx->program->insert_block(std::move(loop_exit));
8029    append_logical_start(ctx->block);
8030
8031    #if 0
8032    // TODO: check if it is beneficial to not branch on continues
8033    /* trim linear phis in loop header */
8034    for (auto&& instr : loop_entry->instructions) {
8035       if (instr->opcode == aco_opcode::p_linear_phi) {
8036          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8037          new_phi->definitions[0] = instr->definitions[0];
8038          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8039             new_phi->operands[i] = instr->operands[i];
8040          /* check that the remaining operands are all the same */
8041          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8042             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8043          instr.swap(new_phi);
8044       } else if (instr->opcode == aco_opcode::p_phi) {
8045          continue;
8046       } else {
8047          break;
8048       }
8049    }
8050    #endif
8051 }
8052
8053 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8054 {
8055    ic->cond = cond;
8056
8057    append_logical_end(ctx->block);
8058    ctx->block->kind |= block_kind_branch;
8059
8060    /* branch to linear then block */
8061    assert(cond.regClass() == ctx->program->lane_mask);
8062    aco_ptr<Pseudo_branch_instruction> branch;
8063    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8064    branch->operands[0] = Operand(cond);
8065    ctx->block->instructions.push_back(std::move(branch));
8066
8067    ic->BB_if_idx = ctx->block->index;
8068    ic->BB_invert = Block();
8069    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8070    /* Invert blocks are intentionally not marked as top level because they
8071     * are not part of the logical cfg. */
8072    ic->BB_invert.kind |= block_kind_invert;
8073    ic->BB_endif = Block();
8074    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8075    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8076
8077    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
8078    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8079    ctx->cf_info.parent_if.is_divergent = true;
8080    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8081
8082    /** emit logical then block */
8083    Block* BB_then_logical = ctx->program->create_and_insert_block();
8084    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8085    add_edge(ic->BB_if_idx, BB_then_logical);
8086    ctx->block = BB_then_logical;
8087    append_logical_start(BB_then_logical);
8088 }
8089
8090 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8091 {
8092    Block *BB_then_logical = ctx->block;
8093    append_logical_end(BB_then_logical);
8094     /* branch from logical then block to invert block */
8095    aco_ptr<Pseudo_branch_instruction> branch;
8096    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8097    BB_then_logical->instructions.emplace_back(std::move(branch));
8098    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8099    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8100       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8101    BB_then_logical->kind |= block_kind_uniform;
8102    assert(!ctx->cf_info.has_branch);
8103    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8104    ctx->cf_info.parent_loop.has_divergent_branch = false;
8105
8106    /** emit linear then block */
8107    Block* BB_then_linear = ctx->program->create_and_insert_block();
8108    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8109    BB_then_linear->kind |= block_kind_uniform;
8110    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8111    /* branch from linear then block to invert block */
8112    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8113    BB_then_linear->instructions.emplace_back(std::move(branch));
8114    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8115
8116    /** emit invert merge block */
8117    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8118    ic->invert_idx = ctx->block->index;
8119
8120    /* branch to linear else block (skip else) */
8121    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8122    branch->operands[0] = Operand(ic->cond);
8123    ctx->block->instructions.push_back(std::move(branch));
8124
8125    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
8126    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8127
8128    /** emit logical else block */
8129    Block* BB_else_logical = ctx->program->create_and_insert_block();
8130    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8131    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8132    add_linear_edge(ic->invert_idx, BB_else_logical);
8133    ctx->block = BB_else_logical;
8134    append_logical_start(BB_else_logical);
8135 }
8136
8137 static void end_divergent_if(isel_context *ctx, if_context *ic)
8138 {
8139    Block *BB_else_logical = ctx->block;
8140    append_logical_end(BB_else_logical);
8141
8142    /* branch from logical else block to endif block */
8143    aco_ptr<Pseudo_branch_instruction> branch;
8144    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8145    BB_else_logical->instructions.emplace_back(std::move(branch));
8146    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8147    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8148       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8149    BB_else_logical->kind |= block_kind_uniform;
8150
8151    assert(!ctx->cf_info.has_branch);
8152    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8153
8154
8155    /** emit linear else block */
8156    Block* BB_else_linear = ctx->program->create_and_insert_block();
8157    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8158    BB_else_linear->kind |= block_kind_uniform;
8159    add_linear_edge(ic->invert_idx, BB_else_linear);
8160
8161    /* branch from linear else block to endif block */
8162    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8163    BB_else_linear->instructions.emplace_back(std::move(branch));
8164    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8165
8166
8167    /** emit endif merge block */
8168    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8169    append_logical_start(ctx->block);
8170
8171
8172    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8173    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
8174    /* uniform control flow never has an empty exec-mask */
8175    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
8176       ctx->cf_info.exec_potentially_empty = false;
8177 }
8178
8179 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8180 {
8181    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8182    Builder bld(ctx->program, ctx->block);
8183    aco_ptr<Pseudo_branch_instruction> branch;
8184
8185    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8186       /**
8187        * Uniform conditionals are represented in the following way*) :
8188        *
8189        * The linear and logical CFG:
8190        *                        BB_IF
8191        *                        /    \
8192        *       BB_THEN (logical)      BB_ELSE (logical)
8193        *                        \    /
8194        *                        BB_ENDIF
8195        *
8196        * *) Exceptions may be due to break and continue statements within loops
8197        *    If a break/continue happens within uniform control flow, it branches
8198        *    to the loop exit/entry block. Otherwise, it branches to the next
8199        *    merge block.
8200        **/
8201       append_logical_end(ctx->block);
8202       ctx->block->kind |= block_kind_uniform;
8203
8204       /* emit branch */
8205       assert(cond.regClass() == bld.lm);
8206       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8207       cond = bool_to_scalar_condition(ctx, cond);
8208
8209       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8210       branch->operands[0] = Operand(cond);
8211       branch->operands[0].setFixed(scc);
8212       ctx->block->instructions.emplace_back(std::move(branch));
8213
8214       unsigned BB_if_idx = ctx->block->index;
8215       Block BB_endif = Block();
8216       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8217       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8218
8219       /** emit then block */
8220       Block* BB_then = ctx->program->create_and_insert_block();
8221       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8222       add_edge(BB_if_idx, BB_then);
8223       append_logical_start(BB_then);
8224       ctx->block = BB_then;
8225       visit_cf_list(ctx, &if_stmt->then_list);
8226       BB_then = ctx->block;
8227       bool then_branch = ctx->cf_info.has_branch;
8228       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8229
8230       if (!then_branch) {
8231          append_logical_end(BB_then);
8232          /* branch from then block to endif block */
8233          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8234          BB_then->instructions.emplace_back(std::move(branch));
8235          add_linear_edge(BB_then->index, &BB_endif);
8236          if (!then_branch_divergent)
8237             add_logical_edge(BB_then->index, &BB_endif);
8238          BB_then->kind |= block_kind_uniform;
8239       }
8240
8241       ctx->cf_info.has_branch = false;
8242       ctx->cf_info.parent_loop.has_divergent_branch = false;
8243
8244       /** emit else block */
8245       Block* BB_else = ctx->program->create_and_insert_block();
8246       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8247       add_edge(BB_if_idx, BB_else);
8248       append_logical_start(BB_else);
8249       ctx->block = BB_else;
8250       visit_cf_list(ctx, &if_stmt->else_list);
8251       BB_else = ctx->block;
8252
8253       if (!ctx->cf_info.has_branch) {
8254          append_logical_end(BB_else);
8255          /* branch from then block to endif block */
8256          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8257          BB_else->instructions.emplace_back(std::move(branch));
8258          add_linear_edge(BB_else->index, &BB_endif);
8259          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8260             add_logical_edge(BB_else->index, &BB_endif);
8261          BB_else->kind |= block_kind_uniform;
8262       }
8263
8264       ctx->cf_info.has_branch &= then_branch;
8265       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8266
8267       /** emit endif merge block */
8268       if (!ctx->cf_info.has_branch) {
8269          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8270          append_logical_start(ctx->block);
8271       }
8272    } else { /* non-uniform condition */
8273       /**
8274        * To maintain a logical and linear CFG without critical edges,
8275        * non-uniform conditionals are represented in the following way*) :
8276        *
8277        * The linear CFG:
8278        *                        BB_IF
8279        *                        /    \
8280        *       BB_THEN (logical)      BB_THEN (linear)
8281        *                        \    /
8282        *                        BB_INVERT (linear)
8283        *                        /    \
8284        *       BB_ELSE (logical)      BB_ELSE (linear)
8285        *                        \    /
8286        *                        BB_ENDIF
8287        *
8288        * The logical CFG:
8289        *                        BB_IF
8290        *                        /    \
8291        *       BB_THEN (logical)      BB_ELSE (logical)
8292        *                        \    /
8293        *                        BB_ENDIF
8294        *
8295        * *) Exceptions may be due to break and continue statements within loops
8296        **/
8297
8298       if_context ic;
8299
8300       begin_divergent_if_then(ctx, &ic, cond);
8301       visit_cf_list(ctx, &if_stmt->then_list);
8302
8303       begin_divergent_if_else(ctx, &ic);
8304       visit_cf_list(ctx, &if_stmt->else_list);
8305
8306       end_divergent_if(ctx, &ic);
8307    }
8308 }
8309
8310 static void visit_cf_list(isel_context *ctx,
8311                           struct exec_list *list)
8312 {
8313    foreach_list_typed(nir_cf_node, node, node, list) {
8314       switch (node->type) {
8315       case nir_cf_node_block:
8316          visit_block(ctx, nir_cf_node_as_block(node));
8317          break;
8318       case nir_cf_node_if:
8319          visit_if(ctx, nir_cf_node_as_if(node));
8320          break;
8321       case nir_cf_node_loop:
8322          visit_loop(ctx, nir_cf_node_as_loop(node));
8323          break;
8324       default:
8325          unreachable("unimplemented cf list type");
8326       }
8327    }
8328 }
8329
8330 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8331 {
8332    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8333    uint64_t mask = ctx->outputs.mask[slot];
8334    if (!is_pos && !mask)
8335       return;
8336    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8337       return;
8338    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8339    exp->enabled_mask = mask;
8340    for (unsigned i = 0; i < 4; ++i) {
8341       if (mask & (1 << i))
8342          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8343       else
8344          exp->operands[i] = Operand(v1);
8345    }
8346    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8347     * Setting valid_mask=1 prevents it and has no other effect.
8348     */
8349    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8350    exp->done = false;
8351    exp->compressed = false;
8352    if (is_pos)
8353       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8354    else
8355       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8356    ctx->block->instructions.emplace_back(std::move(exp));
8357 }
8358
8359 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8360 {
8361    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8362    exp->enabled_mask = 0;
8363    for (unsigned i = 0; i < 4; ++i)
8364       exp->operands[i] = Operand(v1);
8365    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8366       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8367       exp->enabled_mask |= 0x1;
8368    }
8369    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8370       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8371       exp->enabled_mask |= 0x4;
8372    }
8373    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8374       if (ctx->options->chip_class < GFX9) {
8375          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8376          exp->enabled_mask |= 0x8;
8377       } else {
8378          Builder bld(ctx->program, ctx->block);
8379
8380          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8381                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8382          if (exp->operands[2].isTemp())
8383             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8384
8385          exp->operands[2] = Operand(out);
8386          exp->enabled_mask |= 0x4;
8387       }
8388    }
8389    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8390    exp->done = false;
8391    exp->compressed = false;
8392    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8393    ctx->block->instructions.emplace_back(std::move(exp));
8394 }
8395
8396 static void create_vs_exports(isel_context *ctx)
8397 {
8398    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8399
8400    if (outinfo->export_prim_id) {
8401       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8402       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8403    }
8404
8405    if (ctx->options->key.has_multiview_view_index) {
8406       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8407       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8408    }
8409
8410    /* the order these position exports are created is important */
8411    int next_pos = 0;
8412    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8413    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8414       export_vs_psiz_layer_viewport(ctx, &next_pos);
8415    }
8416    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8417       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8418    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8419       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8420
8421    if (ctx->export_clip_dists) {
8422       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8423          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8424       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8425          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8426    }
8427
8428    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8429       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8430           i != VARYING_SLOT_PRIMITIVE_ID)
8431          continue;
8432
8433       export_vs_varying(ctx, i, false, NULL);
8434    }
8435 }
8436
8437 static void export_fs_mrt_z(isel_context *ctx)
8438 {
8439    Builder bld(ctx->program, ctx->block);
8440    unsigned enabled_channels = 0;
8441    bool compr = false;
8442    Operand values[4];
8443
8444    for (unsigned i = 0; i < 4; ++i) {
8445       values[i] = Operand(v1);
8446    }
8447
8448    /* Both stencil and sample mask only need 16-bits. */
8449    if (!ctx->program->info->ps.writes_z &&
8450        (ctx->program->info->ps.writes_stencil ||
8451         ctx->program->info->ps.writes_sample_mask)) {
8452       compr = true; /* COMPR flag */
8453
8454       if (ctx->program->info->ps.writes_stencil) {
8455          /* Stencil should be in X[23:16]. */
8456          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8457          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8458          enabled_channels |= 0x3;
8459       }
8460
8461       if (ctx->program->info->ps.writes_sample_mask) {
8462          /* SampleMask should be in Y[15:0]. */
8463          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8464          enabled_channels |= 0xc;
8465      }
8466    } else {
8467       if (ctx->program->info->ps.writes_z) {
8468          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8469          enabled_channels |= 0x1;
8470       }
8471
8472       if (ctx->program->info->ps.writes_stencil) {
8473          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8474          enabled_channels |= 0x2;
8475       }
8476
8477       if (ctx->program->info->ps.writes_sample_mask) {
8478          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8479          enabled_channels |= 0x4;
8480       }
8481    }
8482
8483    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8484     * writemask component.
8485     */
8486    if (ctx->options->chip_class == GFX6 &&
8487        ctx->options->family != CHIP_OLAND &&
8488        ctx->options->family != CHIP_HAINAN) {
8489             enabled_channels |= 0x1;
8490    }
8491
8492    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8493            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8494 }
8495
8496 static void export_fs_mrt_color(isel_context *ctx, int slot)
8497 {
8498    Builder bld(ctx->program, ctx->block);
8499    unsigned write_mask = ctx->outputs.mask[slot];
8500    Operand values[4];
8501
8502    for (unsigned i = 0; i < 4; ++i) {
8503       if (write_mask & (1 << i)) {
8504          values[i] = Operand(ctx->outputs.outputs[slot][i]);
8505       } else {
8506          values[i] = Operand(v1);
8507       }
8508    }
8509
8510    unsigned target, col_format;
8511    unsigned enabled_channels = 0;
8512    aco_opcode compr_op = (aco_opcode)0;
8513
8514    slot -= FRAG_RESULT_DATA0;
8515    target = V_008DFC_SQ_EXP_MRT + slot;
8516    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
8517
8518    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
8519    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
8520
8521    switch (col_format)
8522    {
8523    case V_028714_SPI_SHADER_ZERO:
8524       enabled_channels = 0; /* writemask */
8525       target = V_008DFC_SQ_EXP_NULL;
8526       break;
8527
8528    case V_028714_SPI_SHADER_32_R:
8529       enabled_channels = 1;
8530       break;
8531
8532    case V_028714_SPI_SHADER_32_GR:
8533       enabled_channels = 0x3;
8534       break;
8535
8536    case V_028714_SPI_SHADER_32_AR:
8537       if (ctx->options->chip_class >= GFX10) {
8538          /* Special case: on GFX10, the outputs are different for 32_AR */
8539          enabled_channels = 0x3;
8540          values[1] = values[3];
8541          values[3] = Operand(v1);
8542       } else {
8543          enabled_channels = 0x9;
8544       }
8545       break;
8546
8547    case V_028714_SPI_SHADER_FP16_ABGR:
8548       enabled_channels = 0x5;
8549       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
8550       break;
8551
8552    case V_028714_SPI_SHADER_UNORM16_ABGR:
8553       enabled_channels = 0x5;
8554       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
8555       break;
8556
8557    case V_028714_SPI_SHADER_SNORM16_ABGR:
8558       enabled_channels = 0x5;
8559       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
8560       break;
8561
8562    case V_028714_SPI_SHADER_UINT16_ABGR: {
8563       enabled_channels = 0x5;
8564       compr_op = aco_opcode::v_cvt_pk_u16_u32;
8565       if (is_int8 || is_int10) {
8566          /* clamp */
8567          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
8568          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8569
8570          for (unsigned i = 0; i < 4; i++) {
8571             if ((write_mask >> i) & 1) {
8572                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
8573                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
8574                                     values[i]);
8575             }
8576          }
8577       }
8578       break;
8579    }
8580
8581    case V_028714_SPI_SHADER_SINT16_ABGR:
8582       enabled_channels = 0x5;
8583       compr_op = aco_opcode::v_cvt_pk_i16_i32;
8584       if (is_int8 || is_int10) {
8585          /* clamp */
8586          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
8587          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
8588          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8589          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
8590
8591          for (unsigned i = 0; i < 4; i++) {
8592             if ((write_mask >> i) & 1) {
8593                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
8594                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
8595                                     values[i]);
8596                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
8597                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
8598                                     values[i]);
8599             }
8600          }
8601       }
8602       break;
8603
8604    case V_028714_SPI_SHADER_32_ABGR:
8605       enabled_channels = 0xF;
8606       break;
8607
8608    default:
8609       break;
8610    }
8611
8612    if (target == V_008DFC_SQ_EXP_NULL)
8613       return;
8614
8615    if ((bool) compr_op) {
8616       for (int i = 0; i < 2; i++) {
8617          /* check if at least one of the values to be compressed is enabled */
8618          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
8619          if (enabled) {
8620             enabled_channels |= enabled << (i*2);
8621             values[i] = bld.vop3(compr_op, bld.def(v1),
8622                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
8623                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
8624          } else {
8625             values[i] = Operand(v1);
8626          }
8627       }
8628       values[2] = Operand(v1);
8629       values[3] = Operand(v1);
8630    } else {
8631       for (int i = 0; i < 4; i++)
8632          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
8633    }
8634
8635    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8636            enabled_channels, target, (bool) compr_op);
8637 }
8638
8639 static void create_fs_exports(isel_context *ctx)
8640 {
8641    /* Export depth, stencil and sample mask. */
8642    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
8643        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
8644        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
8645       export_fs_mrt_z(ctx);
8646    }
8647
8648    /* Export all color render targets. */
8649    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
8650       if (ctx->outputs.mask[i])
8651          export_fs_mrt_color(ctx, i);
8652    }
8653 }
8654
8655 static void emit_stream_output(isel_context *ctx,
8656                                Temp const *so_buffers,
8657                                Temp const *so_write_offset,
8658                                const struct radv_stream_output *output)
8659 {
8660    unsigned num_comps = util_bitcount(output->component_mask);
8661    unsigned writemask = (1 << num_comps) - 1;
8662    unsigned loc = output->location;
8663    unsigned buf = output->buffer;
8664
8665    assert(num_comps && num_comps <= 4);
8666    if (!num_comps || num_comps > 4)
8667       return;
8668
8669    unsigned start = ffs(output->component_mask) - 1;
8670
8671    Temp out[4];
8672    bool all_undef = true;
8673    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
8674    for (unsigned i = 0; i < num_comps; i++) {
8675       out[i] = ctx->outputs.outputs[loc][start + i];
8676       all_undef = all_undef && !out[i].id();
8677    }
8678    if (all_undef)
8679       return;
8680
8681    while (writemask) {
8682       int start, count;
8683       u_bit_scan_consecutive_range(&writemask, &start, &count);
8684       if (count == 3 && ctx->options->chip_class == GFX6) {
8685          /* GFX6 doesn't support storing vec3, split it. */
8686          writemask |= 1u << (start + 2);
8687          count = 2;
8688       }
8689
8690       unsigned offset = output->offset + start * 4;
8691
8692       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8693       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8694       for (int i = 0; i < count; ++i)
8695          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8696       vec->definitions[0] = Definition(write_data);
8697       ctx->block->instructions.emplace_back(std::move(vec));
8698
8699       aco_opcode opcode;
8700       switch (count) {
8701       case 1:
8702          opcode = aco_opcode::buffer_store_dword;
8703          break;
8704       case 2:
8705          opcode = aco_opcode::buffer_store_dwordx2;
8706          break;
8707       case 3:
8708          opcode = aco_opcode::buffer_store_dwordx3;
8709          break;
8710       case 4:
8711          opcode = aco_opcode::buffer_store_dwordx4;
8712          break;
8713       default:
8714          unreachable("Unsupported dword count.");
8715       }
8716
8717       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8718       store->operands[0] = Operand(so_write_offset[buf]);
8719       store->operands[1] = Operand(so_buffers[buf]);
8720       store->operands[2] = Operand((uint32_t) 0);
8721       store->operands[3] = Operand(write_data);
8722       if (offset > 4095) {
8723          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8724          Builder bld(ctx->program, ctx->block);
8725          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8726       } else {
8727          store->offset = offset;
8728       }
8729       store->offen = true;
8730       store->glc = true;
8731       store->dlc = false;
8732       store->slc = true;
8733       store->can_reorder = true;
8734       ctx->block->instructions.emplace_back(std::move(store));
8735    }
8736 }
8737
8738 static void emit_streamout(isel_context *ctx, unsigned stream)
8739 {
8740    Builder bld(ctx->program, ctx->block);
8741
8742    Temp so_buffers[4];
8743    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8744    for (unsigned i = 0; i < 4; i++) {
8745       unsigned stride = ctx->program->info->so.strides[i];
8746       if (!stride)
8747          continue;
8748
8749       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
8750       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
8751    }
8752
8753    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8754                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8755
8756    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8757
8758    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8759
8760    if_context ic;
8761    begin_divergent_if_then(ctx, &ic, can_emit);
8762
8763    bld.reset(ctx->block);
8764
8765    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8766
8767    Temp so_write_offset[4];
8768
8769    for (unsigned i = 0; i < 4; i++) {
8770       unsigned stride = ctx->program->info->so.strides[i];
8771       if (!stride)
8772          continue;
8773
8774       if (stride == 1) {
8775          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8776                                 get_arg(ctx, ctx->args->streamout_write_idx),
8777                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8778          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8779
8780          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8781       } else {
8782          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8783          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8784                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8785          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8786       }
8787    }
8788
8789    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8790       struct radv_stream_output *output =
8791          &ctx->program->info->so.outputs[i];
8792       if (stream != output->stream)
8793          continue;
8794
8795       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8796    }
8797
8798    begin_divergent_if_else(ctx, &ic);
8799    end_divergent_if(ctx, &ic);
8800 }
8801
8802 } /* end namespace */
8803
8804 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8805 {
8806    /* Split all arguments except for the first (ring_offsets) and the last
8807     * (exec) so that the dead channels don't stay live throughout the program.
8808     */
8809    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8810       if (startpgm->definitions[i].regClass().size() > 1) {
8811          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8812                            startpgm->definitions[i].regClass().size());
8813       }
8814    }
8815 }
8816
8817 void handle_bc_optimize(isel_context *ctx)
8818 {
8819    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8820    Builder bld(ctx->program, ctx->block);
8821    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8822    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8823    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8824    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8825    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8826    if (uses_center && uses_centroid) {
8827       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8828                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8829
8830       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8831          Temp new_coord[2];
8832          for (unsigned i = 0; i < 2; i++) {
8833             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8834             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8835             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8836                                     persp_centroid, persp_center, sel);
8837          }
8838          ctx->persp_centroid = bld.tmp(v2);
8839          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8840                     Operand(new_coord[0]), Operand(new_coord[1]));
8841          emit_split_vector(ctx, ctx->persp_centroid, 2);
8842       }
8843
8844       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8845          Temp new_coord[2];
8846          for (unsigned i = 0; i < 2; i++) {
8847             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8848             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8849             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8850                                     linear_centroid, linear_center, sel);
8851          }
8852          ctx->linear_centroid = bld.tmp(v2);
8853          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8854                     Operand(new_coord[0]), Operand(new_coord[1]));
8855          emit_split_vector(ctx, ctx->linear_centroid, 2);
8856       }
8857    }
8858 }
8859
8860 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8861 {
8862    Program *program = ctx->program;
8863
8864    unsigned float_controls = shader->info.float_controls_execution_mode;
8865
8866    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8867       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8868    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8869       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8870                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8871
8872    program->next_fp_mode.must_flush_denorms32 =
8873       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8874    program->next_fp_mode.must_flush_denorms16_64 =
8875       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8876                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8877
8878    program->next_fp_mode.care_about_round32 =
8879       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8880
8881    program->next_fp_mode.care_about_round16_64 =
8882       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8883                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8884
8885    /* default to preserving fp16 and fp64 denorms, since it's free */
8886    if (program->next_fp_mode.must_flush_denorms16_64)
8887       program->next_fp_mode.denorm16_64 = 0;
8888    else
8889       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8890
8891    /* preserving fp32 denorms is expensive, so only do it if asked */
8892    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8893       program->next_fp_mode.denorm32 = fp_denorm_keep;
8894    else
8895       program->next_fp_mode.denorm32 = 0;
8896
8897    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8898       program->next_fp_mode.round32 = fp_round_tz;
8899    else
8900       program->next_fp_mode.round32 = fp_round_ne;
8901
8902    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8903       program->next_fp_mode.round16_64 = fp_round_tz;
8904    else
8905       program->next_fp_mode.round16_64 = fp_round_ne;
8906
8907    ctx->block->fp_mode = program->next_fp_mode;
8908 }
8909
8910 void cleanup_cfg(Program *program)
8911 {
8912    /* create linear_succs/logical_succs */
8913    for (Block& BB : program->blocks) {
8914       for (unsigned idx : BB.linear_preds)
8915          program->blocks[idx].linear_succs.emplace_back(BB.index);
8916       for (unsigned idx : BB.logical_preds)
8917          program->blocks[idx].logical_succs.emplace_back(BB.index);
8918    }
8919 }
8920
8921 void select_program(Program *program,
8922                     unsigned shader_count,
8923                     struct nir_shader *const *shaders,
8924                     ac_shader_config* config,
8925                     struct radv_shader_args *args)
8926 {
8927    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
8928
8929    for (unsigned i = 0; i < shader_count; i++) {
8930       nir_shader *nir = shaders[i];
8931       init_context(&ctx, nir);
8932
8933       setup_fp_mode(&ctx, nir);
8934
8935       if (!i) {
8936          /* needs to be after init_context() for FS */
8937          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8938          append_logical_start(ctx.block);
8939          split_arguments(&ctx, startpgm);
8940       }
8941
8942       if_context ic;
8943       if (shader_count >= 2) {
8944          Builder bld(ctx.program, ctx.block);
8945          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
8946          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8947          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8948
8949          begin_divergent_if_then(&ctx, &ic, cond);
8950       }
8951
8952       if (i) {
8953          Builder bld(ctx.program, ctx.block);
8954          assert(ctx.stage == vertex_geometry_gs);
8955          bld.barrier(aco_opcode::p_memory_barrier_shared);
8956          bld.sopp(aco_opcode::s_barrier);
8957
8958          ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
8959       } else if (ctx.stage == geometry_gs)
8960          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
8961
8962       if (ctx.stage == fragment_fs)
8963          handle_bc_optimize(&ctx);
8964
8965       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8966       visit_cf_list(&ctx, &func->body);
8967
8968       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
8969          emit_streamout(&ctx, 0);
8970
8971       if (ctx.stage == vertex_vs) {
8972          create_vs_exports(&ctx);
8973       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
8974          Builder bld(ctx.program, ctx.block);
8975          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
8976          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
8977       }
8978
8979       if (ctx.stage == fragment_fs)
8980          create_fs_exports(&ctx);
8981
8982       if (shader_count >= 2) {
8983          begin_divergent_if_else(&ctx, &ic);
8984          end_divergent_if(&ctx, &ic);
8985       }
8986
8987       ralloc_free(ctx.divergent_vals);
8988    }
8989
8990    program->config->float_mode = program->blocks[0].fp_mode.val;
8991
8992    append_logical_end(ctx.block);
8993    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
8994    Builder bld(ctx.program, ctx.block);
8995    if (ctx.program->wb_smem_l1_on_end)
8996       bld.smem(aco_opcode::s_dcache_wb, false);
8997    bld.sopp(aco_opcode::s_endpgm);
8998
8999    cleanup_cfg(program);
9000 }
9001
9002 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
9003                            ac_shader_config* config,
9004                            struct radv_shader_args *args)
9005 {
9006    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
9007
9008    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
9009    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
9010    program->next_fp_mode.must_flush_denorms32 = false;
9011    program->next_fp_mode.must_flush_denorms16_64 = false;
9012    program->next_fp_mode.care_about_round32 = false;
9013    program->next_fp_mode.care_about_round16_64 = false;
9014    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9015    program->next_fp_mode.denorm32 = 0;
9016    program->next_fp_mode.round32 = fp_round_ne;
9017    program->next_fp_mode.round16_64 = fp_round_ne;
9018    ctx.block->fp_mode = program->next_fp_mode;
9019
9020    add_startpgm(&ctx);
9021    append_logical_start(ctx.block);
9022
9023    Builder bld(ctx.program, ctx.block);
9024
9025    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
9026
9027    Operand stream_id(0u);
9028    if (args->shader_info->so.num_outputs)
9029       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9030                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
9031
9032    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9033
9034    std::stack<Block> endif_blocks;
9035
9036    for (unsigned stream = 0; stream < 4; stream++) {
9037       if (stream_id.isConstant() && stream != stream_id.constantValue())
9038          continue;
9039
9040       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9041       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9042          continue;
9043
9044       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9045
9046       unsigned BB_if_idx = ctx.block->index;
9047       Block BB_endif = Block();
9048       if (!stream_id.isConstant()) {
9049          /* begin IF */
9050          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9051          append_logical_end(ctx.block);
9052          ctx.block->kind |= block_kind_uniform;
9053          bld.branch(aco_opcode::p_cbranch_z, cond);
9054
9055          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9056
9057          ctx.block = ctx.program->create_and_insert_block();
9058          add_edge(BB_if_idx, ctx.block);
9059          bld.reset(ctx.block);
9060          append_logical_start(ctx.block);
9061       }
9062
9063       unsigned offset = 0;
9064       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9065          if (args->shader_info->gs.output_streams[i] != stream)
9066             continue;
9067
9068          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9069          unsigned length = util_last_bit(output_usage_mask);
9070          for (unsigned j = 0; j < length; ++j) {
9071             if (!(output_usage_mask & (1 << j)))
9072                continue;
9073
9074             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9075             Temp voffset = vtx_offset;
9076             if (const_offset >= 4096u) {
9077                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9078                const_offset %= 4096u;
9079             }
9080
9081             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9082             mubuf->definitions[0] = bld.def(v1);
9083             mubuf->operands[0] = Operand(voffset);
9084             mubuf->operands[1] = Operand(gsvs_ring);
9085             mubuf->operands[2] = Operand(0u);
9086             mubuf->offen = true;
9087             mubuf->offset = const_offset;
9088             mubuf->glc = true;
9089             mubuf->slc = true;
9090             mubuf->dlc = args->options->chip_class >= GFX10;
9091             mubuf->barrier = barrier_none;
9092             mubuf->can_reorder = true;
9093
9094             ctx.outputs.mask[i] |= 1 << j;
9095             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
9096
9097             bld.insert(std::move(mubuf));
9098
9099             offset++;
9100          }
9101       }
9102
9103       if (args->shader_info->so.num_outputs) {
9104          emit_streamout(&ctx, stream);
9105          bld.reset(ctx.block);
9106       }
9107
9108       if (stream == 0) {
9109          create_vs_exports(&ctx);
9110          ctx.block->kind |= block_kind_export_end;
9111       }
9112
9113       if (!stream_id.isConstant()) {
9114          append_logical_end(ctx.block);
9115
9116          /* branch from then block to endif block */
9117          bld.branch(aco_opcode::p_branch);
9118          add_edge(ctx.block->index, &BB_endif);
9119          ctx.block->kind |= block_kind_uniform;
9120
9121          /* emit else block */
9122          ctx.block = ctx.program->create_and_insert_block();
9123          add_edge(BB_if_idx, ctx.block);
9124          bld.reset(ctx.block);
9125          append_logical_start(ctx.block);
9126
9127          endif_blocks.push(std::move(BB_endif));
9128       }
9129    }
9130
9131    while (!endif_blocks.empty()) {
9132       Block BB_endif = std::move(endif_blocks.top());
9133       endif_blocks.pop();
9134
9135       Block *BB_else = ctx.block;
9136
9137       append_logical_end(BB_else);
9138       /* branch from else block to endif block */
9139       bld.branch(aco_opcode::p_branch);
9140       add_edge(BB_else->index, &BB_endif);
9141       BB_else->kind |= block_kind_uniform;
9142
9143       /** emit endif merge block */
9144       ctx.block = program->insert_block(std::move(BB_endif));
9145       bld.reset(ctx.block);
9146       append_logical_start(ctx.block);
9147    }
9148
9149    program->config->float_mode = program->blocks[0].fp_mode.val;
9150
9151    append_logical_end(ctx.block);
9152    ctx.block->kind |= block_kind_uniform;
9153    bld.sopp(aco_opcode::s_endpgm);
9154
9155    cleanup_cfg(program);
9156 }
9157 }