src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_old;
  83
  84    unsigned BB_if_idx;
  85    unsigned invert_idx;
  86    bool then_branch_divergent;
  87    Block BB_invert;
  88    Block BB_endif;
  89 };
  90
  91 static void visit_cf_list(struct isel_context *ctx,
  92                           struct exec_list *list);
  93
  94 static void add_logical_edge(unsigned pred_idx, Block *succ)
  95 {
  96    succ->logical_preds.emplace_back(pred_idx);
  97 }
  98
  99
 100 static void add_linear_edge(unsigned pred_idx, Block *succ)
 101 {
 102    succ->linear_preds.emplace_back(pred_idx);
 103 }
 104
 105 static void add_edge(unsigned pred_idx, Block *succ)
 106 {
 107    add_logical_edge(pred_idx, succ);
 108    add_linear_edge(pred_idx, succ);
 109 }
 110
 111 static void append_logical_start(Block *b)
 112 {
 113    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 114 }
 115
 116 static void append_logical_end(Block *b)
 117 {
 118    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 119 }
 120
 121 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 122 {
 123    assert(ctx->allocated[def->index].id());
 124    return ctx->allocated[def->index];
 125 }
 126
 127 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 128                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 129 {
 130    Builder bld(ctx->program, ctx->block);
 131    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 132    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 133
 134    if (ctx->program->wave_size == 32) {
 135       return thread_id_lo;
 136    } else {
 137       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 138       return thread_id_hi;
 139    }
 140 }
 141
 142 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 143 {
 144    Builder bld(ctx->program, ctx->block);
 145
 146    if (!dst.id())
 147       dst = bld.tmp(src.regClass());
 148
 149    assert(src.size() == dst.size());
 150
 151    if (ctx->stage != fragment_fs) {
 152       if (!dst.id())
 153          return src;
 154
 155       bld.copy(Definition(dst), src);
 156       return dst;
 157    }
 158
 159    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 160    ctx->program->needs_wqm |= program_needs_wqm;
 161    return dst;
 162 }
 163
 164 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 165 {
 166    if (index.regClass() == s1)
 167       return bld.readlane(bld.def(s1), data, index);
 168
 169    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 170
 171    /* Currently not implemented on GFX6-7 */
 172    assert(ctx->options->chip_class >= GFX8);
 173
 174    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 175       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 176    }
 177
 178    /* GFX10, wave64 mode:
 179     * The bpermute instruction is limited to half-wave operation, which means that it can't
 180     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 181     * emulate it here.
 182     */
 183    if (!ctx->has_gfx10_wave64_bpermute) {
 184       ctx->has_gfx10_wave64_bpermute = true;
 185       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 186       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 187    }
 188
 189    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 190    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 191    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 192    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 193
 194    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 195                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 196 }
 197
 198 Temp as_vgpr(isel_context *ctx, Temp val)
 199 {
 200    if (val.type() == RegType::sgpr) {
 201       Builder bld(ctx->program, ctx->block);
 202       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 203    }
 204    assert(val.type() == RegType::vgpr);
 205    return val;
 206 }
 207
 208 //assumes a != 0xffffffff
 209 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 210 {
 211    assert(b != 0);
 212    Builder bld(ctx->program, ctx->block);
 213
 214    if (util_is_power_of_two_or_zero(b)) {
 215       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 216       return;
 217    }
 218
 219    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 220
 221    assert(info.multiplier <= 0xffffffff);
 222
 223    bool pre_shift = info.pre_shift != 0;
 224    bool increment = info.increment != 0;
 225    bool multiply = true;
 226    bool post_shift = info.post_shift != 0;
 227
 228    if (!pre_shift && !increment && !multiply && !post_shift) {
 229       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 230       return;
 231    }
 232
 233    Temp pre_shift_dst = a;
 234    if (pre_shift) {
 235       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 237    }
 238
 239    Temp increment_dst = pre_shift_dst;
 240    if (increment) {
 241       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 242       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 243    }
 244
 245    Temp multiply_dst = increment_dst;
 246    if (multiply) {
 247       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 248       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 249                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 250    }
 251
 252    if (post_shift) {
 253       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 254    }
 255 }
 256
 257 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 258 {
 259    Builder bld(ctx->program, ctx->block);
 260    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 261 }
 262
 263
 264 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 265 {
 266    /* no need to extract the whole vector */
 267    if (src.regClass() == dst_rc) {
 268       assert(idx == 0);
 269       return src;
 270    }
 271    assert(src.size() > idx);
 272    Builder bld(ctx->program, ctx->block);
 273    auto it = ctx->allocated_vec.find(src.id());
 274    /* the size check needs to be early because elements other than 0 may be garbage */
 275    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 276       if (it->second[idx].regClass() == dst_rc) {
 277          return it->second[idx];
 278       } else {
 279          assert(dst_rc.size() == it->second[idx].regClass().size());
 280          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 281          return bld.copy(bld.def(dst_rc), it->second[idx]);
 282       }
 283    }
 284
 285    if (src.size() == dst_rc.size()) {
 286       assert(idx == 0);
 287       return bld.copy(bld.def(dst_rc), src);
 288    } else {
 289       Temp dst = bld.tmp(dst_rc);
 290       emit_extract_vector(ctx, src, idx, dst);
 291       return dst;
 292    }
 293 }
 294
 295 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 296 {
 297    if (num_components == 1)
 298       return;
 299    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 300       return;
 301    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 302    split->operands[0] = Operand(vec_src);
 303    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 304    for (unsigned i = 0; i < num_components; i++) {
 305       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 306       split->definitions[i] = Definition(elems[i]);
 307    }
 308    ctx->block->instructions.emplace_back(std::move(split));
 309    ctx->allocated_vec.emplace(vec_src.id(), elems);
 310 }
 311
 312 /* This vector expansion uses a mask to determine which elements in the new vector
 313  * come from the original vector. The other elements are undefined. */
 314 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 315 {
 316    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 317
 318    if (vec_src == dst)
 319       return;
 320
 321    Builder bld(ctx->program, ctx->block);
 322    if (num_components == 1) {
 323       if (dst.type() == RegType::sgpr)
 324          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 325       else
 326          bld.copy(Definition(dst), vec_src);
 327       return;
 328    }
 329
 330    unsigned component_size = dst.size() / num_components;
 331    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 332
 333    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 334    vec->definitions[0] = Definition(dst);
 335    unsigned k = 0;
 336    for (unsigned i = 0; i < num_components; i++) {
 337       if (mask & (1 << i)) {
 338          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 339          if (dst.type() == RegType::sgpr)
 340             src = bld.as_uniform(src);
 341          vec->operands[i] = Operand(src);
 342       } else {
 343          vec->operands[i] = Operand(0u);
 344       }
 345       elems[i] = vec->operands[i].getTemp();
 346    }
 347    ctx->block->instructions.emplace_back(std::move(vec));
 348    ctx->allocated_vec.emplace(dst.id(), elems);
 349 }
 350
 351 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 352 {
 353    Builder bld(ctx->program, ctx->block);
 354    if (!dst.id())
 355       dst = bld.tmp(bld.lm);
 356
 357    assert(val.regClass() == s1);
 358    assert(dst.regClass() == bld.lm);
 359
 360    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 361 }
 362
 363 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 364 {
 365    Builder bld(ctx->program, ctx->block);
 366    if (!dst.id())
 367       dst = bld.tmp(s1);
 368
 369    assert(val.regClass() == bld.lm);
 370    assert(dst.regClass() == s1);
 371
 372    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 373    Temp tmp = bld.tmp(s1);
 374    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 375    return emit_wqm(ctx, tmp, dst);
 376 }
 377
 378 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 379 {
 380    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 381       return get_ssa_temp(ctx, src.src.ssa);
 382
 383    if (src.src.ssa->num_components == size) {
 384       bool identity_swizzle = true;
 385       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 386          if (src.swizzle[i] != i)
 387             identity_swizzle = false;
 388       }
 389       if (identity_swizzle)
 390          return get_ssa_temp(ctx, src.src.ssa);
 391    }
 392
 393    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 394    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 395    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 396    assert(vec.size() % elem_size == 0);
 397
 398    RegClass elem_rc = RegClass(vec.type(), elem_size);
 399    if (size == 1) {
 400       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 401    } else {
 402       assert(size <= 4);
 403       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 404       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 405       for (unsigned i = 0; i < size; ++i) {
 406          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 407          vec_instr->operands[i] = Operand{elems[i]};
 408       }
 409       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 410       vec_instr->definitions[0] = Definition(dst);
 411       ctx->block->instructions.emplace_back(std::move(vec_instr));
 412       ctx->allocated_vec.emplace(dst.id(), elems);
 413       return dst;
 414    }
 415 }
 416
 417 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 418 {
 419    if (ptr.size() == 2)
 420       return ptr;
 421    Builder bld(ctx->program, ctx->block);
 422    if (ptr.type() == RegType::vgpr)
 423       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 424    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 425                      ptr, Operand((unsigned)ctx->options->address32_hi));
 426 }
 427
 428 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 429 {
 430    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 431    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 432    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 433    sop2->definitions[0] = Definition(dst);
 434    if (writes_scc)
 435       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 436    ctx->block->instructions.emplace_back(std::move(sop2));
 437 }
 438
 439 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 440                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 441 {
 442    Builder bld(ctx->program, ctx->block);
 443    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 444    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 445    if (src1.type() == RegType::sgpr) {
 446       if (commutative && src0.type() == RegType::vgpr) {
 447          Temp t = src0;
 448          src0 = src1;
 449          src1 = t;
 450       } else if (src0.type() == RegType::vgpr &&
 451                  op != aco_opcode::v_madmk_f32 &&
 452                  op != aco_opcode::v_madak_f32 &&
 453                  op != aco_opcode::v_madmk_f16 &&
 454                  op != aco_opcode::v_madak_f16) {
 455          /* If the instruction is not commutative, we emit a VOP3A instruction */
 456          bld.vop2_e64(op, Definition(dst), src0, src1);
 457          return;
 458       } else {
 459          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 460       }
 461    }
 462
 463    if (flush_denorms && ctx->program->chip_class < GFX9) {
 464       assert(dst.size() == 1);
 465       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 466       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 467    } else {
 468       bld.vop2(op, Definition(dst), src0, src1);
 469    }
 470 }
 471
 472 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 473                             bool flush_denorms = false)
 474 {
 475    Temp src0 = get_alu_src(ctx, instr->src[0]);
 476    Temp src1 = get_alu_src(ctx, instr->src[1]);
 477    Temp src2 = get_alu_src(ctx, instr->src[2]);
 478
 479    /* ensure that the instruction has at most 1 sgpr operand
 480     * The optimizer will inline constants for us */
 481    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 482       src0 = as_vgpr(ctx, src0);
 483    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 484       src1 = as_vgpr(ctx, src1);
 485    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 486       src2 = as_vgpr(ctx, src2);
 487
 488    Builder bld(ctx->program, ctx->block);
 489    if (flush_denorms && ctx->program->chip_class < GFX9) {
 490       assert(dst.size() == 1);
 491       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 492       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 493    } else {
 494       bld.vop3(op, Definition(dst), src0, src1, src2);
 495    }
 496 }
 497
 498 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 499 {
 500    Builder bld(ctx->program, ctx->block);
 501    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 502 }
 503
 504 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 505 {
 506    Temp src0 = get_alu_src(ctx, instr->src[0]);
 507    Temp src1 = get_alu_src(ctx, instr->src[1]);
 508    assert(src0.size() == src1.size());
 509
 510    aco_ptr<Instruction> vopc;
 511    if (src1.type() == RegType::sgpr) {
 512       if (src0.type() == RegType::vgpr) {
 513          /* to swap the operands, we might also have to change the opcode */
 514          switch (op) {
 515             case aco_opcode::v_cmp_lt_f32:
 516                op = aco_opcode::v_cmp_gt_f32;
 517                break;
 518             case aco_opcode::v_cmp_ge_f32:
 519                op = aco_opcode::v_cmp_le_f32;
 520                break;
 521             case aco_opcode::v_cmp_lt_i32:
 522                op = aco_opcode::v_cmp_gt_i32;
 523                break;
 524             case aco_opcode::v_cmp_ge_i32:
 525                op = aco_opcode::v_cmp_le_i32;
 526                break;
 527             case aco_opcode::v_cmp_lt_u32:
 528                op = aco_opcode::v_cmp_gt_u32;
 529                break;
 530             case aco_opcode::v_cmp_ge_u32:
 531                op = aco_opcode::v_cmp_le_u32;
 532                break;
 533             case aco_opcode::v_cmp_lt_f64:
 534                op = aco_opcode::v_cmp_gt_f64;
 535                break;
 536             case aco_opcode::v_cmp_ge_f64:
 537                op = aco_opcode::v_cmp_le_f64;
 538                break;
 539             case aco_opcode::v_cmp_lt_i64:
 540                op = aco_opcode::v_cmp_gt_i64;
 541                break;
 542             case aco_opcode::v_cmp_ge_i64:
 543                op = aco_opcode::v_cmp_le_i64;
 544                break;
 545             case aco_opcode::v_cmp_lt_u64:
 546                op = aco_opcode::v_cmp_gt_u64;
 547                break;
 548             case aco_opcode::v_cmp_ge_u64:
 549                op = aco_opcode::v_cmp_le_u64;
 550                break;
 551             default: /* eq and ne are commutative */
 552                break;
 553          }
 554          Temp t = src0;
 555          src0 = src1;
 556          src1 = t;
 557       } else {
 558          src1 = as_vgpr(ctx, src1);
 559       }
 560    }
 561
 562    Builder bld(ctx->program, ctx->block);
 563    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 564 }
 565
 566 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 567 {
 568    Temp src0 = get_alu_src(ctx, instr->src[0]);
 569    Temp src1 = get_alu_src(ctx, instr->src[1]);
 570    Builder bld(ctx->program, ctx->block);
 571
 572    assert(dst.regClass() == bld.lm);
 573    assert(src0.type() == RegType::sgpr);
 574    assert(src1.type() == RegType::sgpr);
 575    assert(src0.regClass() == src1.regClass());
 576
 577    /* Emit the SALU comparison instruction */
 578    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 579    /* Turn the result into a per-lane bool */
 580    bool_to_vector_condition(ctx, cmp, dst);
 581 }
 582
 583 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 584                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 585 {
 586    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 587    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 588    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 589    bool use_valu = s_op == aco_opcode::num_opcodes ||
 590                    divergent_vals ||
 591                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 592                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 593    aco_opcode op = use_valu ? v_op : s_op;
 594    assert(op != aco_opcode::num_opcodes);
 595    assert(dst.regClass() == ctx->program->lane_mask);
 596
 597    if (use_valu)
 598       emit_vopc_instruction(ctx, instr, op, dst);
 599    else
 600       emit_sopc_instruction(ctx, instr, op, dst);
 601 }
 602
 603 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 604 {
 605    Builder bld(ctx->program, ctx->block);
 606    Temp src0 = get_alu_src(ctx, instr->src[0]);
 607    Temp src1 = get_alu_src(ctx, instr->src[1]);
 608
 609    assert(dst.regClass() == bld.lm);
 610    assert(src0.regClass() == bld.lm);
 611    assert(src1.regClass() == bld.lm);
 612
 613    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 614 }
 615
 616 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 617 {
 618    Builder bld(ctx->program, ctx->block);
 619    Temp cond = get_alu_src(ctx, instr->src[0]);
 620    Temp then = get_alu_src(ctx, instr->src[1]);
 621    Temp els = get_alu_src(ctx, instr->src[2]);
 622
 623    assert(cond.regClass() == bld.lm);
 624
 625    if (dst.type() == RegType::vgpr) {
 626       aco_ptr<Instruction> bcsel;
 627       if (dst.size() == 1) {
 628          then = as_vgpr(ctx, then);
 629          els = as_vgpr(ctx, els);
 630
 631          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 632       } else if (dst.size() == 2) {
 633          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 634          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 635          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 637
 638          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 639          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 640
 641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 642       } else {
 643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 644          nir_print_instr(&instr->instr, stderr);
 645          fprintf(stderr, "\n");
 646       }
 647       return;
 648    }
 649
 650    if (instr->dest.dest.ssa.bit_size == 1) {
 651       assert(dst.regClass() == bld.lm);
 652       assert(then.regClass() == bld.lm);
 653       assert(els.regClass() == bld.lm);
 654    }
 655
 656    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 657       if (dst.regClass() == s1 || dst.regClass() == s2) {
 658          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 659          assert(dst.size() == then.size());
 660          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 661          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 662       } else {
 663          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 664          nir_print_instr(&instr->instr, stderr);
 665          fprintf(stderr, "\n");
 666       }
 667       return;
 668    }
 669
 670    /* divergent boolean bcsel
 671     * this implements bcsel on bools: dst = s0 ? s1 : s2
 672     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 673    assert(instr->dest.dest.ssa.bit_size == 1);
 674
 675    if (cond.id() != then.id())
 676       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 677
 678    if (cond.id() == els.id())
 679       bld.sop1(Builder::s_mov, Definition(dst), then);
 680    else
 681       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 682                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 683 }
 684
 685 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 686                     aco_opcode op, uint32_t undo)
 687 {
 688    /* multiply by 16777216 to handle denormals */
 689    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 690                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 691    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 692    scaled = bld.vop1(op, bld.def(v1), scaled);
 693    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 694
 695    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 696
 697    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 698 }
 699
 700 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 701 {
 702    if (ctx->block->fp_mode.denorm32 == 0) {
 703       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 704       return;
 705    }
 706
 707    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 708 }
 709
 710 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 711 {
 712    if (ctx->block->fp_mode.denorm32 == 0) {
 713       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 714       return;
 715    }
 716
 717    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 718 }
 719
 720 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 721 {
 722    if (ctx->block->fp_mode.denorm32 == 0) {
 723       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 724       return;
 725    }
 726
 727    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 728 }
 729
 730 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 731 {
 732    if (ctx->block->fp_mode.denorm32 == 0) {
 733       bld.vop1(aco_opcode::v_log_f32, dst, val);
 734       return;
 735    }
 736
 737    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 738 }
 739
 740 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 741 {
 742    if (ctx->options->chip_class >= GFX7)
 743       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 744
 745    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 746    /* TODO: create more efficient code! */
 747    if (val.type() == RegType::sgpr)
 748       val = as_vgpr(ctx, val);
 749
 750    /* Split the input value. */
 751    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 752    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 753
 754    /* Extract the exponent and compute the unbiased value. */
 755    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 756
 757    /* Extract the fractional part. */
 758    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 759    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 760
 761    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 762    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 763
 764    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 765    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 766    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 767    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 768    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 769
 770    /* Get the sign bit. */
 771    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 772
 773    /* Decide the operation to apply depending on the unbiased exponent. */
 774    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 775    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 776    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 777    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 778    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 779    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 780
 781    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 782 }
 783
 784 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 785 {
 786    if (ctx->options->chip_class >= GFX7)
 787       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 788
 789    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 790    Temp src0 = as_vgpr(ctx, val);
 791
 792    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 793    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 794
 795    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 796    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 797    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 798
 799    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 800    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 801    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 803
 804    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 805    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 806
 807    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 808
 809    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 810    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 811
 812    return add->definitions[0].getTemp();
 813 }
 814
 815 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 816 {
 817    if (!instr->dest.dest.is_ssa) {
 818       fprintf(stderr, "nir alu dst not in ssa: ");
 819       nir_print_instr(&instr->instr, stderr);
 820       fprintf(stderr, "\n");
 821       abort();
 822    }
 823    Builder bld(ctx->program, ctx->block);
 824    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 825    switch(instr->op) {
 826    case nir_op_vec2:
 827    case nir_op_vec3:
 828    case nir_op_vec4: {
 829       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 830       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 831       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 832          elems[i] = get_alu_src(ctx, instr->src[i]);
 833          vec->operands[i] = Operand{elems[i]};
 834       }
 835       vec->definitions[0] = Definition(dst);
 836       ctx->block->instructions.emplace_back(std::move(vec));
 837       ctx->allocated_vec.emplace(dst.id(), elems);
 838       break;
 839    }
 840    case nir_op_mov: {
 841       Temp src = get_alu_src(ctx, instr->src[0]);
 842       aco_ptr<Instruction> mov;
 843       if (dst.type() == RegType::sgpr) {
 844          if (src.type() == RegType::vgpr)
 845             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 846          else if (src.regClass() == s1)
 847             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 848          else if (src.regClass() == s2)
 849             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 850          else
 851             unreachable("wrong src register class for nir_op_imov");
 852       } else if (dst.regClass() == v1) {
 853          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 854       } else if (dst.regClass() == v2) {
 855          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 856       } else {
 857          nir_print_instr(&instr->instr, stderr);
 858          unreachable("Should have been lowered to scalar.");
 859       }
 860       break;
 861    }
 862    case nir_op_inot: {
 863       Temp src = get_alu_src(ctx, instr->src[0]);
 864       if (instr->dest.dest.ssa.bit_size == 1) {
 865          assert(src.regClass() == bld.lm);
 866          assert(dst.regClass() == bld.lm);
 867          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 868          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 869          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 870       } else if (dst.regClass() == v1) {
 871          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 872       } else if (dst.type() == RegType::sgpr) {
 873          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 874          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_ineg: {
 883       Temp src = get_alu_src(ctx, instr->src[0]);
 884       if (dst.regClass() == v1) {
 885          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 886       } else if (dst.regClass() == s1) {
 887          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 888       } else if (dst.size() == 2) {
 889          Temp src0 = bld.tmp(dst.type(), 1);
 890          Temp src1 = bld.tmp(dst.type(), 1);
 891          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 892
 893          if (dst.regClass() == s2) {
 894             Temp carry = bld.tmp(s1);
 895             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 896             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 897             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 898          } else {
 899             Temp lower = bld.tmp(v1);
 900             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 901             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 902             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 903          }
 904       } else {
 905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 906          nir_print_instr(&instr->instr, stderr);
 907          fprintf(stderr, "\n");
 908       }
 909       break;
 910    }
 911    case nir_op_iabs: {
 912       if (dst.regClass() == s1) {
 913          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 914       } else if (dst.regClass() == v1) {
 915          Temp src = get_alu_src(ctx, instr->src[0]);
 916          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 917       } else {
 918          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 919          nir_print_instr(&instr->instr, stderr);
 920          fprintf(stderr, "\n");
 921       }
 922       break;
 923    }
 924    case nir_op_isign: {
 925       Temp src = get_alu_src(ctx, instr->src[0]);
 926       if (dst.regClass() == s1) {
 927          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 928          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 929          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 930       } else if (dst.regClass() == s2) {
 931          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 932          Temp neqz;
 933          if (ctx->program->chip_class >= GFX8)
 934             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 935          else
 936             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 937          /* SCC gets zero-extended to 64 bit */
 938          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 939       } else if (dst.regClass() == v1) {
 940          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 941          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 942          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 943       } else if (dst.regClass() == v2) {
 944          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 945          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 946          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 947          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 948          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 949          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 950       } else {
 951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 952          nir_print_instr(&instr->instr, stderr);
 953          fprintf(stderr, "\n");
 954       }
 955       break;
 956    }
 957    case nir_op_imax: {
 958       if (dst.regClass() == v1) {
 959          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 960       } else if (dst.regClass() == s1) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 962       } else {
 963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 964          nir_print_instr(&instr->instr, stderr);
 965          fprintf(stderr, "\n");
 966       }
 967       break;
 968    }
 969    case nir_op_umax: {
 970       if (dst.regClass() == v1) {
 971          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 972       } else if (dst.regClass() == s1) {
 973          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 974       } else {
 975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 976          nir_print_instr(&instr->instr, stderr);
 977          fprintf(stderr, "\n");
 978       }
 979       break;
 980    }
 981    case nir_op_imin: {
 982       if (dst.regClass() == v1) {
 983          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 984       } else if (dst.regClass() == s1) {
 985          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 986       } else {
 987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 988          nir_print_instr(&instr->instr, stderr);
 989          fprintf(stderr, "\n");
 990       }
 991       break;
 992    }
 993    case nir_op_umin: {
 994       if (dst.regClass() == v1) {
 995          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 996       } else if (dst.regClass() == s1) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_ior: {
1006       if (instr->dest.dest.ssa.bit_size == 1) {
1007          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1008       } else if (dst.regClass() == v1) {
1009          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1010       } else if (dst.regClass() == s1) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1012       } else if (dst.regClass() == s2) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1014       } else {
1015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1016          nir_print_instr(&instr->instr, stderr);
1017          fprintf(stderr, "\n");
1018       }
1019       break;
1020    }
1021    case nir_op_iand: {
1022       if (instr->dest.dest.ssa.bit_size == 1) {
1023          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1024       } else if (dst.regClass() == v1) {
1025          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1026       } else if (dst.regClass() == s1) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1028       } else if (dst.regClass() == s2) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1030       } else {
1031          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1032          nir_print_instr(&instr->instr, stderr);
1033          fprintf(stderr, "\n");
1034       }
1035       break;
1036    }
1037    case nir_op_ixor: {
1038       if (instr->dest.dest.ssa.bit_size == 1) {
1039          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1040       } else if (dst.regClass() == v1) {
1041          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1042       } else if (dst.regClass() == s1) {
1043          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1044       } else if (dst.regClass() == s2) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1046       } else {
1047          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1048          nir_print_instr(&instr->instr, stderr);
1049          fprintf(stderr, "\n");
1050       }
1051       break;
1052    }
1053    case nir_op_ushr: {
1054       if (dst.regClass() == v1) {
1055          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1056       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1057          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1058                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1059       } else if (dst.regClass() == v2) {
1060          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1061                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1062       } else if (dst.regClass() == s2) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1064       } else if (dst.regClass() == s1) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1066       } else {
1067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1068          nir_print_instr(&instr->instr, stderr);
1069          fprintf(stderr, "\n");
1070       }
1071       break;
1072    }
1073    case nir_op_ishl: {
1074       if (dst.regClass() == v1) {
1075          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1076       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1077          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1078                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1079       } else if (dst.regClass() == v2) {
1080          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1081                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1082       } else if (dst.regClass() == s1) {
1083          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1084       } else if (dst.regClass() == s2) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_ishr: {
1094       if (dst.regClass() == v1) {
1095          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1096       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1097          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1098                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1099       } else if (dst.regClass() == v2) {
1100          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1101                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1102       } else if (dst.regClass() == s1) {
1103          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1104       } else if (dst.regClass() == s2) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1106       } else {
1107          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1108          nir_print_instr(&instr->instr, stderr);
1109          fprintf(stderr, "\n");
1110       }
1111       break;
1112    }
1113    case nir_op_find_lsb: {
1114       Temp src = get_alu_src(ctx, instr->src[0]);
1115       if (src.regClass() == s1) {
1116          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1117       } else if (src.regClass() == v1) {
1118          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1119       } else if (src.regClass() == s2) {
1120          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1121       } else {
1122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1123          nir_print_instr(&instr->instr, stderr);
1124          fprintf(stderr, "\n");
1125       }
1126       break;
1127    }
1128    case nir_op_ufind_msb:
1129    case nir_op_ifind_msb: {
1130       Temp src = get_alu_src(ctx, instr->src[0]);
1131       if (src.regClass() == s1 || src.regClass() == s2) {
1132          aco_opcode op = src.regClass() == s2 ?
1133                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1134                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1135          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1136
1137          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1138                                         Operand(src.size() * 32u - 1u), msb_rev);
1139          Temp msb = sub.def(0).getTemp();
1140          Temp carry = sub.def(1).getTemp();
1141
1142          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, bld.scc(carry));
1143       } else if (src.regClass() == v1) {
1144          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1145          Temp msb_rev = bld.tmp(v1);
1146          emit_vop1_instruction(ctx, instr, op, msb_rev);
1147          Temp msb = bld.tmp(v1);
1148          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1149          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1150       } else {
1151          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1152          nir_print_instr(&instr->instr, stderr);
1153          fprintf(stderr, "\n");
1154       }
1155       break;
1156    }
1157    case nir_op_bitfield_reverse: {
1158       if (dst.regClass() == s1) {
1159          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1160       } else if (dst.regClass() == v1) {
1161          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else {
1163          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1164          nir_print_instr(&instr->instr, stderr);
1165          fprintf(stderr, "\n");
1166       }
1167       break;
1168    }
1169    case nir_op_iadd: {
1170       if (dst.regClass() == s1) {
1171          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1172          break;
1173       }
1174
1175       Temp src0 = get_alu_src(ctx, instr->src[0]);
1176       Temp src1 = get_alu_src(ctx, instr->src[1]);
1177       if (dst.regClass() == v1) {
1178          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1179          break;
1180       }
1181
1182       assert(src0.size() == 2 && src1.size() == 2);
1183       Temp src00 = bld.tmp(src0.type(), 1);
1184       Temp src01 = bld.tmp(dst.type(), 1);
1185       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1186       Temp src10 = bld.tmp(src1.type(), 1);
1187       Temp src11 = bld.tmp(dst.type(), 1);
1188       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1189
1190       if (dst.regClass() == s2) {
1191          Temp carry = bld.tmp(s1);
1192          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1193          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1195       } else if (dst.regClass() == v2) {
1196          Temp dst0 = bld.tmp(v1);
1197          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1198          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1199          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1200       } else {
1201          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1202          nir_print_instr(&instr->instr, stderr);
1203          fprintf(stderr, "\n");
1204       }
1205       break;
1206    }
1207    case nir_op_uadd_sat: {
1208       Temp src0 = get_alu_src(ctx, instr->src[0]);
1209       Temp src1 = get_alu_src(ctx, instr->src[1]);
1210       if (dst.regClass() == s1) {
1211          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1212          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1213                   src0, src1);
1214          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1215       } else if (dst.regClass() == v1) {
1216          if (ctx->options->chip_class >= GFX9) {
1217             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1218             add->operands[0] = Operand(src0);
1219             add->operands[1] = Operand(src1);
1220             add->definitions[0] = Definition(dst);
1221             add->clamp = 1;
1222             ctx->block->instructions.emplace_back(std::move(add));
1223          } else {
1224             if (src1.regClass() != v1)
1225                std::swap(src0, src1);
1226             assert(src1.regClass() == v1);
1227             Temp tmp = bld.tmp(v1);
1228             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1229             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1230          }
1231       } else {
1232          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1233          nir_print_instr(&instr->instr, stderr);
1234          fprintf(stderr, "\n");
1235       }
1236       break;
1237    }
1238    case nir_op_uadd_carry: {
1239       Temp src0 = get_alu_src(ctx, instr->src[0]);
1240       Temp src1 = get_alu_src(ctx, instr->src[1]);
1241       if (dst.regClass() == s1) {
1242          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1243          break;
1244       }
1245       if (dst.regClass() == v1) {
1246          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1247          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1248          break;
1249       }
1250
1251       Temp src00 = bld.tmp(src0.type(), 1);
1252       Temp src01 = bld.tmp(dst.type(), 1);
1253       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1254       Temp src10 = bld.tmp(src1.type(), 1);
1255       Temp src11 = bld.tmp(dst.type(), 1);
1256       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1257       if (dst.regClass() == s2) {
1258          Temp carry = bld.tmp(s1);
1259          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1260          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1261          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1262       } else if (dst.regClass() == v2) {
1263          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1264          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1265          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1266          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1267       } else {
1268          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1269          nir_print_instr(&instr->instr, stderr);
1270          fprintf(stderr, "\n");
1271       }
1272       break;
1273    }
1274    case nir_op_isub: {
1275       if (dst.regClass() == s1) {
1276          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1277          break;
1278       }
1279
1280       Temp src0 = get_alu_src(ctx, instr->src[0]);
1281       Temp src1 = get_alu_src(ctx, instr->src[1]);
1282       if (dst.regClass() == v1) {
1283          bld.vsub32(Definition(dst), src0, src1);
1284          break;
1285       }
1286
1287       Temp src00 = bld.tmp(src0.type(), 1);
1288       Temp src01 = bld.tmp(dst.type(), 1);
1289       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1290       Temp src10 = bld.tmp(src1.type(), 1);
1291       Temp src11 = bld.tmp(dst.type(), 1);
1292       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1293       if (dst.regClass() == s2) {
1294          Temp carry = bld.tmp(s1);
1295          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1296          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1297          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1298       } else if (dst.regClass() == v2) {
1299          Temp lower = bld.tmp(v1);
1300          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1301          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1302          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1303       } else {
1304          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1305          nir_print_instr(&instr->instr, stderr);
1306          fprintf(stderr, "\n");
1307       }
1308       break;
1309    }
1310    case nir_op_usub_borrow: {
1311       Temp src0 = get_alu_src(ctx, instr->src[0]);
1312       Temp src1 = get_alu_src(ctx, instr->src[1]);
1313       if (dst.regClass() == s1) {
1314          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1315          break;
1316       } else if (dst.regClass() == v1) {
1317          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1318          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1319          break;
1320       }
1321
1322       Temp src00 = bld.tmp(src0.type(), 1);
1323       Temp src01 = bld.tmp(dst.type(), 1);
1324       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1325       Temp src10 = bld.tmp(src1.type(), 1);
1326       Temp src11 = bld.tmp(dst.type(), 1);
1327       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1328       if (dst.regClass() == s2) {
1329          Temp borrow = bld.tmp(s1);
1330          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1331          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1332          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1333       } else if (dst.regClass() == v2) {
1334          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1335          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1336          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1337          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_imul: {
1346       if (dst.regClass() == v1) {
1347          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1349       } else if (dst.regClass() == s1) {
1350          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_umul_high: {
1359       if (dst.regClass() == v1) {
1360          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1361       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1362          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1) {
1364          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1365                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1366          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_imul_high: {
1375       if (dst.regClass() == v1) {
1376          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1377       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1378          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1) {
1380          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1381                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1382          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1383       } else {
1384          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1385          nir_print_instr(&instr->instr, stderr);
1386          fprintf(stderr, "\n");
1387       }
1388       break;
1389    }
1390    case nir_op_fmul: {
1391       if (dst.size() == 1) {
1392          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1393       } else if (dst.size() == 2) {
1394          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1395                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1396       } else {
1397          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1398          nir_print_instr(&instr->instr, stderr);
1399          fprintf(stderr, "\n");
1400       }
1401       break;
1402    }
1403    case nir_op_fadd: {
1404       if (dst.size() == 1) {
1405          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1406       } else if (dst.size() == 2) {
1407          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1408                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1409       } else {
1410          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1411          nir_print_instr(&instr->instr, stderr);
1412          fprintf(stderr, "\n");
1413       }
1414       break;
1415    }
1416    case nir_op_fsub: {
1417       Temp src0 = get_alu_src(ctx, instr->src[0]);
1418       Temp src1 = get_alu_src(ctx, instr->src[1]);
1419       if (dst.size() == 1) {
1420          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1421             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1422          else
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1424       } else if (dst.size() == 2) {
1425          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1426                                      get_alu_src(ctx, instr->src[0]),
1427                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1428          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1429          sub->neg[1] = true;
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_fmax: {
1438       if (dst.size() == 1) {
1439          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1440       } else if (dst.size() == 2) {
1441          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1442             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1443                                 get_alu_src(ctx, instr->src[0]),
1444                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1445             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1446          } else {
1447             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1448                      get_alu_src(ctx, instr->src[0]),
1449                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1450          }
1451       } else {
1452          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1453          nir_print_instr(&instr->instr, stderr);
1454          fprintf(stderr, "\n");
1455       }
1456       break;
1457    }
1458    case nir_op_fmin: {
1459       if (dst.size() == 1) {
1460          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1461       } else if (dst.size() == 2) {
1462          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1463             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1464                                 get_alu_src(ctx, instr->src[0]),
1465                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1466             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1467          } else {
1468             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1469                      get_alu_src(ctx, instr->src[0]),
1470                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1471          }
1472       } else {
1473          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1474          nir_print_instr(&instr->instr, stderr);
1475          fprintf(stderr, "\n");
1476       }
1477       break;
1478    }
1479    case nir_op_fmax3: {
1480       if (dst.size() == 1) {
1481          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1482       } else {
1483          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1484          nir_print_instr(&instr->instr, stderr);
1485          fprintf(stderr, "\n");
1486       }
1487       break;
1488    }
1489    case nir_op_fmin3: {
1490       if (dst.size() == 1) {
1491          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fmed3: {
1500       if (dst.size() == 1) {
1501          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1502       } else {
1503          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1504          nir_print_instr(&instr->instr, stderr);
1505          fprintf(stderr, "\n");
1506       }
1507       break;
1508    }
1509    case nir_op_umax3: {
1510       if (dst.size() == 1) {
1511          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1512       } else {
1513          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1514          nir_print_instr(&instr->instr, stderr);
1515          fprintf(stderr, "\n");
1516       }
1517       break;
1518    }
1519    case nir_op_umin3: {
1520       if (dst.size() == 1) {
1521          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1522       } else {
1523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1524          nir_print_instr(&instr->instr, stderr);
1525          fprintf(stderr, "\n");
1526       }
1527       break;
1528    }
1529    case nir_op_umed3: {
1530       if (dst.size() == 1) {
1531          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_imax3: {
1540       if (dst.size() == 1) {
1541          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_imin3: {
1550       if (dst.size() == 1) {
1551          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_imed3: {
1560       if (dst.size() == 1) {
1561          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1562       } else {
1563          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1564          nir_print_instr(&instr->instr, stderr);
1565          fprintf(stderr, "\n");
1566       }
1567       break;
1568    }
1569    case nir_op_cube_face_coord: {
1570       Temp in = get_alu_src(ctx, instr->src[0], 3);
1571       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1572                       emit_extract_vector(ctx, in, 1, v1),
1573                       emit_extract_vector(ctx, in, 2, v1) };
1574       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1575       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1576       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1577       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1578       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1579       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1580       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1581       break;
1582    }
1583    case nir_op_cube_face_index: {
1584       Temp in = get_alu_src(ctx, instr->src[0], 3);
1585       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1586                       emit_extract_vector(ctx, in, 1, v1),
1587                       emit_extract_vector(ctx, in, 2, v1) };
1588       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1589       break;
1590    }
1591    case nir_op_bcsel: {
1592       emit_bcsel(ctx, instr, dst);
1593       break;
1594    }
1595    case nir_op_frsq: {
1596       if (dst.size() == 1) {
1597          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1598       } else if (dst.size() == 2) {
1599          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1600       } else {
1601          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1602          nir_print_instr(&instr->instr, stderr);
1603          fprintf(stderr, "\n");
1604       }
1605       break;
1606    }
1607    case nir_op_fneg: {
1608       Temp src = get_alu_src(ctx, instr->src[0]);
1609       if (dst.size() == 1) {
1610          if (ctx->block->fp_mode.must_flush_denorms32)
1611             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1612          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1613       } else if (dst.size() == 2) {
1614          if (ctx->block->fp_mode.must_flush_denorms16_64)
1615             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1616          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1617          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1618          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1619          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1620       } else {
1621          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1622          nir_print_instr(&instr->instr, stderr);
1623          fprintf(stderr, "\n");
1624       }
1625       break;
1626    }
1627    case nir_op_fabs: {
1628       Temp src = get_alu_src(ctx, instr->src[0]);
1629       if (dst.size() == 1) {
1630          if (ctx->block->fp_mode.must_flush_denorms32)
1631             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1632          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1633       } else if (dst.size() == 2) {
1634          if (ctx->block->fp_mode.must_flush_denorms16_64)
1635             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1636          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1637          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1638          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1640       } else {
1641          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1642          nir_print_instr(&instr->instr, stderr);
1643          fprintf(stderr, "\n");
1644       }
1645       break;
1646    }
1647    case nir_op_fsat: {
1648       Temp src = get_alu_src(ctx, instr->src[0]);
1649       if (dst.size() == 1) {
1650          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1651          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1652          // TODO: confirm that this holds under any circumstances
1653       } else if (dst.size() == 2) {
1654          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1655          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1656          vop3->clamp = true;
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_flog2: {
1665       if (dst.size() == 1) {
1666          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_frcp: {
1675       if (dst.size() == 1) {
1676          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1677       } else if (dst.size() == 2) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1679       } else {
1680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1681          nir_print_instr(&instr->instr, stderr);
1682          fprintf(stderr, "\n");
1683       }
1684       break;
1685    }
1686    case nir_op_fexp2: {
1687       if (dst.size() == 1) {
1688          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1689       } else {
1690          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1691          nir_print_instr(&instr->instr, stderr);
1692          fprintf(stderr, "\n");
1693       }
1694       break;
1695    }
1696    case nir_op_fsqrt: {
1697       if (dst.size() == 1) {
1698          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1699       } else if (dst.size() == 2) {
1700          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1701       } else {
1702          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1703          nir_print_instr(&instr->instr, stderr);
1704          fprintf(stderr, "\n");
1705       }
1706       break;
1707    }
1708    case nir_op_ffract: {
1709       if (dst.size() == 1) {
1710          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1711       } else if (dst.size() == 2) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1713       } else {
1714          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1715          nir_print_instr(&instr->instr, stderr);
1716          fprintf(stderr, "\n");
1717       }
1718       break;
1719    }
1720    case nir_op_ffloor: {
1721       if (dst.size() == 1) {
1722          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1723       } else if (dst.size() == 2) {
1724          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1725       } else {
1726          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1727          nir_print_instr(&instr->instr, stderr);
1728          fprintf(stderr, "\n");
1729       }
1730       break;
1731    }
1732    case nir_op_fceil: {
1733       if (dst.size() == 1) {
1734          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1735       } else if (dst.size() == 2) {
1736          if (ctx->options->chip_class >= GFX7) {
1737             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1738          } else {
1739             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1740             Temp src0 = get_alu_src(ctx, instr->src[0]);
1741
1742             /* trunc = trunc(src0)
1743              * if (src0 > 0.0 && src0 != trunc)
1744              *    trunc += 1.0
1745              */
1746             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1747             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1748             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1749             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1750             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1751             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1752             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1753          }
1754       } else {
1755          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1756          nir_print_instr(&instr->instr, stderr);
1757          fprintf(stderr, "\n");
1758       }
1759       break;
1760    }
1761    case nir_op_ftrunc: {
1762       if (dst.size() == 1) {
1763          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1764       } else if (dst.size() == 2) {
1765          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1766       } else {
1767          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1768          nir_print_instr(&instr->instr, stderr);
1769          fprintf(stderr, "\n");
1770       }
1771       break;
1772    }
1773    case nir_op_fround_even: {
1774       if (dst.size() == 1) {
1775          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1776       } else if (dst.size() == 2) {
1777          if (ctx->options->chip_class >= GFX7) {
1778             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1779          } else {
1780             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1781             Temp src0 = get_alu_src(ctx, instr->src[0]);
1782
1783             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1784             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1785
1786             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1787             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1788             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1789             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1790             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1791             tmp = sub->definitions[0].getTemp();
1792
1793             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1794             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1795             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1796             Temp cond = vop3->definitions[0].getTemp();
1797
1798             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1799             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1800             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1801             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1802
1803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1804          }
1805       } else {
1806          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1807          nir_print_instr(&instr->instr, stderr);
1808          fprintf(stderr, "\n");
1809       }
1810       break;
1811    }
1812    case nir_op_fsin:
1813    case nir_op_fcos: {
1814       Temp src = get_alu_src(ctx, instr->src[0]);
1815       aco_ptr<Instruction> norm;
1816       if (dst.size() == 1) {
1817          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1818          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1819
1820          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1821          if (ctx->options->chip_class < GFX9)
1822             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1823
1824          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1825          bld.vop1(opcode, Definition(dst), tmp);
1826       } else {
1827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1828          nir_print_instr(&instr->instr, stderr);
1829          fprintf(stderr, "\n");
1830       }
1831       break;
1832    }
1833    case nir_op_ldexp: {
1834       if (dst.size() == 1) {
1835          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1836                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1837                   get_alu_src(ctx, instr->src[1]));
1838       } else if (dst.size() == 2) {
1839          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1840                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1841                   get_alu_src(ctx, instr->src[1]));
1842       } else {
1843          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1844          nir_print_instr(&instr->instr, stderr);
1845          fprintf(stderr, "\n");
1846       }
1847       break;
1848    }
1849    case nir_op_frexp_sig: {
1850       if (dst.size() == 1) {
1851          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1852                   get_alu_src(ctx, instr->src[0]));
1853       } else if (dst.size() == 2) {
1854          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1855                   get_alu_src(ctx, instr->src[0]));
1856       } else {
1857          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1858          nir_print_instr(&instr->instr, stderr);
1859          fprintf(stderr, "\n");
1860       }
1861       break;
1862    }
1863    case nir_op_frexp_exp: {
1864       if (instr->src[0].src.ssa->bit_size == 32) {
1865          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1866                   get_alu_src(ctx, instr->src[0]));
1867       } else if (instr->src[0].src.ssa->bit_size == 64) {
1868          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1869                   get_alu_src(ctx, instr->src[0]));
1870       } else {
1871          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1872          nir_print_instr(&instr->instr, stderr);
1873          fprintf(stderr, "\n");
1874       }
1875       break;
1876    }
1877    case nir_op_fsign: {
1878       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1879       if (dst.size() == 1) {
1880          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1881          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1882          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1884       } else if (dst.size() == 2) {
1885          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1886          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1887          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1888
1889          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1890          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1891          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1892
1893          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1894       } else {
1895          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1896          nir_print_instr(&instr->instr, stderr);
1897          fprintf(stderr, "\n");
1898       }
1899       break;
1900    }
1901    case nir_op_f2f32: {
1902       if (instr->src[0].src.ssa->bit_size == 64) {
1903          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_f2f64: {
1912       if (instr->src[0].src.ssa->bit_size == 32) {
1913          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1914       } else {
1915          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1916          nir_print_instr(&instr->instr, stderr);
1917          fprintf(stderr, "\n");
1918       }
1919       break;
1920    }
1921    case nir_op_i2f32: {
1922       assert(dst.size() == 1);
1923       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1924       break;
1925    }
1926    case nir_op_i2f64: {
1927       if (instr->src[0].src.ssa->bit_size == 32) {
1928          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1929       } else if (instr->src[0].src.ssa->bit_size == 64) {
1930          Temp src = get_alu_src(ctx, instr->src[0]);
1931          RegClass rc = RegClass(src.type(), 1);
1932          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1933          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1934          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1935          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1936          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1937          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1938
1939       } else {
1940          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1941          nir_print_instr(&instr->instr, stderr);
1942          fprintf(stderr, "\n");
1943       }
1944       break;
1945    }
1946    case nir_op_u2f32: {
1947       assert(dst.size() == 1);
1948       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1949       break;
1950    }
1951    case nir_op_u2f64: {
1952       if (instr->src[0].src.ssa->bit_size == 32) {
1953          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1954       } else if (instr->src[0].src.ssa->bit_size == 64) {
1955          Temp src = get_alu_src(ctx, instr->src[0]);
1956          RegClass rc = RegClass(src.type(), 1);
1957          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1958          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1959          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1960          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1961          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1962          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_f2i32: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32) {
1973          if (dst.type() == RegType::vgpr)
1974             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1975          else
1976             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1977                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1978
1979       } else if (instr->src[0].src.ssa->bit_size == 64) {
1980          if (dst.type() == RegType::vgpr)
1981             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1982          else
1983             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1984                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1985
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_f2u32: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       if (instr->src[0].src.ssa->bit_size == 32) {
1996          if (dst.type() == RegType::vgpr)
1997             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1998          else
1999             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2000                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2001
2002       } else if (instr->src[0].src.ssa->bit_size == 64) {
2003          if (dst.type() == RegType::vgpr)
2004             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2005          else
2006             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2007                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2008
2009       } else {
2010          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2011          nir_print_instr(&instr->instr, stderr);
2012          fprintf(stderr, "\n");
2013       }
2014       break;
2015    }
2016    case nir_op_f2i64: {
2017       Temp src = get_alu_src(ctx, instr->src[0]);
2018       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2019          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2020          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2021          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2022          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2023          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2024          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2025          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2026          Temp new_exponent = bld.tmp(v1);
2027          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2028          if (ctx->program->chip_class >= GFX8)
2029             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2030          else
2031             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2032          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2033          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2034          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2035          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2036          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2037          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2038          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2039          Temp new_lower = bld.tmp(v1);
2040          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2041          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2042          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2043
2044       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2045          if (src.type() == RegType::vgpr)
2046             src = bld.as_uniform(src);
2047          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2048          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2049          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2050          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2051          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2052          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2053          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2054          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2055          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2056          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2057          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2058          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2059          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2060          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2061          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2062          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2063          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2064          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2065          Temp borrow = bld.tmp(s1);
2066          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2067          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2068          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2069
2070       } else if (instr->src[0].src.ssa->bit_size == 64) {
2071          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2072          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2073          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2074          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2075          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2076          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2077          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2078          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2079          if (dst.type() == RegType::sgpr) {
2080             lower = bld.as_uniform(lower);
2081             upper = bld.as_uniform(upper);
2082          }
2083          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2084
2085       } else {
2086          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2087          nir_print_instr(&instr->instr, stderr);
2088          fprintf(stderr, "\n");
2089       }
2090       break;
2091    }
2092    case nir_op_f2u64: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2095          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2096          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2097          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2098          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2099          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2100          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2101          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2102          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2103          Temp new_exponent = bld.tmp(v1);
2104          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2105          if (ctx->program->chip_class >= GFX8)
2106             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2107          else
2108             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2109          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2110          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2111          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2112          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2114          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2116
2117       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2118          if (src.type() == RegType::vgpr)
2119             src = bld.as_uniform(src);
2120          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2121          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2122          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2123          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2124          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2125          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2126          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2127          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2128          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2129          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2130          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2131          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2132          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2133          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2134          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2135          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2136          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2137          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2138
2139       } else if (instr->src[0].src.ssa->bit_size == 64) {
2140          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2141          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2142          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2143          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2144          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2145          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2146          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2147          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2148          if (dst.type() == RegType::sgpr) {
2149             lower = bld.as_uniform(lower);
2150             upper = bld.as_uniform(upper);
2151          }
2152          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2153
2154       } else {
2155          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2156          nir_print_instr(&instr->instr, stderr);
2157          fprintf(stderr, "\n");
2158       }
2159       break;
2160    }
2161    case nir_op_b2f32: {
2162       Temp src = get_alu_src(ctx, instr->src[0]);
2163       assert(src.regClass() == bld.lm);
2164
2165       if (dst.regClass() == s1) {
2166          src = bool_to_scalar_condition(ctx, src);
2167          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2168       } else if (dst.regClass() == v1) {
2169          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2170       } else {
2171          unreachable("Wrong destination register class for nir_op_b2f32.");
2172       }
2173       break;
2174    }
2175    case nir_op_b2f64: {
2176       Temp src = get_alu_src(ctx, instr->src[0]);
2177       assert(src.regClass() == bld.lm);
2178
2179       if (dst.regClass() == s2) {
2180          src = bool_to_scalar_condition(ctx, src);
2181          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2182       } else if (dst.regClass() == v2) {
2183          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2184          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2185          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2186       } else {
2187          unreachable("Wrong destination register class for nir_op_b2f64.");
2188       }
2189       break;
2190    }
2191    case nir_op_i2i32: {
2192       Temp src = get_alu_src(ctx, instr->src[0]);
2193       if (instr->src[0].src.ssa->bit_size == 64) {
2194          /* we can actually just say dst = src, as it would map the lower register */
2195          emit_extract_vector(ctx, src, 0, dst);
2196       } else {
2197          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2198          nir_print_instr(&instr->instr, stderr);
2199          fprintf(stderr, "\n");
2200       }
2201       break;
2202    }
2203    case nir_op_u2u32: {
2204       Temp src = get_alu_src(ctx, instr->src[0]);
2205       if (instr->src[0].src.ssa->bit_size == 16) {
2206          if (dst.regClass() == s1) {
2207             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2208          } else {
2209             // TODO: do better with SDWA
2210             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2211          }
2212       } else if (instr->src[0].src.ssa->bit_size == 64) {
2213          /* we can actually just say dst = src, as it would map the lower register */
2214          emit_extract_vector(ctx, src, 0, dst);
2215       } else {
2216          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2217          nir_print_instr(&instr->instr, stderr);
2218          fprintf(stderr, "\n");
2219       }
2220       break;
2221    }
2222    case nir_op_i2i64: {
2223       Temp src = get_alu_src(ctx, instr->src[0]);
2224       if (src.regClass() == s1) {
2225          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2226          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2227       } else if (src.regClass() == v1) {
2228          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2230       } else {
2231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2232          nir_print_instr(&instr->instr, stderr);
2233          fprintf(stderr, "\n");
2234       }
2235       break;
2236    }
2237    case nir_op_u2u64: {
2238       Temp src = get_alu_src(ctx, instr->src[0]);
2239       if (instr->src[0].src.ssa->bit_size == 32) {
2240          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2241       } else {
2242          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2243          nir_print_instr(&instr->instr, stderr);
2244          fprintf(stderr, "\n");
2245       }
2246       break;
2247    }
2248    case nir_op_b2i32: {
2249       Temp src = get_alu_src(ctx, instr->src[0]);
2250       assert(src.regClass() == bld.lm);
2251
2252       if (dst.regClass() == s1) {
2253          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2254          bool_to_scalar_condition(ctx, src, dst);
2255       } else if (dst.regClass() == v1) {
2256          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2257       } else {
2258          unreachable("Invalid register class for b2i32");
2259       }
2260       break;
2261    }
2262    case nir_op_i2b1: {
2263       Temp src = get_alu_src(ctx, instr->src[0]);
2264       assert(dst.regClass() == bld.lm);
2265
2266       if (src.type() == RegType::vgpr) {
2267          assert(src.regClass() == v1 || src.regClass() == v2);
2268          assert(dst.regClass() == bld.lm);
2269          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2270                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2271       } else {
2272          assert(src.regClass() == s1 || src.regClass() == s2);
2273          Temp tmp;
2274          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2275             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2276          } else {
2277             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2278                            bld.scc(bld.def(s1)), Operand(0u), src);
2279          }
2280          bool_to_vector_condition(ctx, tmp, dst);
2281       }
2282       break;
2283    }
2284    case nir_op_pack_64_2x32_split: {
2285       Temp src0 = get_alu_src(ctx, instr->src[0]);
2286       Temp src1 = get_alu_src(ctx, instr->src[1]);
2287
2288       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2289       break;
2290    }
2291    case nir_op_unpack_64_2x32_split_x:
2292       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2293       break;
2294    case nir_op_unpack_64_2x32_split_y:
2295       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2296       break;
2297    case nir_op_pack_half_2x16: {
2298       Temp src = get_alu_src(ctx, instr->src[0], 2);
2299
2300       if (dst.regClass() == v1) {
2301          Temp src0 = bld.tmp(v1);
2302          Temp src1 = bld.tmp(v1);
2303          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2304          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2305             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2306          else
2307             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2308                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2309                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2310       } else {
2311          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2312          nir_print_instr(&instr->instr, stderr);
2313          fprintf(stderr, "\n");
2314       }
2315       break;
2316    }
2317    case nir_op_unpack_half_2x16_split_x: {
2318       if (dst.regClass() == v1) {
2319          Builder bld(ctx->program, ctx->block);
2320          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_unpack_half_2x16_split_y: {
2329       if (dst.regClass() == v1) {
2330          Builder bld(ctx->program, ctx->block);
2331          /* TODO: use SDWA here */
2332          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2333                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2334       } else {
2335          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2336          nir_print_instr(&instr->instr, stderr);
2337          fprintf(stderr, "\n");
2338       }
2339       break;
2340    }
2341    case nir_op_fquantize2f16: {
2342       Temp src = get_alu_src(ctx, instr->src[0]);
2343       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2344       Temp f32, cmp_res;
2345
2346       if (ctx->program->chip_class >= GFX8) {
2347          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2348          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2349          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2350       } else {
2351          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2352           * so compare the result and flush to 0 if it's smaller.
2353           */
2354          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2355          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2356          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2357          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2358          cmp_res = vop3->definitions[0].getTemp();
2359       }
2360
2361       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2362          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2363          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2364       } else {
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2366       }
2367       break;
2368    }
2369    case nir_op_bfm: {
2370       Temp bits = get_alu_src(ctx, instr->src[0]);
2371       Temp offset = get_alu_src(ctx, instr->src[1]);
2372
2373       if (dst.regClass() == s1) {
2374          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2375       } else if (dst.regClass() == v1) {
2376          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2377       } else {
2378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2379          nir_print_instr(&instr->instr, stderr);
2380          fprintf(stderr, "\n");
2381       }
2382       break;
2383    }
2384    case nir_op_bitfield_select: {
2385       /* (mask & insert) | (~mask & base) */
2386       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2387       Temp insert = get_alu_src(ctx, instr->src[1]);
2388       Temp base = get_alu_src(ctx, instr->src[2]);
2389
2390       /* dst = (insert & bitmask) | (base & ~bitmask) */
2391       if (dst.regClass() == s1) {
2392          aco_ptr<Instruction> sop2;
2393          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2394          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2395          Operand lhs;
2396          if (const_insert && const_bitmask) {
2397             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2398          } else {
2399             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2400             lhs = Operand(insert);
2401          }
2402
2403          Operand rhs;
2404          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2405          if (const_base && const_bitmask) {
2406             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2407          } else {
2408             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2409             rhs = Operand(base);
2410          }
2411
2412          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2413
2414       } else if (dst.regClass() == v1) {
2415          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2416             base = as_vgpr(ctx, base);
2417          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2418             insert = as_vgpr(ctx, insert);
2419
2420          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2421
2422       } else {
2423          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2424          nir_print_instr(&instr->instr, stderr);
2425          fprintf(stderr, "\n");
2426       }
2427       break;
2428    }
2429    case nir_op_ubfe:
2430    case nir_op_ibfe: {
2431       Temp base = get_alu_src(ctx, instr->src[0]);
2432       Temp offset = get_alu_src(ctx, instr->src[1]);
2433       Temp bits = get_alu_src(ctx, instr->src[2]);
2434
2435       if (dst.type() == RegType::sgpr) {
2436          Operand extract;
2437          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2438          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2439          if (const_offset && const_bits) {
2440             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2441             extract = Operand(const_extract);
2442          } else {
2443             Operand width;
2444             if (const_bits) {
2445                width = Operand(const_bits->u32 << 16);
2446             } else {
2447                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2448             }
2449             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2450          }
2451
2452          aco_opcode opcode;
2453          if (dst.regClass() == s1) {
2454             if (instr->op == nir_op_ubfe)
2455                opcode = aco_opcode::s_bfe_u32;
2456             else
2457                opcode = aco_opcode::s_bfe_i32;
2458          } else if (dst.regClass() == s2) {
2459             if (instr->op == nir_op_ubfe)
2460                opcode = aco_opcode::s_bfe_u64;
2461             else
2462                opcode = aco_opcode::s_bfe_i64;
2463          } else {
2464             unreachable("Unsupported BFE bit size");
2465          }
2466
2467          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2468
2469       } else {
2470          aco_opcode opcode;
2471          if (dst.regClass() == v1) {
2472             if (instr->op == nir_op_ubfe)
2473                opcode = aco_opcode::v_bfe_u32;
2474             else
2475                opcode = aco_opcode::v_bfe_i32;
2476          } else {
2477             unreachable("Unsupported BFE bit size");
2478          }
2479
2480          emit_vop3a_instruction(ctx, instr, opcode, dst);
2481       }
2482       break;
2483    }
2484    case nir_op_bit_count: {
2485       Temp src = get_alu_src(ctx, instr->src[0]);
2486       if (src.regClass() == s1) {
2487          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2488       } else if (src.regClass() == v1) {
2489          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2490       } else if (src.regClass() == v2) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2492                   emit_extract_vector(ctx, src, 1, v1),
2493                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2494                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2495       } else if (src.regClass() == s2) {
2496          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2497       } else {
2498          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2499          nir_print_instr(&instr->instr, stderr);
2500          fprintf(stderr, "\n");
2501       }
2502       break;
2503    }
2504    case nir_op_flt: {
2505       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2506       break;
2507    }
2508    case nir_op_fge: {
2509       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2510       break;
2511    }
2512    case nir_op_feq: {
2513       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2514       break;
2515    }
2516    case nir_op_fne: {
2517       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2518       break;
2519    }
2520    case nir_op_ilt: {
2521       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2522       break;
2523    }
2524    case nir_op_ige: {
2525       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2526       break;
2527    }
2528    case nir_op_ieq: {
2529       if (instr->src[0].src.ssa->bit_size == 1)
2530          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2531       else
2532          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2533                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2534       break;
2535    }
2536    case nir_op_ine: {
2537       if (instr->src[0].src.ssa->bit_size == 1)
2538          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2539       else
2540          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2541                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2542       break;
2543    }
2544    case nir_op_ult: {
2545       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2546       break;
2547    }
2548    case nir_op_uge: {
2549       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2550       break;
2551    }
2552    case nir_op_fddx:
2553    case nir_op_fddy:
2554    case nir_op_fddx_fine:
2555    case nir_op_fddy_fine:
2556    case nir_op_fddx_coarse:
2557    case nir_op_fddy_coarse: {
2558       Temp src = get_alu_src(ctx, instr->src[0]);
2559       uint16_t dpp_ctrl1, dpp_ctrl2;
2560       if (instr->op == nir_op_fddx_fine) {
2561          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2562          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2563       } else if (instr->op == nir_op_fddy_fine) {
2564          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2565          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2566       } else {
2567          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2568          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2569             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2570          else
2571             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2572       }
2573
2574       Temp tmp;
2575       if (ctx->program->chip_class >= GFX8) {
2576          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2577          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2578       } else {
2579          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2580          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2581          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2582       }
2583       emit_wqm(ctx, tmp, dst, true);
2584       break;
2585    }
2586    default:
2587       fprintf(stderr, "Unknown NIR ALU instr: ");
2588       nir_print_instr(&instr->instr, stderr);
2589       fprintf(stderr, "\n");
2590    }
2591 }
2592
2593 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2594 {
2595    Temp dst = get_ssa_temp(ctx, &instr->def);
2596
2597    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2598    // which get truncated the lsb if double and msb if int
2599    // for now, we only use s_mov_b64 with 64bit inline constants
2600    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2601    assert(dst.type() == RegType::sgpr);
2602
2603    Builder bld(ctx->program, ctx->block);
2604
2605    if (instr->def.bit_size == 1) {
2606       assert(dst.regClass() == bld.lm);
2607       int val = instr->value[0].b ? -1 : 0;
2608       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2609       bld.sop1(Builder::s_mov, Definition(dst), op);
2610    } else if (dst.size() == 1) {
2611       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2612    } else {
2613       assert(dst.size() != 1);
2614       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2615       if (instr->def.bit_size == 64)
2616          for (unsigned i = 0; i < dst.size(); i++)
2617             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2618       else {
2619          for (unsigned i = 0; i < dst.size(); i++)
2620             vec->operands[i] = Operand{instr->value[i].u32};
2621       }
2622       vec->definitions[0] = Definition(dst);
2623       ctx->block->instructions.emplace_back(std::move(vec));
2624    }
2625 }
2626
2627 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2628 {
2629    uint32_t new_mask = 0;
2630    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2631       if (mask & (1u << i))
2632          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2633    return new_mask;
2634 }
2635
2636 Operand load_lds_size_m0(isel_context *ctx)
2637 {
2638    /* TODO: m0 does not need to be initialized on GFX9+ */
2639    Builder bld(ctx->program, ctx->block);
2640    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2641 }
2642
2643 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2644               Temp address, unsigned base_offset, unsigned align)
2645 {
2646    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2647
2648    Builder bld(ctx->program, ctx->block);
2649
2650    Operand m = load_lds_size_m0(ctx);
2651
2652    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2653    unsigned bytes_read = 0;
2654    unsigned result_size = 0;
2655    unsigned total_bytes = num_components * elem_size_bytes;
2656    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2657    bool large_ds_read = ctx->options->chip_class >= GFX7;
2658
2659    while (bytes_read < total_bytes) {
2660       unsigned todo = total_bytes - bytes_read;
2661       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2662       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2663
2664       aco_opcode op = aco_opcode::last_opcode;
2665       bool read2 = false;
2666       if (todo >= 16 && aligned16 && large_ds_read) {
2667          op = aco_opcode::ds_read_b128;
2668          todo = 16;
2669       } else if (todo >= 16 && aligned8) {
2670          op = aco_opcode::ds_read2_b64;
2671          read2 = true;
2672          todo = 16;
2673       } else if (todo >= 12 && aligned16 && large_ds_read) {
2674          op = aco_opcode::ds_read_b96;
2675          todo = 12;
2676       } else if (todo >= 8 && aligned8) {
2677          op = aco_opcode::ds_read_b64;
2678          todo = 8;
2679       } else if (todo >= 8) {
2680          op = aco_opcode::ds_read2_b32;
2681          read2 = true;
2682          todo = 8;
2683       } else if (todo >= 4) {
2684          op = aco_opcode::ds_read_b32;
2685          todo = 4;
2686       } else {
2687          assert(false);
2688       }
2689       assert(todo % elem_size_bytes == 0);
2690       unsigned num_elements = todo / elem_size_bytes;
2691       unsigned offset = base_offset + bytes_read;
2692       unsigned max_offset = read2 ? 1019 : 65535;
2693
2694       Temp address_offset = address;
2695       if (offset > max_offset) {
2696          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2697          offset = bytes_read;
2698       }
2699       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2700
2701       Temp res;
2702       if (num_components == 1 && dst.type() == RegType::vgpr)
2703          res = dst;
2704       else
2705          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2706
2707       if (read2)
2708          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2709       else
2710          res = bld.ds(op, Definition(res), address_offset, m, offset);
2711
2712       if (num_components == 1) {
2713          assert(todo == total_bytes);
2714          if (dst.type() == RegType::sgpr)
2715             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2716          return;
2717       }
2718
2719       if (dst.type() == RegType::sgpr) {
2720          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2721          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2722          res = new_res;
2723       }
2724
2725       if (num_elements == 1) {
2726          result[result_size++] = res;
2727       } else {
2728          assert(res != dst && res.size() % num_elements == 0);
2729          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2730          split->operands[0] = Operand(res);
2731          for (unsigned i = 0; i < num_elements; i++)
2732             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2733          ctx->block->instructions.emplace_back(std::move(split));
2734       }
2735
2736       bytes_read += todo;
2737    }
2738
2739    assert(result_size == num_components && result_size > 1);
2740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2741    for (unsigned i = 0; i < result_size; i++)
2742       vec->operands[i] = Operand(result[i]);
2743    vec->definitions[0] = Definition(dst);
2744    ctx->block->instructions.emplace_back(std::move(vec));
2745    ctx->allocated_vec.emplace(dst.id(), result);
2746 }
2747
2748 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2749 {
2750    if (start == 0 && size == data.size())
2751       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2752
2753    unsigned size_hint = 1;
2754    auto it = ctx->allocated_vec.find(data.id());
2755    if (it != ctx->allocated_vec.end())
2756       size_hint = it->second[0].size();
2757    if (size % size_hint || start % size_hint)
2758       size_hint = 1;
2759
2760    start /= size_hint;
2761    size /= size_hint;
2762
2763    Temp elems[size];
2764    for (unsigned i = 0; i < size; i++)
2765       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2766
2767    if (size == 1)
2768       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2769
2770    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2771    for (unsigned i = 0; i < size; i++)
2772       vec->operands[i] = Operand(elems[i]);
2773    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2774    vec->definitions[0] = Definition(res);
2775    ctx->block->instructions.emplace_back(std::move(vec));
2776    return res;
2777 }
2778
2779 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2780 {
2781    Builder bld(ctx->program, ctx->block);
2782    unsigned bytes_written = 0;
2783    bool large_ds_write = ctx->options->chip_class >= GFX7;
2784
2785    while (bytes_written < total_size * 4) {
2786       unsigned todo = total_size * 4 - bytes_written;
2787       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2788       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2789
2790       aco_opcode op = aco_opcode::last_opcode;
2791       bool write2 = false;
2792       unsigned size = 0;
2793       if (todo >= 16 && aligned16 && large_ds_write) {
2794          op = aco_opcode::ds_write_b128;
2795          size = 4;
2796       } else if (todo >= 16 && aligned8) {
2797          op = aco_opcode::ds_write2_b64;
2798          write2 = true;
2799          size = 4;
2800       } else if (todo >= 12 && aligned16 && large_ds_write) {
2801          op = aco_opcode::ds_write_b96;
2802          size = 3;
2803       } else if (todo >= 8 && aligned8) {
2804          op = aco_opcode::ds_write_b64;
2805          size = 2;
2806       } else if (todo >= 8) {
2807          op = aco_opcode::ds_write2_b32;
2808          write2 = true;
2809          size = 2;
2810       } else if (todo >= 4) {
2811          op = aco_opcode::ds_write_b32;
2812          size = 1;
2813       } else {
2814          assert(false);
2815       }
2816
2817       unsigned offset = offset0 + offset1 + bytes_written;
2818       unsigned max_offset = write2 ? 1020 : 65535;
2819       Temp address_offset = address;
2820       if (offset > max_offset) {
2821          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2822          offset = offset1 + bytes_written;
2823       }
2824       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2825
2826       if (write2) {
2827          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2828          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2829          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2830       } else {
2831          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2832          bld.ds(op, address_offset, val, m, offset);
2833       }
2834
2835       bytes_written += size * 4;
2836    }
2837 }
2838
2839 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2840                Temp address, unsigned base_offset, unsigned align)
2841 {
2842    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2843
2844    Operand m = load_lds_size_m0(ctx);
2845
2846    /* we need at most two stores for 32bit variables */
2847    int start[2], count[2];
2848    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2849    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2850    assert(wrmask == 0);
2851
2852    /* one combined store is sufficient */
2853    if (count[0] == count[1]) {
2854       Builder bld(ctx->program, ctx->block);
2855
2856       Temp address_offset = address;
2857       if ((base_offset >> 2) + start[1] > 255) {
2858          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2859          base_offset = 0;
2860       }
2861
2862       assert(count[0] == 1);
2863       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2864       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2865       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2866       base_offset = base_offset / elem_size_bytes;
2867       bld.ds(op, address_offset, val0, val1, m,
2868              base_offset + start[0], base_offset + start[1]);
2869       return;
2870    }
2871
2872    for (unsigned i = 0; i < 2; i++) {
2873       if (count[i] == 0)
2874          continue;
2875
2876       unsigned elem_size_words = elem_size_bytes / 4;
2877       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2878                       base_offset, start[i] * elem_size_bytes, align);
2879    }
2880    return;
2881 }
2882
2883 void visit_store_vsgs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2884 {
2885    unsigned write_mask = nir_intrinsic_write_mask(instr);
2886    unsigned component = nir_intrinsic_component(instr);
2887    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2888    unsigned idx = (nir_intrinsic_base(instr) + component) * 4u;
2889    Operand offset(s1);
2890    Builder bld(ctx->program, ctx->block);
2891
2892    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2893    if (off_instr->type != nir_instr_type_load_const)
2894       offset = bld.v_mul24_imm(bld.def(v1), get_ssa_temp(ctx, instr->src[1].ssa), 16u);
2895    else
2896       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 16u;
2897
2898    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
2899    if (ctx->stage == vertex_es) {
2900       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
2901
2902       Temp elems[NIR_MAX_VEC_COMPONENTS * 2];
2903       if (elem_size_bytes == 8) {
2904          for (unsigned i = 0; i < src.size() / 2; i++) {
2905             Temp elem = emit_extract_vector(ctx, src, i, v2);
2906             elems[i*2] = bld.tmp(v1);
2907             elems[i*2+1] = bld.tmp(v1);
2908             bld.pseudo(aco_opcode::p_split_vector, Definition(elems[i*2]), Definition(elems[i*2+1]), elem);
2909          }
2910          write_mask = widen_mask(write_mask, 2);
2911          elem_size_bytes /= 2u;
2912       } else {
2913          for (unsigned i = 0; i < src.size(); i++)
2914             elems[i] = emit_extract_vector(ctx, src, i, v1);
2915       }
2916
2917       while (write_mask) {
2918          unsigned index = u_bit_scan(&write_mask);
2919          unsigned offset = index * elem_size_bytes;
2920          Temp elem = emit_extract_vector(ctx, src, index, RegClass(RegType::vgpr, elem_size_bytes / 4));
2921
2922          Operand vaddr_offset(v1);
2923          unsigned const_offset = idx + offset;
2924          if (const_offset >= 4096u) {
2925             vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
2926             const_offset %= 4096u;
2927          }
2928
2929          aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
2930          mtbuf->operands[0] = vaddr_offset;
2931          mtbuf->operands[1] = Operand(esgs_ring);
2932          mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->es2gs_offset));
2933          mtbuf->operands[3] = Operand(elem);
2934          mtbuf->offen = !vaddr_offset.isUndefined();
2935          mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
2936          mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
2937          mtbuf->offset = const_offset;
2938          mtbuf->glc = true;
2939          mtbuf->slc = true;
2940          mtbuf->barrier = barrier_none;
2941          mtbuf->can_reorder = true;
2942          bld.insert(std::move(mtbuf));
2943       }
2944    } else {
2945       unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
2946
2947       Temp vertex_idx = emit_mbcnt(ctx, bld.def(v1));
2948       Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
2949       vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), vertex_idx,
2950                             bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
2951
2952       Temp lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
2953       if (!offset.isUndefined())
2954          lds_base = bld.vadd32(bld.def(v1), offset, lds_base);
2955
2956       unsigned align = 1 << (ffs(itemsize) - 1);
2957       if (idx)
2958          align = std::min(align, 1u << (ffs(idx) - 1));
2959
2960       store_lds(ctx, elem_size_bytes, src, write_mask, lds_base, idx, align);
2961    }
2962 }
2963
2964 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2965 {
2966    if (ctx->stage == vertex_vs ||
2967        ctx->stage == fragment_fs ||
2968        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
2969       unsigned write_mask = nir_intrinsic_write_mask(instr);
2970       unsigned component = nir_intrinsic_component(instr);
2971       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2972       unsigned idx = nir_intrinsic_base(instr) + component;
2973
2974       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2975       if (off_instr->type != nir_instr_type_load_const) {
2976          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2977          nir_print_instr(off_instr, stderr);
2978          fprintf(stderr, "\n");
2979       }
2980       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2981
2982       if (instr->src[0].ssa->bit_size == 64)
2983          write_mask = widen_mask(write_mask, 2);
2984
2985       for (unsigned i = 0; i < 8; ++i) {
2986          if (write_mask & (1 << i)) {
2987             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
2988             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2989          }
2990          idx++;
2991       }
2992    } else if (ctx->stage == vertex_es ||
2993               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
2994       visit_store_vsgs_output(ctx, instr);
2995    } else {
2996       unreachable("Shader stage not implemented");
2997    }
2998 }
2999
3000 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3001 {
3002    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3003    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3004
3005    Builder bld(ctx->program, ctx->block);
3006    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3007    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3008 }
3009
3010 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3011 {
3012    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3013    for (unsigned i = 0; i < num_components; i++)
3014       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3015    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3016       assert(num_components == 4);
3017       Builder bld(ctx->program, ctx->block);
3018       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3019    }
3020
3021    for (Operand& op : vec->operands)
3022       op = op.isUndefined() ? Operand(0u) : op;
3023
3024    vec->definitions[0] = Definition(dst);
3025    ctx->block->instructions.emplace_back(std::move(vec));
3026    emit_split_vector(ctx, dst, num_components);
3027    return;
3028 }
3029
3030 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3031 {
3032    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3033    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3034    unsigned idx = nir_intrinsic_base(instr);
3035    unsigned component = nir_intrinsic_component(instr);
3036    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3037
3038    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3039    if (offset) {
3040       assert(offset->u32 == 0);
3041    } else {
3042       /* the lower 15bit of the prim_mask contain the offset into LDS
3043        * while the upper bits contain the number of prims */
3044       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3045       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3046       Builder bld(ctx->program, ctx->block);
3047       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3048       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3049       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3050       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3051       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3052    }
3053
3054    if (instr->dest.ssa.num_components == 1) {
3055       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3056    } else {
3057       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3058       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3059       {
3060          Temp tmp = {ctx->program->allocateId(), v1};
3061          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3062          vec->operands[i] = Operand(tmp);
3063       }
3064       vec->definitions[0] = Definition(dst);
3065       ctx->block->instructions.emplace_back(std::move(vec));
3066    }
3067 }
3068
3069 bool check_vertex_fetch_size(isel_context *ctx, const ac_data_format_info *vtx_info,
3070                              unsigned offset, unsigned stride, unsigned channels)
3071 {
3072    unsigned vertex_byte_size = vtx_info->chan_byte_size * channels;
3073    if (vtx_info->chan_byte_size != 4 && channels == 3)
3074       return false;
3075    return (ctx->options->chip_class != GFX6 && ctx->options->chip_class != GFX10) ||
3076           (offset % vertex_byte_size == 0 && stride % vertex_byte_size == 0);
3077 }
3078
3079 uint8_t get_fetch_data_format(isel_context *ctx, const ac_data_format_info *vtx_info,
3080                               unsigned offset, unsigned stride, unsigned *channels)
3081 {
3082    if (!vtx_info->chan_byte_size) {
3083       *channels = vtx_info->num_channels;
3084       return vtx_info->chan_format;
3085    }
3086
3087    unsigned num_channels = *channels;
3088    if (!check_vertex_fetch_size(ctx, vtx_info, offset, stride, *channels)) {
3089       unsigned new_channels = num_channels + 1;
3090       /* first, assume more loads is worse and try using a larger data format */
3091       while (new_channels <= 4 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels)) {
3092          new_channels++;
3093          /* don't make the attribute potentially out-of-bounds */
3094          if (offset + new_channels * vtx_info->chan_byte_size > stride)
3095             new_channels = 5;
3096       }
3097
3098       if (new_channels == 5) {
3099          /* then try decreasing load size (at the cost of more loads) */
3100          new_channels = *channels;
3101          while (new_channels > 1 && !check_vertex_fetch_size(ctx, vtx_info, offset, stride, new_channels))
3102             new_channels--;
3103       }
3104
3105       if (new_channels < *channels)
3106          *channels = new_channels;
3107       num_channels = new_channels;
3108    }
3109
3110    switch (vtx_info->chan_format) {
3111    case V_008F0C_BUF_DATA_FORMAT_8:
3112       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_8, V_008F0C_BUF_DATA_FORMAT_8_8,
3113                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_8_8_8_8}[num_channels - 1];
3114    case V_008F0C_BUF_DATA_FORMAT_16:
3115       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_16, V_008F0C_BUF_DATA_FORMAT_16_16,
3116                          V_008F0C_BUF_DATA_FORMAT_INVALID, V_008F0C_BUF_DATA_FORMAT_16_16_16_16}[num_channels - 1];
3117    case V_008F0C_BUF_DATA_FORMAT_32:
3118       return (uint8_t[]){V_008F0C_BUF_DATA_FORMAT_32, V_008F0C_BUF_DATA_FORMAT_32_32,
3119                          V_008F0C_BUF_DATA_FORMAT_32_32_32, V_008F0C_BUF_DATA_FORMAT_32_32_32_32}[num_channels - 1];
3120    }
3121    unreachable("shouldn't reach here");
3122    return V_008F0C_BUF_DATA_FORMAT_INVALID;
3123 }
3124
3125 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3126  * so we may need to fix it up. */
3127 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3128 {
3129    Builder bld(ctx->program, ctx->block);
3130
3131    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3132       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3133
3134    /* For the integer-like cases, do a natural sign extension.
3135     *
3136     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3137     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3138     * exponent.
3139     */
3140    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3141    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3142
3143    /* Convert back to the right type. */
3144    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3145       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3146       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3147       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3148    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3149       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3150    }
3151
3152    return alpha;
3153 }
3154
3155 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3156 {
3157    Builder bld(ctx->program, ctx->block);
3158    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3159    if (ctx->stage & sw_vs) {
3160
3161       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3162       if (off_instr->type != nir_instr_type_load_const) {
3163          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3164          nir_print_instr(off_instr, stderr);
3165          fprintf(stderr, "\n");
3166       }
3167       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3168
3169       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3170
3171       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3172       unsigned component = nir_intrinsic_component(instr);
3173       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3174       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3175       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3176       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3177
3178       unsigned dfmt = attrib_format & 0xf;
3179       unsigned nfmt = (attrib_format >> 4) & 0x7;
3180       const struct ac_data_format_info *vtx_info = ac_get_data_format_info(dfmt);
3181
3182       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3183       unsigned num_channels = MIN2(util_last_bit(mask), vtx_info->num_channels);
3184       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3185       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3186       if (post_shuffle)
3187          num_channels = MAX2(num_channels, 3);
3188
3189       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3190       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3191
3192       Temp index;
3193       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3194          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3195          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3196          if (divisor) {
3197             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3198             if (divisor != 1) {
3199                Temp divided = bld.tmp(v1);
3200                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3201                index = bld.vadd32(bld.def(v1), start_instance, divided);
3202             } else {
3203                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3204             }
3205          } else {
3206             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3207          }
3208       } else {
3209          index = bld.vadd32(bld.def(v1),
3210                             get_arg(ctx, ctx->args->ac.base_vertex),
3211                             get_arg(ctx, ctx->args->ac.vertex_id));
3212       }
3213
3214       Temp channels[num_channels];
3215       unsigned channel_start = 0;
3216       bool direct_fetch = false;
3217
3218       /* skip unused channels at the start */
3219       if (vtx_info->chan_byte_size && !post_shuffle) {
3220          channel_start = ffs(mask) - 1;
3221          for (unsigned i = 0; i < channel_start; i++)
3222             channels[i] = Temp(0, s1);
3223       } else if (vtx_info->chan_byte_size && post_shuffle && !(mask & 0x8)) {
3224          num_channels = 3 - (ffs(mask) - 1);
3225       }
3226
3227       /* load channels */
3228       while (channel_start < num_channels) {
3229          unsigned fetch_size = num_channels - channel_start;
3230          unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size;
3231
3232          /* use MUBUF when possible to avoid possible alignment issues */
3233          /* TODO: we could use SDWA to unpack 8/16-bit attributes without extra instructions */
3234          bool use_mubuf = (nfmt == V_008F0C_BUF_NUM_FORMAT_FLOAT ||
3235                            nfmt == V_008F0C_BUF_NUM_FORMAT_UINT ||
3236                            nfmt == V_008F0C_BUF_NUM_FORMAT_SINT) &&
3237                           vtx_info->chan_byte_size == 4;
3238          unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID;
3239          if (!use_mubuf) {
3240             fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size);
3241          } else {
3242             if (fetch_size == 3 && ctx->options->chip_class == GFX6) {
3243                /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */
3244                fetch_size = 4;
3245             }
3246          }
3247
3248          Temp fetch_index = index;
3249          if (attrib_stride != 0 && fetch_offset > attrib_stride) {
3250             fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index);
3251             fetch_offset = fetch_offset % attrib_stride;
3252          }
3253
3254          Operand soffset(0u);
3255          if (fetch_offset >= 4096) {
3256             soffset = bld.copy(bld.def(s1), Operand(fetch_offset / 4096 * 4096));
3257             fetch_offset %= 4096;
3258          }
3259
3260          aco_opcode opcode;
3261          switch (fetch_size) {
3262          case 1:
3263             opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x;
3264             break;
3265          case 2:
3266             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy;
3267             break;
3268          case 3:
3269             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz;
3270             break;
3271          case 4:
3272             opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw;
3273             break;
3274          default:
3275             unreachable("Unimplemented load_input vector size");
3276          }
3277
3278          Temp fetch_dst;
3279          if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle &&
3280              (alpha_adjust == RADV_ALPHA_ADJUST_NONE || num_channels <= 3)) {
3281             direct_fetch = true;
3282             fetch_dst = dst;
3283          } else {
3284             fetch_dst = bld.tmp(RegType::vgpr, fetch_size);
3285          }
3286
3287          if (use_mubuf) {
3288             Instruction *mubuf = bld.mubuf(opcode,
3289                                            Definition(fetch_dst), fetch_index, list, soffset,
3290                                            fetch_offset, false, true).instr;
3291             static_cast<MUBUF_instruction*>(mubuf)->can_reorder = true;
3292          } else {
3293             Instruction *mtbuf = bld.mtbuf(opcode,
3294                                            Definition(fetch_dst), fetch_index, list, soffset,
3295                                            fetch_dfmt, nfmt, fetch_offset, false, true).instr;
3296             static_cast<MTBUF_instruction*>(mtbuf)->can_reorder = true;
3297          }
3298
3299          emit_split_vector(ctx, fetch_dst, fetch_dst.size());
3300
3301          if (fetch_size == 1) {
3302             channels[channel_start] = fetch_dst;
3303          } else {
3304             for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++)
3305                channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1);
3306          }
3307
3308          channel_start += fetch_size;
3309       }
3310
3311       if (!direct_fetch) {
3312          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3313                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3314
3315          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3316          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3317          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3318
3319          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3320          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3321          unsigned num_temp = 0;
3322          for (unsigned i = 0; i < dst.size(); i++) {
3323             unsigned idx = i + component;
3324             if (swizzle[idx] < num_channels && channels[swizzle[idx]].id()) {
3325                Temp channel = channels[swizzle[idx]];
3326                if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE)
3327                   channel = adjust_vertex_fetch_alpha(ctx, alpha_adjust, channel);
3328                vec->operands[i] = Operand(channel);
3329
3330                num_temp++;
3331                elems[i] = channel;
3332             } else if (is_float && idx == 3) {
3333                vec->operands[i] = Operand(0x3f800000u);
3334             } else if (!is_float && idx == 3) {
3335                vec->operands[i] = Operand(1u);
3336             } else {
3337                vec->operands[i] = Operand(0u);
3338             }
3339          }
3340          vec->definitions[0] = Definition(dst);
3341          ctx->block->instructions.emplace_back(std::move(vec));
3342          emit_split_vector(ctx, dst, dst.size());
3343
3344          if (num_temp == dst.size())
3345             ctx->allocated_vec.emplace(dst.id(), elems);
3346       }
3347    } else if (ctx->stage == fragment_fs) {
3348       unsigned offset_idx = instr->intrinsic == nir_intrinsic_load_input ? 0 : 1;
3349       nir_instr *off_instr = instr->src[offset_idx].ssa->parent_instr;
3350       if (off_instr->type != nir_instr_type_load_const ||
3351           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3352          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3353          nir_print_instr(off_instr, stderr);
3354          fprintf(stderr, "\n");
3355       }
3356
3357       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3358       nir_const_value* offset = nir_src_as_const_value(instr->src[offset_idx]);
3359       if (offset) {
3360          assert(offset->u32 == 0);
3361       } else {
3362          /* the lower 15bit of the prim_mask contain the offset into LDS
3363           * while the upper bits contain the number of prims */
3364          Temp offset_src = get_ssa_temp(ctx, instr->src[offset_idx].ssa);
3365          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3366          Builder bld(ctx->program, ctx->block);
3367          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3368          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3369          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3370          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3371          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3372       }
3373
3374       unsigned idx = nir_intrinsic_base(instr);
3375       unsigned component = nir_intrinsic_component(instr);
3376       unsigned vertex_id = 2; /* P0 */
3377
3378       if (instr->intrinsic == nir_intrinsic_load_input_vertex) {
3379          nir_const_value* src0 = nir_src_as_const_value(instr->src[0]);
3380          switch (src0->u32) {
3381          case 0:
3382             vertex_id = 2; /* P0 */
3383             break;
3384          case 1:
3385             vertex_id = 0; /* P10 */
3386             break;
3387          case 2:
3388             vertex_id = 1; /* P20 */
3389             break;
3390          default:
3391             unreachable("invalid vertex index");
3392          }
3393       }
3394
3395       if (dst.size() == 1) {
3396          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(vertex_id), bld.m0(prim_mask), idx, component);
3397       } else {
3398          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3399          for (unsigned i = 0; i < dst.size(); i++)
3400             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(vertex_id), bld.m0(prim_mask), idx, component + i);
3401          vec->definitions[0] = Definition(dst);
3402          bld.insert(std::move(vec));
3403       }
3404
3405    } else {
3406       unreachable("Shader stage not implemented");
3407    }
3408 }
3409
3410 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3411 {
3412    assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
3413    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3414
3415    Builder bld(ctx->program, ctx->block);
3416    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3417
3418    Temp offset = Temp();
3419    if (instr->src[0].ssa->parent_instr->type != nir_instr_type_load_const) {
3420       /* better code could be created, but this case probably doesn't happen
3421        * much in practice */
3422       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
3423       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3424          Temp elem;
3425          if (ctx->stage == vertex_geometry_gs) {
3426             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3427             if (i % 2u)
3428                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3429          } else {
3430             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3431          }
3432          if (offset.id()) {
3433             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(s2)),
3434                                  Operand(i), indirect_vertex);
3435             offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), offset, elem, cond);
3436          } else {
3437             offset = elem;
3438          }
3439       }
3440       if (ctx->stage == vertex_geometry_gs)
3441          offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), offset);
3442    } else {
3443       unsigned vertex = nir_src_as_uint(instr->src[0]);
3444       if (ctx->stage == vertex_geometry_gs)
3445          offset = bld.vop3(
3446          aco_opcode::v_bfe_u32, bld.def(v1), get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3447             Operand((vertex % 2u) * 16u), Operand(16u));
3448       else
3449          offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3450    }
3451
3452    unsigned const_offset = nir_intrinsic_base(instr);
3453    const_offset += nir_intrinsic_component(instr);
3454
3455    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3456    if (off_instr->type != nir_instr_type_load_const) {
3457       Temp indirect_offset = get_ssa_temp(ctx, instr->src[1].ssa);
3458       offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u),
3459                         bld.vadd32(bld.def(v1), indirect_offset, offset));
3460    } else {
3461       const_offset += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3462    }
3463    const_offset *= 4u;
3464
3465    offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), offset);
3466
3467    unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3468
3469    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3470    if (ctx->stage == geometry_gs) {
3471       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3472
3473       const_offset *= ctx->program->wave_size;
3474
3475       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3476       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3477          aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1)};
3478       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++) {
3479          Temp subelems[2];
3480          for (unsigned j = 0; j < elem_size_bytes / 4; j++) {
3481             Operand soffset(0u);
3482             if (const_offset >= 4096u)
3483                soffset = bld.copy(bld.def(s1), Operand(const_offset / 4096u * 4096u));
3484
3485             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
3486             mubuf->definitions[0] = bld.def(v1);
3487             subelems[j] = mubuf->definitions[0].getTemp();
3488             mubuf->operands[0] = Operand(offset);
3489             mubuf->operands[1] = Operand(esgs_ring);
3490             mubuf->operands[2] = Operand(soffset);
3491             mubuf->offen = true;
3492             mubuf->offset = const_offset % 4096u;
3493             mubuf->glc = true;
3494             mubuf->dlc = ctx->options->chip_class >= GFX10;
3495             mubuf->barrier = barrier_none;
3496             mubuf->can_reorder = true;
3497             bld.insert(std::move(mubuf));
3498
3499             const_offset += ctx->program->wave_size * 4u;
3500          }
3501
3502          if (elem_size_bytes == 4)
3503             elems[i] = subelems[0];
3504          else
3505             elems[i] = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), subelems[0], subelems[1]);
3506          vec->operands[i] = Operand(elems[i]);
3507       }
3508       vec->definitions[0] = Definition(dst);
3509       ctx->block->instructions.emplace_back(std::move(vec));
3510       ctx->allocated_vec.emplace(dst.id(), elems);
3511    } else {
3512       unsigned align = 16; /* alignment of indirect offset */
3513       align = std::min(align, 1u << (ffs(itemsize) - 1));
3514       if (const_offset)
3515          align = std::min(align, 1u << (ffs(const_offset) - 1));
3516
3517       load_lds(ctx, elem_size_bytes, dst, offset, const_offset, align);
3518    }
3519 }
3520
3521 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3522 {
3523    if (ctx->program->info->need_indirect_descriptor_sets) {
3524       Builder bld(ctx->program, ctx->block);
3525       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3526       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3527       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3528    }
3529
3530    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3531 }
3532
3533
3534 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3535 {
3536    Builder bld(ctx->program, ctx->block);
3537    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3538    if (!ctx->divergent_vals[instr->dest.ssa.index])
3539       index = bld.as_uniform(index);
3540    unsigned desc_set = nir_intrinsic_desc_set(instr);
3541    unsigned binding = nir_intrinsic_binding(instr);
3542
3543    Temp desc_ptr;
3544    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3545    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3546    unsigned offset = layout->binding[binding].offset;
3547    unsigned stride;
3548    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3549        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3550       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3551       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3552       offset = pipeline_layout->push_constant_size + 16 * idx;
3553       stride = 16;
3554    } else {
3555       desc_ptr = load_desc_ptr(ctx, desc_set);
3556       stride = layout->binding[binding].size;
3557    }
3558
3559    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3560    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3561    if (stride != 1) {
3562       if (nir_const_index) {
3563          const_index = const_index * stride;
3564       } else if (index.type() == RegType::vgpr) {
3565          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3566          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3567       } else {
3568          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3569       }
3570    }
3571    if (offset) {
3572       if (nir_const_index) {
3573          const_index = const_index + offset;
3574       } else if (index.type() == RegType::vgpr) {
3575          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3576       } else {
3577          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3578       }
3579    }
3580
3581    if (nir_const_index && const_index == 0) {
3582       index = desc_ptr;
3583    } else if (index.type() == RegType::vgpr) {
3584       index = bld.vadd32(bld.def(v1),
3585                          nir_const_index ? Operand(const_index) : Operand(index),
3586                          Operand(desc_ptr));
3587    } else {
3588       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3589                        nir_const_index ? Operand(const_index) : Operand(index),
3590                        Operand(desc_ptr));
3591    }
3592
3593    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3594 }
3595
3596 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3597                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3598 {
3599    Builder bld(ctx->program, ctx->block);
3600
3601    unsigned num_bytes = dst.size() * 4;
3602    bool dlc = glc && ctx->options->chip_class >= GFX10;
3603
3604    aco_opcode op;
3605    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3606       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3607       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3608       unsigned const_offset = 0;
3609
3610       Temp lower = Temp();
3611       if (num_bytes > 16) {
3612          assert(num_components == 3 || num_components == 4);
3613          op = aco_opcode::buffer_load_dwordx4;
3614          lower = bld.tmp(v4);
3615          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3616          mubuf->definitions[0] = Definition(lower);
3617          mubuf->operands[0] = vaddr;
3618          mubuf->operands[1] = Operand(rsrc);
3619          mubuf->operands[2] = soffset;
3620          mubuf->offen = (offset.type() == RegType::vgpr);
3621          mubuf->glc = glc;
3622          mubuf->dlc = dlc;
3623          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3624          mubuf->can_reorder = readonly;
3625          bld.insert(std::move(mubuf));
3626          emit_split_vector(ctx, lower, 2);
3627          num_bytes -= 16;
3628          const_offset = 16;
3629       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3630          /* GFX6 doesn't support loading vec3, expand to vec4. */
3631          num_bytes = 16;
3632       }
3633
3634       switch (num_bytes) {
3635          case 4:
3636             op = aco_opcode::buffer_load_dword;
3637             break;
3638          case 8:
3639             op = aco_opcode::buffer_load_dwordx2;
3640             break;
3641          case 12:
3642             assert(ctx->options->chip_class > GFX6);
3643             op = aco_opcode::buffer_load_dwordx3;
3644             break;
3645          case 16:
3646             op = aco_opcode::buffer_load_dwordx4;
3647             break;
3648          default:
3649             unreachable("Load SSBO not implemented for this size.");
3650       }
3651       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3652       mubuf->operands[0] = vaddr;
3653       mubuf->operands[1] = Operand(rsrc);
3654       mubuf->operands[2] = soffset;
3655       mubuf->offen = (offset.type() == RegType::vgpr);
3656       mubuf->glc = glc;
3657       mubuf->dlc = dlc;
3658       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3659       mubuf->can_reorder = readonly;
3660       mubuf->offset = const_offset;
3661       aco_ptr<Instruction> instr = std::move(mubuf);
3662
3663       if (dst.size() > 4) {
3664          assert(lower != Temp());
3665          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3666          instr->definitions[0] = Definition(upper);
3667          bld.insert(std::move(instr));
3668          if (dst.size() == 8)
3669             emit_split_vector(ctx, upper, 2);
3670          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3671          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3672          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3673          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3674          if (dst.size() == 8)
3675             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3676       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3677          Temp vec = bld.tmp(v4);
3678          instr->definitions[0] = Definition(vec);
3679          bld.insert(std::move(instr));
3680          emit_split_vector(ctx, vec, 4);
3681
3682          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3683          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3684          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3685          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3686       }
3687
3688       if (dst.type() == RegType::sgpr) {
3689          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3690          instr->definitions[0] = Definition(vec);
3691          bld.insert(std::move(instr));
3692          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3693       } else {
3694          instr->definitions[0] = Definition(dst);
3695          bld.insert(std::move(instr));
3696          emit_split_vector(ctx, dst, num_components);
3697       }
3698    } else {
3699       switch (num_bytes) {
3700          case 4:
3701             op = aco_opcode::s_buffer_load_dword;
3702             break;
3703          case 8:
3704             op = aco_opcode::s_buffer_load_dwordx2;
3705             break;
3706          case 12:
3707          case 16:
3708             op = aco_opcode::s_buffer_load_dwordx4;
3709             break;
3710          case 24:
3711          case 32:
3712             op = aco_opcode::s_buffer_load_dwordx8;
3713             break;
3714          default:
3715             unreachable("Load SSBO not implemented for this size.");
3716       }
3717       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3718       load->operands[0] = Operand(rsrc);
3719       load->operands[1] = Operand(bld.as_uniform(offset));
3720       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3721       load->definitions[0] = Definition(dst);
3722       load->glc = glc;
3723       load->dlc = dlc;
3724       load->barrier = readonly ? barrier_none : barrier_buffer;
3725       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3726       assert(ctx->options->chip_class >= GFX8 || !glc);
3727
3728       /* trim vector */
3729       if (dst.size() == 3) {
3730          Temp vec = bld.tmp(s4);
3731          load->definitions[0] = Definition(vec);
3732          bld.insert(std::move(load));
3733          emit_split_vector(ctx, vec, 4);
3734
3735          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3736                     emit_extract_vector(ctx, vec, 0, s1),
3737                     emit_extract_vector(ctx, vec, 1, s1),
3738                     emit_extract_vector(ctx, vec, 2, s1));
3739       } else if (dst.size() == 6) {
3740          Temp vec = bld.tmp(s8);
3741          load->definitions[0] = Definition(vec);
3742          bld.insert(std::move(load));
3743          emit_split_vector(ctx, vec, 4);
3744
3745          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3746                     emit_extract_vector(ctx, vec, 0, s2),
3747                     emit_extract_vector(ctx, vec, 1, s2),
3748                     emit_extract_vector(ctx, vec, 2, s2));
3749       } else {
3750          bld.insert(std::move(load));
3751       }
3752       emit_split_vector(ctx, dst, num_components);
3753    }
3754 }
3755
3756 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3757 {
3758    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3759    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3760
3761    Builder bld(ctx->program, ctx->block);
3762
3763    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3764    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3765    unsigned binding = nir_intrinsic_binding(idx_instr);
3766    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3767
3768    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3769       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3770                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3771                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3772                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3773       if (ctx->options->chip_class >= GFX10) {
3774          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3775                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3776                       S_008F0C_RESOURCE_LEVEL(1);
3777       } else {
3778          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3779                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3780       }
3781       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3782                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3783                                      Operand(0xFFFFFFFFu),
3784                                      Operand(desc_type));
3785       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3786                         rsrc, upper_dwords);
3787    } else {
3788       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3789       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3790    }
3791
3792    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3793 }
3794
3795 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3796 {
3797    Builder bld(ctx->program, ctx->block);
3798    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3799
3800    unsigned offset = nir_intrinsic_base(instr);
3801    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3802    if (index_cv && instr->dest.ssa.bit_size == 32) {
3803
3804       unsigned count = instr->dest.ssa.num_components;
3805       unsigned start = (offset + index_cv->u32) / 4u;
3806       start -= ctx->args->ac.base_inline_push_consts;
3807       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3808          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3809          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3810          for (unsigned i = 0; i < count; ++i) {
3811             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3812             vec->operands[i] = Operand{elems[i]};
3813          }
3814          vec->definitions[0] = Definition(dst);
3815          ctx->block->instructions.emplace_back(std::move(vec));
3816          ctx->allocated_vec.emplace(dst.id(), elems);
3817          return;
3818       }
3819    }
3820
3821    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3822    if (offset != 0) // TODO check if index != 0 as well
3823       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3824    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3825    Temp vec = dst;
3826    bool trim = false;
3827    aco_opcode op;
3828
3829    switch (dst.size()) {
3830    case 1:
3831       op = aco_opcode::s_load_dword;
3832       break;
3833    case 2:
3834       op = aco_opcode::s_load_dwordx2;
3835       break;
3836    case 3:
3837       vec = bld.tmp(s4);
3838       trim = true;
3839    case 4:
3840       op = aco_opcode::s_load_dwordx4;
3841       break;
3842    case 6:
3843       vec = bld.tmp(s8);
3844       trim = true;
3845    case 8:
3846       op = aco_opcode::s_load_dwordx8;
3847       break;
3848    default:
3849       unreachable("unimplemented or forbidden load_push_constant.");
3850    }
3851
3852    bld.smem(op, Definition(vec), ptr, index);
3853
3854    if (trim) {
3855       emit_split_vector(ctx, vec, 4);
3856       RegClass rc = dst.size() == 3 ? s1 : s2;
3857       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3858                  emit_extract_vector(ctx, vec, 0, rc),
3859                  emit_extract_vector(ctx, vec, 1, rc),
3860                  emit_extract_vector(ctx, vec, 2, rc));
3861
3862    }
3863    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3864 }
3865
3866 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3867 {
3868    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3869
3870    Builder bld(ctx->program, ctx->block);
3871
3872    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3873                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3874                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3875                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3876    if (ctx->options->chip_class >= GFX10) {
3877       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3878                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3879                    S_008F0C_RESOURCE_LEVEL(1);
3880    } else {
3881       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3882                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3883    }
3884
3885    unsigned base = nir_intrinsic_base(instr);
3886    unsigned range = nir_intrinsic_range(instr);
3887
3888    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3889    if (base && offset.type() == RegType::sgpr)
3890       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3891    else if (base && offset.type() == RegType::vgpr)
3892       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3893
3894    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3895                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3896                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3897                           Operand(desc_type));
3898
3899    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3900 }
3901
3902 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3903 {
3904    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3905       ctx->cf_info.exec_potentially_empty = true;
3906
3907    ctx->program->needs_exact = true;
3908
3909    // TODO: optimize uniform conditions
3910    Builder bld(ctx->program, ctx->block);
3911    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3912    assert(src.regClass() == bld.lm);
3913    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3914    bld.pseudo(aco_opcode::p_discard_if, src);
3915    ctx->block->kind |= block_kind_uses_discard_if;
3916    return;
3917 }
3918
3919 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3920 {
3921    Builder bld(ctx->program, ctx->block);
3922
3923    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3924       ctx->cf_info.exec_potentially_empty = true;
3925
3926    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3927                     ctx->cf_info.parent_loop.has_divergent_continue;
3928
3929    if (ctx->block->loop_nest_depth &&
3930        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3931       /* we handle discards the same way as jump instructions */
3932       append_logical_end(ctx->block);
3933
3934       /* in loops, discard behaves like break */
3935       Block *linear_target = ctx->cf_info.parent_loop.exit;
3936       ctx->block->kind |= block_kind_discard;
3937
3938       if (!divergent) {
3939          /* uniform discard - loop ends here */
3940          assert(nir_instr_is_last(&instr->instr));
3941          ctx->block->kind |= block_kind_uniform;
3942          ctx->cf_info.has_branch = true;
3943          bld.branch(aco_opcode::p_branch);
3944          add_linear_edge(ctx->block->index, linear_target);
3945          return;
3946       }
3947
3948       /* we add a break right behind the discard() instructions */
3949       ctx->block->kind |= block_kind_break;
3950       unsigned idx = ctx->block->index;
3951
3952       /* remove critical edges from linear CFG */
3953       bld.branch(aco_opcode::p_branch);
3954       Block* break_block = ctx->program->create_and_insert_block();
3955       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3956       break_block->kind |= block_kind_uniform;
3957       add_linear_edge(idx, break_block);
3958       add_linear_edge(break_block->index, linear_target);
3959       bld.reset(break_block);
3960       bld.branch(aco_opcode::p_branch);
3961
3962       Block* continue_block = ctx->program->create_and_insert_block();
3963       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3964       add_linear_edge(idx, continue_block);
3965       append_logical_start(continue_block);
3966       ctx->block = continue_block;
3967
3968       return;
3969    }
3970
3971    /* it can currently happen that NIR doesn't remove the unreachable code */
3972    if (!nir_instr_is_last(&instr->instr)) {
3973       ctx->program->needs_exact = true;
3974       /* save exec somewhere temporarily so that it doesn't get
3975        * overwritten before the discard from outer exec masks */
3976       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3977       bld.pseudo(aco_opcode::p_discard_if, cond);
3978       ctx->block->kind |= block_kind_uses_discard_if;
3979       return;
3980    }
3981
3982    /* This condition is incorrect for uniformly branched discards in a loop
3983     * predicated by a divergent condition, but the above code catches that case
3984     * and the discard would end up turning into a discard_if.
3985     * For example:
3986     * if (divergent) {
3987     *    while (...) {
3988     *       if (uniform) {
3989     *          discard;
3990     *       }
3991     *    }
3992     * }
3993     */
3994    if (!ctx->cf_info.parent_if.is_divergent) {
3995       /* program just ends here */
3996       ctx->block->kind |= block_kind_uniform;
3997       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3998               0 /* enabled mask */, 9 /* dest */,
3999               false /* compressed */, true/* done */, true /* valid mask */);
4000       bld.sopp(aco_opcode::s_endpgm);
4001       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
4002    } else {
4003       ctx->block->kind |= block_kind_discard;
4004       /* branch and linear edge is added by visit_if() */
4005    }
4006 }
4007
4008 enum aco_descriptor_type {
4009    ACO_DESC_IMAGE,
4010    ACO_DESC_FMASK,
4011    ACO_DESC_SAMPLER,
4012    ACO_DESC_BUFFER,
4013    ACO_DESC_PLANE_0,
4014    ACO_DESC_PLANE_1,
4015    ACO_DESC_PLANE_2,
4016 };
4017
4018 static bool
4019 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
4020    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
4021       return false;
4022    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
4023    return dim == ac_image_cube ||
4024           dim == ac_image_1darray ||
4025           dim == ac_image_2darray ||
4026           dim == ac_image_2darraymsaa;
4027 }
4028
4029 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
4030                       enum aco_descriptor_type desc_type,
4031                       const nir_tex_instr *tex_instr, bool image, bool write)
4032 {
4033 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
4034    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
4035    if (it != ctx->tex_desc.end())
4036       return it->second;
4037 */
4038    Temp index = Temp();
4039    bool index_set = false;
4040    unsigned constant_index = 0;
4041    unsigned descriptor_set;
4042    unsigned base_index;
4043    Builder bld(ctx->program, ctx->block);
4044
4045    if (!deref_instr) {
4046       assert(tex_instr && !image);
4047       descriptor_set = 0;
4048       base_index = tex_instr->sampler_index;
4049    } else {
4050       while(deref_instr->deref_type != nir_deref_type_var) {
4051          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
4052          if (!array_size)
4053             array_size = 1;
4054
4055          assert(deref_instr->deref_type == nir_deref_type_array);
4056          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
4057          if (const_value) {
4058             constant_index += array_size * const_value->u32;
4059          } else {
4060             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
4061             if (indirect.type() == RegType::vgpr)
4062                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
4063
4064             if (array_size != 1)
4065                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
4066
4067             if (!index_set) {
4068                index = indirect;
4069                index_set = true;
4070             } else {
4071                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
4072             }
4073          }
4074
4075          deref_instr = nir_src_as_deref(deref_instr->parent);
4076       }
4077       descriptor_set = deref_instr->var->data.descriptor_set;
4078       base_index = deref_instr->var->data.binding;
4079    }
4080
4081    Temp list = load_desc_ptr(ctx, descriptor_set);
4082    list = convert_pointer_to_64_bit(ctx, list);
4083
4084    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
4085    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
4086    unsigned offset = binding->offset;
4087    unsigned stride = binding->size;
4088    aco_opcode opcode;
4089    RegClass type;
4090
4091    assert(base_index < layout->binding_count);
4092
4093    switch (desc_type) {
4094    case ACO_DESC_IMAGE:
4095       type = s8;
4096       opcode = aco_opcode::s_load_dwordx8;
4097       break;
4098    case ACO_DESC_FMASK:
4099       type = s8;
4100       opcode = aco_opcode::s_load_dwordx8;
4101       offset += 32;
4102       break;
4103    case ACO_DESC_SAMPLER:
4104       type = s4;
4105       opcode = aco_opcode::s_load_dwordx4;
4106       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4107          offset += radv_combined_image_descriptor_sampler_offset(binding);
4108       break;
4109    case ACO_DESC_BUFFER:
4110       type = s4;
4111       opcode = aco_opcode::s_load_dwordx4;
4112       break;
4113    case ACO_DESC_PLANE_0:
4114    case ACO_DESC_PLANE_1:
4115       type = s8;
4116       opcode = aco_opcode::s_load_dwordx8;
4117       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4118       break;
4119    case ACO_DESC_PLANE_2:
4120       type = s4;
4121       opcode = aco_opcode::s_load_dwordx4;
4122       offset += 64;
4123       break;
4124    default:
4125       unreachable("invalid desc_type\n");
4126    }
4127
4128    offset += constant_index * stride;
4129
4130    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4131       (!index_set || binding->immutable_samplers_equal)) {
4132       if (binding->immutable_samplers_equal)
4133          constant_index = 0;
4134
4135       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4136       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4137                         Operand(samplers[constant_index * 4 + 0]),
4138                         Operand(samplers[constant_index * 4 + 1]),
4139                         Operand(samplers[constant_index * 4 + 2]),
4140                         Operand(samplers[constant_index * 4 + 3]));
4141    }
4142
4143    Operand off;
4144    if (!index_set) {
4145       off = bld.copy(bld.def(s1), Operand(offset));
4146    } else {
4147       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4148                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4149    }
4150
4151    Temp res = bld.smem(opcode, bld.def(type), list, off);
4152
4153    if (desc_type == ACO_DESC_PLANE_2) {
4154       Temp components[8];
4155       for (unsigned i = 0; i < 8; i++)
4156          components[i] = bld.tmp(s1);
4157       bld.pseudo(aco_opcode::p_split_vector,
4158                  Definition(components[0]),
4159                  Definition(components[1]),
4160                  Definition(components[2]),
4161                  Definition(components[3]),
4162                  res);
4163
4164       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4165       bld.pseudo(aco_opcode::p_split_vector,
4166                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4167                  Definition(components[4]),
4168                  Definition(components[5]),
4169                  Definition(components[6]),
4170                  Definition(components[7]),
4171                  desc2);
4172
4173       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4174                        components[0], components[1], components[2], components[3],
4175                        components[4], components[5], components[6], components[7]);
4176    }
4177
4178    return res;
4179 }
4180
4181 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4182 {
4183    switch (dim) {
4184    case GLSL_SAMPLER_DIM_BUF:
4185       return 1;
4186    case GLSL_SAMPLER_DIM_1D:
4187       return array ? 2 : 1;
4188    case GLSL_SAMPLER_DIM_2D:
4189       return array ? 3 : 2;
4190    case GLSL_SAMPLER_DIM_MS:
4191       return array ? 4 : 3;
4192    case GLSL_SAMPLER_DIM_3D:
4193    case GLSL_SAMPLER_DIM_CUBE:
4194       return 3;
4195    case GLSL_SAMPLER_DIM_RECT:
4196    case GLSL_SAMPLER_DIM_SUBPASS:
4197       return 2;
4198    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4199       return 3;
4200    default:
4201       break;
4202    }
4203    return 0;
4204 }
4205
4206
4207 /* Adjust the sample index according to FMASK.
4208  *
4209  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4210  * which is the identity mapping. Each nibble says which physical sample
4211  * should be fetched to get that sample.
4212  *
4213  * For example, 0x11111100 means there are only 2 samples stored and
4214  * the second sample covers 3/4 of the pixel. When reading samples 0
4215  * and 1, return physical sample 0 (determined by the first two 0s
4216  * in FMASK), otherwise return physical sample 1.
4217  *
4218  * The sample index should be adjusted as follows:
4219  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4220  */
4221 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4222 {
4223    Builder bld(ctx->program, ctx->block);
4224    Temp fmask = bld.tmp(v1);
4225    unsigned dim = ctx->options->chip_class >= GFX10
4226                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4227                   : 0;
4228
4229    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4230    load->operands[0] = Operand(coords);
4231    load->operands[1] = Operand(fmask_desc_ptr);
4232    load->definitions[0] = Definition(fmask);
4233    load->glc = false;
4234    load->dlc = false;
4235    load->dmask = 0x1;
4236    load->unrm = true;
4237    load->da = da;
4238    load->dim = dim;
4239    load->can_reorder = true; /* fmask images shouldn't be modified */
4240    ctx->block->instructions.emplace_back(std::move(load));
4241
4242    Operand sample_index4;
4243    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4244       sample_index4 = Operand(sample_index.constantValue() << 2);
4245    } else if (sample_index.regClass() == s1) {
4246       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4247    } else {
4248       assert(sample_index.regClass() == v1);
4249       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4250    }
4251
4252    Temp final_sample;
4253    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4254       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4255    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4256       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4257    else
4258       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4259
4260    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4261     * resource descriptor is 0 (invalid),
4262     */
4263    Temp compare = bld.tmp(bld.lm);
4264    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4265                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4266
4267    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4268
4269    /* Replace the MSAA sample index. */
4270    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4271 }
4272
4273 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4274 {
4275
4276    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4277    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4278    bool is_array = glsl_sampler_type_is_array(type);
4279    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4280    assert(!add_frag_pos && "Input attachments should be lowered.");
4281    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4282    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4283    int count = image_type_to_components_count(dim, is_array);
4284    std::vector<Operand> coords(count);
4285
4286    if (is_ms) {
4287       Operand sample_index;
4288       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4289       if (sample_cv)
4290          sample_index = Operand(sample_cv->u32);
4291       else
4292          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4293
4294       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4295          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4296          for (unsigned i = 0; i < vec->operands.size(); i++)
4297             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4298          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4299          vec->definitions[0] = Definition(fmask_load_address);
4300          ctx->block->instructions.emplace_back(std::move(vec));
4301
4302          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4303          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4304       }
4305       count--;
4306       coords[count] = sample_index;
4307    }
4308
4309    if (count == 1 && !gfx9_1d)
4310       return emit_extract_vector(ctx, src0, 0, v1);
4311
4312    if (gfx9_1d) {
4313       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4314       coords.resize(coords.size() + 1);
4315       coords[1] = Operand((uint32_t) 0);
4316       if (is_array)
4317          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4318    } else {
4319       for (int i = 0; i < count; i++)
4320          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4321    }
4322
4323    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4324        instr->intrinsic == nir_intrinsic_image_deref_store) {
4325       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4326       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4327
4328       if (!level_zero)
4329          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4330    }
4331
4332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4333    for (unsigned i = 0; i < coords.size(); i++)
4334       vec->operands[i] = coords[i];
4335    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4336    vec->definitions[0] = Definition(res);
4337    ctx->block->instructions.emplace_back(std::move(vec));
4338    return res;
4339 }
4340
4341
4342 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4343 {
4344    Builder bld(ctx->program, ctx->block);
4345    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4346    const struct glsl_type *type = glsl_without_array(var->type);
4347    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4348    bool is_array = glsl_sampler_type_is_array(type);
4349    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4350
4351    if (dim == GLSL_SAMPLER_DIM_BUF) {
4352       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4353       unsigned num_channels = util_last_bit(mask);
4354       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4355       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4356
4357       aco_opcode opcode;
4358       switch (num_channels) {
4359       case 1:
4360          opcode = aco_opcode::buffer_load_format_x;
4361          break;
4362       case 2:
4363          opcode = aco_opcode::buffer_load_format_xy;
4364          break;
4365       case 3:
4366          opcode = aco_opcode::buffer_load_format_xyz;
4367          break;
4368       case 4:
4369          opcode = aco_opcode::buffer_load_format_xyzw;
4370          break;
4371       default:
4372          unreachable(">4 channel buffer image load");
4373       }
4374       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4375       load->operands[0] = Operand(vindex);
4376       load->operands[1] = Operand(rsrc);
4377       load->operands[2] = Operand((uint32_t) 0);
4378       Temp tmp;
4379       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4380          tmp = dst;
4381       else
4382          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4383       load->definitions[0] = Definition(tmp);
4384       load->idxen = true;
4385       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4386       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4387       load->barrier = barrier_image;
4388       ctx->block->instructions.emplace_back(std::move(load));
4389
4390       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4391       return;
4392    }
4393
4394    Temp coords = get_image_coords(ctx, instr, type);
4395    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4396
4397    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4398    unsigned num_components = util_bitcount(dmask);
4399    Temp tmp;
4400    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4401       tmp = dst;
4402    else
4403       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4404
4405    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4406    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4407
4408    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4409    load->operands[0] = Operand(coords);
4410    load->operands[1] = Operand(resource);
4411    load->definitions[0] = Definition(tmp);
4412    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4413    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4414    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4415    load->dmask = dmask;
4416    load->unrm = true;
4417    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4418    load->barrier = barrier_image;
4419    ctx->block->instructions.emplace_back(std::move(load));
4420
4421    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4422    return;
4423 }
4424
4425 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4426 {
4427    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4428    const struct glsl_type *type = glsl_without_array(var->type);
4429    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4430    bool is_array = glsl_sampler_type_is_array(type);
4431    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4432
4433    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4434
4435    if (dim == GLSL_SAMPLER_DIM_BUF) {
4436       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4437       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4438       aco_opcode opcode;
4439       switch (data.size()) {
4440       case 1:
4441          opcode = aco_opcode::buffer_store_format_x;
4442          break;
4443       case 2:
4444          opcode = aco_opcode::buffer_store_format_xy;
4445          break;
4446       case 3:
4447          opcode = aco_opcode::buffer_store_format_xyz;
4448          break;
4449       case 4:
4450          opcode = aco_opcode::buffer_store_format_xyzw;
4451          break;
4452       default:
4453          unreachable(">4 channel buffer image store");
4454       }
4455       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4456       store->operands[0] = Operand(vindex);
4457       store->operands[1] = Operand(rsrc);
4458       store->operands[2] = Operand((uint32_t) 0);
4459       store->operands[3] = Operand(data);
4460       store->idxen = true;
4461       store->glc = glc;
4462       store->dlc = false;
4463       store->disable_wqm = true;
4464       store->barrier = barrier_image;
4465       ctx->program->needs_exact = true;
4466       ctx->block->instructions.emplace_back(std::move(store));
4467       return;
4468    }
4469
4470    assert(data.type() == RegType::vgpr);
4471    Temp coords = get_image_coords(ctx, instr, type);
4472    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4473
4474    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4475    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4476
4477    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4478    store->operands[0] = Operand(coords);
4479    store->operands[1] = Operand(resource);
4480    store->operands[2] = Operand(s4);
4481    store->operands[3] = Operand(data);
4482    store->glc = glc;
4483    store->dlc = false;
4484    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4485    store->dmask = (1 << data.size()) - 1;
4486    store->unrm = true;
4487    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4488    store->disable_wqm = true;
4489    store->barrier = barrier_image;
4490    ctx->program->needs_exact = true;
4491    ctx->block->instructions.emplace_back(std::move(store));
4492    return;
4493 }
4494
4495 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4496 {
4497    /* return the previous value if dest is ever used */
4498    bool return_previous = false;
4499    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4500       return_previous = true;
4501       break;
4502    }
4503    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4504       return_previous = true;
4505       break;
4506    }
4507
4508    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4509    const struct glsl_type *type = glsl_without_array(var->type);
4510    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4511    bool is_array = glsl_sampler_type_is_array(type);
4512    Builder bld(ctx->program, ctx->block);
4513
4514    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4515    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4516
4517    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4518       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4519
4520    aco_opcode buf_op, image_op;
4521    switch (instr->intrinsic) {
4522       case nir_intrinsic_image_deref_atomic_add:
4523          buf_op = aco_opcode::buffer_atomic_add;
4524          image_op = aco_opcode::image_atomic_add;
4525          break;
4526       case nir_intrinsic_image_deref_atomic_umin:
4527          buf_op = aco_opcode::buffer_atomic_umin;
4528          image_op = aco_opcode::image_atomic_umin;
4529          break;
4530       case nir_intrinsic_image_deref_atomic_imin:
4531          buf_op = aco_opcode::buffer_atomic_smin;
4532          image_op = aco_opcode::image_atomic_smin;
4533          break;
4534       case nir_intrinsic_image_deref_atomic_umax:
4535          buf_op = aco_opcode::buffer_atomic_umax;
4536          image_op = aco_opcode::image_atomic_umax;
4537          break;
4538       case nir_intrinsic_image_deref_atomic_imax:
4539          buf_op = aco_opcode::buffer_atomic_smax;
4540          image_op = aco_opcode::image_atomic_smax;
4541          break;
4542       case nir_intrinsic_image_deref_atomic_and:
4543          buf_op = aco_opcode::buffer_atomic_and;
4544          image_op = aco_opcode::image_atomic_and;
4545          break;
4546       case nir_intrinsic_image_deref_atomic_or:
4547          buf_op = aco_opcode::buffer_atomic_or;
4548          image_op = aco_opcode::image_atomic_or;
4549          break;
4550       case nir_intrinsic_image_deref_atomic_xor:
4551          buf_op = aco_opcode::buffer_atomic_xor;
4552          image_op = aco_opcode::image_atomic_xor;
4553          break;
4554       case nir_intrinsic_image_deref_atomic_exchange:
4555          buf_op = aco_opcode::buffer_atomic_swap;
4556          image_op = aco_opcode::image_atomic_swap;
4557          break;
4558       case nir_intrinsic_image_deref_atomic_comp_swap:
4559          buf_op = aco_opcode::buffer_atomic_cmpswap;
4560          image_op = aco_opcode::image_atomic_cmpswap;
4561          break;
4562       default:
4563          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4564    }
4565
4566    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4567
4568    if (dim == GLSL_SAMPLER_DIM_BUF) {
4569       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4570       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4571       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4572       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4573       mubuf->operands[0] = Operand(vindex);
4574       mubuf->operands[1] = Operand(resource);
4575       mubuf->operands[2] = Operand((uint32_t)0);
4576       mubuf->operands[3] = Operand(data);
4577       if (return_previous)
4578          mubuf->definitions[0] = Definition(dst);
4579       mubuf->offset = 0;
4580       mubuf->idxen = true;
4581       mubuf->glc = return_previous;
4582       mubuf->dlc = false; /* Not needed for atomics */
4583       mubuf->disable_wqm = true;
4584       mubuf->barrier = barrier_image;
4585       ctx->program->needs_exact = true;
4586       ctx->block->instructions.emplace_back(std::move(mubuf));
4587       return;
4588    }
4589
4590    Temp coords = get_image_coords(ctx, instr, type);
4591    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4592    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4593    mimg->operands[0] = Operand(coords);
4594    mimg->operands[1] = Operand(resource);
4595    mimg->operands[2] = Operand(s4); /* no sampler */
4596    mimg->operands[3] = Operand(data);
4597    if (return_previous)
4598       mimg->definitions[0] = Definition(dst);
4599    mimg->glc = return_previous;
4600    mimg->dlc = false; /* Not needed for atomics */
4601    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4602    mimg->dmask = (1 << data.size()) - 1;
4603    mimg->unrm = true;
4604    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4605    mimg->disable_wqm = true;
4606    mimg->barrier = barrier_image;
4607    ctx->program->needs_exact = true;
4608    ctx->block->instructions.emplace_back(std::move(mimg));
4609    return;
4610 }
4611
4612 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4613 {
4614    if (in_elements && ctx->options->chip_class == GFX8) {
4615       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4616       Builder bld(ctx->program, ctx->block);
4617
4618       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4619
4620       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4621       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4622
4623       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4624       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4625
4626       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4627       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4628
4629       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4630       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4631                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4632       if (dst.type() == RegType::vgpr)
4633          bld.copy(Definition(dst), shr_dst);
4634
4635       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4636    } else {
4637       emit_extract_vector(ctx, desc, 2, dst);
4638    }
4639 }
4640
4641 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4642 {
4643    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4644    const struct glsl_type *type = glsl_without_array(var->type);
4645    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4646    bool is_array = glsl_sampler_type_is_array(type);
4647    Builder bld(ctx->program, ctx->block);
4648
4649    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4650       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4651       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4652    }
4653
4654    /* LOD */
4655    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4656
4657    /* Resource */
4658    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4659
4660    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4661
4662    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4663    mimg->operands[0] = Operand(lod);
4664    mimg->operands[1] = Operand(resource);
4665    uint8_t& dmask = mimg->dmask;
4666    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4667    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4668    mimg->da = glsl_sampler_type_is_array(type);
4669    mimg->can_reorder = true;
4670    Definition& def = mimg->definitions[0];
4671    ctx->block->instructions.emplace_back(std::move(mimg));
4672
4673    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4674        glsl_sampler_type_is_array(type)) {
4675
4676       assert(instr->dest.ssa.num_components == 3);
4677       Temp tmp = {ctx->program->allocateId(), v3};
4678       def = Definition(tmp);
4679       emit_split_vector(ctx, tmp, 3);
4680
4681       /* divide 3rd value by 6 by multiplying with magic number */
4682       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4683       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4684
4685       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4686                  emit_extract_vector(ctx, tmp, 0, v1),
4687                  emit_extract_vector(ctx, tmp, 1, v1),
4688                  by_6);
4689
4690    } else if (ctx->options->chip_class == GFX9 &&
4691               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4692               glsl_sampler_type_is_array(type)) {
4693       assert(instr->dest.ssa.num_components == 2);
4694       def = Definition(dst);
4695       dmask = 0x5;
4696    } else {
4697       def = Definition(dst);
4698    }
4699
4700    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4701 }
4702
4703 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4704 {
4705    Builder bld(ctx->program, ctx->block);
4706    unsigned num_components = instr->num_components;
4707
4708    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4709    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4710    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4711
4712    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4713    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4714 }
4715
4716 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4717 {
4718    Builder bld(ctx->program, ctx->block);
4719    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4720    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4721    unsigned writemask = nir_intrinsic_write_mask(instr);
4722    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4723
4724    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4725    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4726
4727    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4728                ctx->options->chip_class >= GFX8;
4729    if (smem)
4730       offset = bld.as_uniform(offset);
4731    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4732
4733    while (writemask) {
4734       int start, count;
4735       u_bit_scan_consecutive_range(&writemask, &start, &count);
4736       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4737          /* GFX6 doesn't support storing vec3, split it. */
4738          writemask |= 1u << (start + 2);
4739          count = 2;
4740       }
4741       int num_bytes = count * elem_size_bytes;
4742
4743       if (num_bytes > 16) {
4744          assert(elem_size_bytes == 8);
4745          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4746          count = 2;
4747          num_bytes = 16;
4748       }
4749
4750       // TODO: check alignment of sub-dword stores
4751       // TODO: split 3 bytes. there is no store instruction for that
4752
4753       Temp write_data;
4754       if (count != instr->num_components) {
4755          emit_split_vector(ctx, data, instr->num_components);
4756          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4757          for (int i = 0; i < count; i++) {
4758             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4759             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4760          }
4761          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4762          vec->definitions[0] = Definition(write_data);
4763          ctx->block->instructions.emplace_back(std::move(vec));
4764       } else if (!smem && data.type() != RegType::vgpr) {
4765          assert(num_bytes % 4 == 0);
4766          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4767       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4768          assert(num_bytes % 4 == 0);
4769          write_data = bld.as_uniform(data);
4770       } else {
4771          write_data = data;
4772       }
4773
4774       aco_opcode vmem_op, smem_op;
4775       switch (num_bytes) {
4776          case 4:
4777             vmem_op = aco_opcode::buffer_store_dword;
4778             smem_op = aco_opcode::s_buffer_store_dword;
4779             break;
4780          case 8:
4781             vmem_op = aco_opcode::buffer_store_dwordx2;
4782             smem_op = aco_opcode::s_buffer_store_dwordx2;
4783             break;
4784          case 12:
4785             vmem_op = aco_opcode::buffer_store_dwordx3;
4786             smem_op = aco_opcode::last_opcode;
4787             assert(!smem && ctx->options->chip_class > GFX6);
4788             break;
4789          case 16:
4790             vmem_op = aco_opcode::buffer_store_dwordx4;
4791             smem_op = aco_opcode::s_buffer_store_dwordx4;
4792             break;
4793          default:
4794             unreachable("Store SSBO not implemented for this size.");
4795       }
4796       if (ctx->stage == fragment_fs)
4797          smem_op = aco_opcode::p_fs_buffer_store_smem;
4798
4799       if (smem) {
4800          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4801          store->operands[0] = Operand(rsrc);
4802          if (start) {
4803             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4804                                 offset, Operand(start * elem_size_bytes));
4805             store->operands[1] = Operand(off);
4806          } else {
4807             store->operands[1] = Operand(offset);
4808          }
4809          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4810             store->operands[1].setFixed(m0);
4811          store->operands[2] = Operand(write_data);
4812          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4813          store->dlc = false;
4814          store->disable_wqm = true;
4815          store->barrier = barrier_buffer;
4816          ctx->block->instructions.emplace_back(std::move(store));
4817          ctx->program->wb_smem_l1_on_end = true;
4818          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4819             ctx->block->kind |= block_kind_needs_lowering;
4820             ctx->program->needs_exact = true;
4821          }
4822       } else {
4823          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4824          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4825          store->operands[1] = Operand(rsrc);
4826          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4827          store->operands[3] = Operand(write_data);
4828          store->offset = start * elem_size_bytes;
4829          store->offen = (offset.type() == RegType::vgpr);
4830          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4831          store->dlc = false;
4832          store->disable_wqm = true;
4833          store->barrier = barrier_buffer;
4834          ctx->program->needs_exact = true;
4835          ctx->block->instructions.emplace_back(std::move(store));
4836       }
4837    }
4838 }
4839
4840 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4841 {
4842    /* return the previous value if dest is ever used */
4843    bool return_previous = false;
4844    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4845       return_previous = true;
4846       break;
4847    }
4848    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4849       return_previous = true;
4850       break;
4851    }
4852
4853    Builder bld(ctx->program, ctx->block);
4854    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4855
4856    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4857       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4858                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4859
4860    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4861    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4862    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4863
4864    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4865
4866    aco_opcode op32, op64;
4867    switch (instr->intrinsic) {
4868       case nir_intrinsic_ssbo_atomic_add:
4869          op32 = aco_opcode::buffer_atomic_add;
4870          op64 = aco_opcode::buffer_atomic_add_x2;
4871          break;
4872       case nir_intrinsic_ssbo_atomic_imin:
4873          op32 = aco_opcode::buffer_atomic_smin;
4874          op64 = aco_opcode::buffer_atomic_smin_x2;
4875          break;
4876       case nir_intrinsic_ssbo_atomic_umin:
4877          op32 = aco_opcode::buffer_atomic_umin;
4878          op64 = aco_opcode::buffer_atomic_umin_x2;
4879          break;
4880       case nir_intrinsic_ssbo_atomic_imax:
4881          op32 = aco_opcode::buffer_atomic_smax;
4882          op64 = aco_opcode::buffer_atomic_smax_x2;
4883          break;
4884       case nir_intrinsic_ssbo_atomic_umax:
4885          op32 = aco_opcode::buffer_atomic_umax;
4886          op64 = aco_opcode::buffer_atomic_umax_x2;
4887          break;
4888       case nir_intrinsic_ssbo_atomic_and:
4889          op32 = aco_opcode::buffer_atomic_and;
4890          op64 = aco_opcode::buffer_atomic_and_x2;
4891          break;
4892       case nir_intrinsic_ssbo_atomic_or:
4893          op32 = aco_opcode::buffer_atomic_or;
4894          op64 = aco_opcode::buffer_atomic_or_x2;
4895          break;
4896       case nir_intrinsic_ssbo_atomic_xor:
4897          op32 = aco_opcode::buffer_atomic_xor;
4898          op64 = aco_opcode::buffer_atomic_xor_x2;
4899          break;
4900       case nir_intrinsic_ssbo_atomic_exchange:
4901          op32 = aco_opcode::buffer_atomic_swap;
4902          op64 = aco_opcode::buffer_atomic_swap_x2;
4903          break;
4904       case nir_intrinsic_ssbo_atomic_comp_swap:
4905          op32 = aco_opcode::buffer_atomic_cmpswap;
4906          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4907          break;
4908       default:
4909          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4910    }
4911    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4912    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4913    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4914    mubuf->operands[1] = Operand(rsrc);
4915    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4916    mubuf->operands[3] = Operand(data);
4917    if (return_previous)
4918       mubuf->definitions[0] = Definition(dst);
4919    mubuf->offset = 0;
4920    mubuf->offen = (offset.type() == RegType::vgpr);
4921    mubuf->glc = return_previous;
4922    mubuf->dlc = false; /* Not needed for atomics */
4923    mubuf->disable_wqm = true;
4924    mubuf->barrier = barrier_buffer;
4925    ctx->program->needs_exact = true;
4926    ctx->block->instructions.emplace_back(std::move(mubuf));
4927 }
4928
4929 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4930
4931    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4932    Builder bld(ctx->program, ctx->block);
4933    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4934    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4935 }
4936
4937 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4938 {
4939    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4940                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4941
4942    if (addr.type() == RegType::vgpr)
4943       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4944    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4945 }
4946
4947 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4948 {
4949    Builder bld(ctx->program, ctx->block);
4950    unsigned num_components = instr->num_components;
4951    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4952
4953    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4954    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4955
4956    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4957    bool dlc = glc && ctx->options->chip_class >= GFX10;
4958    aco_opcode op;
4959    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4960       bool global = ctx->options->chip_class >= GFX9;
4961
4962       if (ctx->options->chip_class >= GFX7) {
4963          aco_opcode op;
4964          switch (num_bytes) {
4965          case 4:
4966             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4967             break;
4968          case 8:
4969             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4970             break;
4971          case 12:
4972             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4973             break;
4974          case 16:
4975             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4976             break;
4977          default:
4978             unreachable("load_global not implemented for this size.");
4979          }
4980
4981          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4982          flat->operands[0] = Operand(addr);
4983          flat->operands[1] = Operand(s1);
4984          flat->glc = glc;
4985          flat->dlc = dlc;
4986          flat->barrier = barrier_buffer;
4987
4988          if (dst.type() == RegType::sgpr) {
4989             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4990             flat->definitions[0] = Definition(vec);
4991             ctx->block->instructions.emplace_back(std::move(flat));
4992             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4993          } else {
4994             flat->definitions[0] = Definition(dst);
4995             ctx->block->instructions.emplace_back(std::move(flat));
4996          }
4997          emit_split_vector(ctx, dst, num_components);
4998       } else {
4999          assert(ctx->options->chip_class == GFX6);
5000
5001          /* GFX6 doesn't support loading vec3, expand to vec4. */
5002          num_bytes = num_bytes == 12 ? 16 : num_bytes;
5003
5004          aco_opcode op;
5005          switch (num_bytes) {
5006          case 4:
5007             op = aco_opcode::buffer_load_dword;
5008             break;
5009          case 8:
5010             op = aco_opcode::buffer_load_dwordx2;
5011             break;
5012          case 16:
5013             op = aco_opcode::buffer_load_dwordx4;
5014             break;
5015          default:
5016             unreachable("load_global not implemented for this size.");
5017          }
5018
5019          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5020
5021          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
5022          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5023          mubuf->operands[1] = Operand(rsrc);
5024          mubuf->operands[2] = Operand(0u);
5025          mubuf->glc = glc;
5026          mubuf->dlc = false;
5027          mubuf->offset = 0;
5028          mubuf->addr64 = addr.type() == RegType::vgpr;
5029          mubuf->disable_wqm = false;
5030          mubuf->barrier = barrier_buffer;
5031          aco_ptr<Instruction> instr = std::move(mubuf);
5032
5033          /* expand vector */
5034          if (dst.size() == 3) {
5035             Temp vec = bld.tmp(v4);
5036             instr->definitions[0] = Definition(vec);
5037             bld.insert(std::move(instr));
5038             emit_split_vector(ctx, vec, 4);
5039
5040             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
5041             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
5042             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
5043             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
5044          }
5045
5046          if (dst.type() == RegType::sgpr) {
5047             Temp vec = bld.tmp(RegType::vgpr, dst.size());
5048             instr->definitions[0] = Definition(vec);
5049             bld.insert(std::move(instr));
5050             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
5051             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
5052          } else {
5053             instr->definitions[0] = Definition(dst);
5054             bld.insert(std::move(instr));
5055             emit_split_vector(ctx, dst, num_components);
5056          }
5057       }
5058    } else {
5059       switch (num_bytes) {
5060          case 4:
5061             op = aco_opcode::s_load_dword;
5062             break;
5063          case 8:
5064             op = aco_opcode::s_load_dwordx2;
5065             break;
5066          case 12:
5067          case 16:
5068             op = aco_opcode::s_load_dwordx4;
5069             break;
5070          default:
5071             unreachable("load_global not implemented for this size.");
5072       }
5073       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
5074       load->operands[0] = Operand(addr);
5075       load->operands[1] = Operand(0u);
5076       load->definitions[0] = Definition(dst);
5077       load->glc = glc;
5078       load->dlc = dlc;
5079       load->barrier = barrier_buffer;
5080       assert(ctx->options->chip_class >= GFX8 || !glc);
5081
5082       if (dst.size() == 3) {
5083          /* trim vector */
5084          Temp vec = bld.tmp(s4);
5085          load->definitions[0] = Definition(vec);
5086          ctx->block->instructions.emplace_back(std::move(load));
5087          emit_split_vector(ctx, vec, 4);
5088
5089          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5090                     emit_extract_vector(ctx, vec, 0, s1),
5091                     emit_extract_vector(ctx, vec, 1, s1),
5092                     emit_extract_vector(ctx, vec, 2, s1));
5093       } else {
5094          ctx->block->instructions.emplace_back(std::move(load));
5095       }
5096    }
5097 }
5098
5099 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
5100 {
5101    Builder bld(ctx->program, ctx->block);
5102    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5103
5104    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5105    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5106
5107    if (ctx->options->chip_class >= GFX7)
5108       addr = as_vgpr(ctx, addr);
5109
5110    unsigned writemask = nir_intrinsic_write_mask(instr);
5111    while (writemask) {
5112       int start, count;
5113       u_bit_scan_consecutive_range(&writemask, &start, &count);
5114       if (count == 3 && ctx->options->chip_class == GFX6) {
5115          /* GFX6 doesn't support storing vec3, split it. */
5116          writemask |= 1u << (start + 2);
5117          count = 2;
5118       }
5119       unsigned num_bytes = count * elem_size_bytes;
5120
5121       Temp write_data = data;
5122       if (count != instr->num_components) {
5123          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5124          for (int i = 0; i < count; i++)
5125             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5126          write_data = bld.tmp(RegType::vgpr, count);
5127          vec->definitions[0] = Definition(write_data);
5128          ctx->block->instructions.emplace_back(std::move(vec));
5129       }
5130
5131       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5132       unsigned offset = start * elem_size_bytes;
5133
5134       if (ctx->options->chip_class >= GFX7) {
5135          if (offset > 0 && ctx->options->chip_class < GFX9) {
5136             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5137             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5138             Temp carry = bld.tmp(bld.lm);
5139             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5140
5141             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5142                      Operand(offset), addr0);
5143             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5144                      Operand(0u), addr1,
5145                      carry).def(1).setHint(vcc);
5146
5147             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5148
5149             offset = 0;
5150          }
5151
5152          bool global = ctx->options->chip_class >= GFX9;
5153          aco_opcode op;
5154          switch (num_bytes) {
5155          case 4:
5156             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5157             break;
5158          case 8:
5159             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5160             break;
5161          case 12:
5162             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5163             break;
5164          case 16:
5165             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5166             break;
5167          default:
5168             unreachable("store_global not implemented for this size.");
5169          }
5170
5171          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5172          flat->operands[0] = Operand(addr);
5173          flat->operands[1] = Operand(s1);
5174          flat->operands[2] = Operand(data);
5175          flat->glc = glc;
5176          flat->dlc = false;
5177          flat->offset = offset;
5178          flat->disable_wqm = true;
5179          flat->barrier = barrier_buffer;
5180          ctx->program->needs_exact = true;
5181          ctx->block->instructions.emplace_back(std::move(flat));
5182       } else {
5183          assert(ctx->options->chip_class == GFX6);
5184
5185          aco_opcode op;
5186          switch (num_bytes) {
5187          case 4:
5188             op = aco_opcode::buffer_store_dword;
5189             break;
5190          case 8:
5191             op = aco_opcode::buffer_store_dwordx2;
5192             break;
5193          case 16:
5194             op = aco_opcode::buffer_store_dwordx4;
5195             break;
5196          default:
5197             unreachable("store_global not implemented for this size.");
5198          }
5199
5200          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5201
5202          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5203          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5204          mubuf->operands[1] = Operand(rsrc);
5205          mubuf->operands[2] = Operand(0u);
5206          mubuf->operands[3] = Operand(write_data);
5207          mubuf->glc = glc;
5208          mubuf->dlc = false;
5209          mubuf->offset = offset;
5210          mubuf->addr64 = addr.type() == RegType::vgpr;
5211          mubuf->disable_wqm = true;
5212          mubuf->barrier = barrier_buffer;
5213          ctx->program->needs_exact = true;
5214          ctx->block->instructions.emplace_back(std::move(mubuf));
5215       }
5216    }
5217 }
5218
5219 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5220 {
5221    /* return the previous value if dest is ever used */
5222    bool return_previous = false;
5223    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5224       return_previous = true;
5225       break;
5226    }
5227    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5228       return_previous = true;
5229       break;
5230    }
5231
5232    Builder bld(ctx->program, ctx->block);
5233    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5234    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5235
5236    if (ctx->options->chip_class >= GFX7)
5237       addr = as_vgpr(ctx, addr);
5238
5239    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5240       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5241                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5242
5243    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5244
5245    aco_opcode op32, op64;
5246
5247    if (ctx->options->chip_class >= GFX7) {
5248       bool global = ctx->options->chip_class >= GFX9;
5249       switch (instr->intrinsic) {
5250          case nir_intrinsic_global_atomic_add:
5251             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5252             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5253             break;
5254          case nir_intrinsic_global_atomic_imin:
5255             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5256             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5257             break;
5258          case nir_intrinsic_global_atomic_umin:
5259             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5260             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5261             break;
5262          case nir_intrinsic_global_atomic_imax:
5263             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5264             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5265             break;
5266          case nir_intrinsic_global_atomic_umax:
5267             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5268             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5269             break;
5270          case nir_intrinsic_global_atomic_and:
5271             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5272             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5273             break;
5274          case nir_intrinsic_global_atomic_or:
5275             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5276             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5277             break;
5278          case nir_intrinsic_global_atomic_xor:
5279             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5280             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5281             break;
5282          case nir_intrinsic_global_atomic_exchange:
5283             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5284             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5285             break;
5286          case nir_intrinsic_global_atomic_comp_swap:
5287             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5288             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5289             break;
5290          default:
5291             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5292       }
5293
5294       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5295       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5296       flat->operands[0] = Operand(addr);
5297       flat->operands[1] = Operand(s1);
5298       flat->operands[2] = Operand(data);
5299       if (return_previous)
5300          flat->definitions[0] = Definition(dst);
5301       flat->glc = return_previous;
5302       flat->dlc = false; /* Not needed for atomics */
5303       flat->offset = 0;
5304       flat->disable_wqm = true;
5305       flat->barrier = barrier_buffer;
5306       ctx->program->needs_exact = true;
5307       ctx->block->instructions.emplace_back(std::move(flat));
5308    } else {
5309       assert(ctx->options->chip_class == GFX6);
5310
5311       switch (instr->intrinsic) {
5312          case nir_intrinsic_global_atomic_add:
5313             op32 = aco_opcode::buffer_atomic_add;
5314             op64 = aco_opcode::buffer_atomic_add_x2;
5315             break;
5316          case nir_intrinsic_global_atomic_imin:
5317             op32 = aco_opcode::buffer_atomic_smin;
5318             op64 = aco_opcode::buffer_atomic_smin_x2;
5319             break;
5320          case nir_intrinsic_global_atomic_umin:
5321             op32 = aco_opcode::buffer_atomic_umin;
5322             op64 = aco_opcode::buffer_atomic_umin_x2;
5323             break;
5324          case nir_intrinsic_global_atomic_imax:
5325             op32 = aco_opcode::buffer_atomic_smax;
5326             op64 = aco_opcode::buffer_atomic_smax_x2;
5327             break;
5328          case nir_intrinsic_global_atomic_umax:
5329             op32 = aco_opcode::buffer_atomic_umax;
5330             op64 = aco_opcode::buffer_atomic_umax_x2;
5331             break;
5332          case nir_intrinsic_global_atomic_and:
5333             op32 = aco_opcode::buffer_atomic_and;
5334             op64 = aco_opcode::buffer_atomic_and_x2;
5335             break;
5336          case nir_intrinsic_global_atomic_or:
5337             op32 = aco_opcode::buffer_atomic_or;
5338             op64 = aco_opcode::buffer_atomic_or_x2;
5339             break;
5340          case nir_intrinsic_global_atomic_xor:
5341             op32 = aco_opcode::buffer_atomic_xor;
5342             op64 = aco_opcode::buffer_atomic_xor_x2;
5343             break;
5344          case nir_intrinsic_global_atomic_exchange:
5345             op32 = aco_opcode::buffer_atomic_swap;
5346             op64 = aco_opcode::buffer_atomic_swap_x2;
5347             break;
5348          case nir_intrinsic_global_atomic_comp_swap:
5349             op32 = aco_opcode::buffer_atomic_cmpswap;
5350             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5351             break;
5352          default:
5353             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5354       }
5355
5356       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5357
5358       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5359
5360       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5361       mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5362       mubuf->operands[1] = Operand(rsrc);
5363       mubuf->operands[2] = Operand(0u);
5364       mubuf->operands[3] = Operand(data);
5365       if (return_previous)
5366          mubuf->definitions[0] = Definition(dst);
5367       mubuf->glc = return_previous;
5368       mubuf->dlc = false;
5369       mubuf->offset = 0;
5370       mubuf->addr64 = addr.type() == RegType::vgpr;
5371       mubuf->disable_wqm = true;
5372       mubuf->barrier = barrier_buffer;
5373       ctx->program->needs_exact = true;
5374       ctx->block->instructions.emplace_back(std::move(mubuf));
5375    }
5376 }
5377
5378 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5379    Builder bld(ctx->program, ctx->block);
5380    switch(instr->intrinsic) {
5381       case nir_intrinsic_group_memory_barrier:
5382       case nir_intrinsic_memory_barrier:
5383          bld.barrier(aco_opcode::p_memory_barrier_common);
5384          break;
5385       case nir_intrinsic_memory_barrier_buffer:
5386          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5387          break;
5388       case nir_intrinsic_memory_barrier_image:
5389          bld.barrier(aco_opcode::p_memory_barrier_image);
5390          break;
5391       case nir_intrinsic_memory_barrier_shared:
5392          bld.barrier(aco_opcode::p_memory_barrier_shared);
5393          break;
5394       default:
5395          unreachable("Unimplemented memory barrier intrinsic");
5396          break;
5397    }
5398 }
5399
5400 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5401 {
5402    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5403    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5404    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5405    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5406    Builder bld(ctx->program, ctx->block);
5407
5408    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5409    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5410    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5411 }
5412
5413 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5414 {
5415    unsigned writemask = nir_intrinsic_write_mask(instr);
5416    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5417    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5418    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5419    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5420
5421    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5422    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5423 }
5424
5425 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5426 {
5427    unsigned offset = nir_intrinsic_base(instr);
5428    Operand m = load_lds_size_m0(ctx);
5429    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5430    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5431
5432    unsigned num_operands = 3;
5433    aco_opcode op32, op64, op32_rtn, op64_rtn;
5434    switch(instr->intrinsic) {
5435       case nir_intrinsic_shared_atomic_add:
5436          op32 = aco_opcode::ds_add_u32;
5437          op64 = aco_opcode::ds_add_u64;
5438          op32_rtn = aco_opcode::ds_add_rtn_u32;
5439          op64_rtn = aco_opcode::ds_add_rtn_u64;
5440          break;
5441       case nir_intrinsic_shared_atomic_imin:
5442          op32 = aco_opcode::ds_min_i32;
5443          op64 = aco_opcode::ds_min_i64;
5444          op32_rtn = aco_opcode::ds_min_rtn_i32;
5445          op64_rtn = aco_opcode::ds_min_rtn_i64;
5446          break;
5447       case nir_intrinsic_shared_atomic_umin:
5448          op32 = aco_opcode::ds_min_u32;
5449          op64 = aco_opcode::ds_min_u64;
5450          op32_rtn = aco_opcode::ds_min_rtn_u32;
5451          op64_rtn = aco_opcode::ds_min_rtn_u64;
5452          break;
5453       case nir_intrinsic_shared_atomic_imax:
5454          op32 = aco_opcode::ds_max_i32;
5455          op64 = aco_opcode::ds_max_i64;
5456          op32_rtn = aco_opcode::ds_max_rtn_i32;
5457          op64_rtn = aco_opcode::ds_max_rtn_i64;
5458          break;
5459       case nir_intrinsic_shared_atomic_umax:
5460          op32 = aco_opcode::ds_max_u32;
5461          op64 = aco_opcode::ds_max_u64;
5462          op32_rtn = aco_opcode::ds_max_rtn_u32;
5463          op64_rtn = aco_opcode::ds_max_rtn_u64;
5464          break;
5465       case nir_intrinsic_shared_atomic_and:
5466          op32 = aco_opcode::ds_and_b32;
5467          op64 = aco_opcode::ds_and_b64;
5468          op32_rtn = aco_opcode::ds_and_rtn_b32;
5469          op64_rtn = aco_opcode::ds_and_rtn_b64;
5470          break;
5471       case nir_intrinsic_shared_atomic_or:
5472          op32 = aco_opcode::ds_or_b32;
5473          op64 = aco_opcode::ds_or_b64;
5474          op32_rtn = aco_opcode::ds_or_rtn_b32;
5475          op64_rtn = aco_opcode::ds_or_rtn_b64;
5476          break;
5477       case nir_intrinsic_shared_atomic_xor:
5478          op32 = aco_opcode::ds_xor_b32;
5479          op64 = aco_opcode::ds_xor_b64;
5480          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5481          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5482          break;
5483       case nir_intrinsic_shared_atomic_exchange:
5484          op32 = aco_opcode::ds_write_b32;
5485          op64 = aco_opcode::ds_write_b64;
5486          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5487          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5488          break;
5489       case nir_intrinsic_shared_atomic_comp_swap:
5490          op32 = aco_opcode::ds_cmpst_b32;
5491          op64 = aco_opcode::ds_cmpst_b64;
5492          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5493          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5494          num_operands = 4;
5495          break;
5496       default:
5497          unreachable("Unhandled shared atomic intrinsic");
5498    }
5499
5500    /* return the previous value if dest is ever used */
5501    bool return_previous = false;
5502    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5503       return_previous = true;
5504       break;
5505    }
5506    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5507       return_previous = true;
5508       break;
5509    }
5510
5511    aco_opcode op;
5512    if (data.size() == 1) {
5513       assert(instr->dest.ssa.bit_size == 32);
5514       op = return_previous ? op32_rtn : op32;
5515    } else {
5516       assert(instr->dest.ssa.bit_size == 64);
5517       op = return_previous ? op64_rtn : op64;
5518    }
5519
5520    if (offset > 65535) {
5521       Builder bld(ctx->program, ctx->block);
5522       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5523       offset = 0;
5524    }
5525
5526    aco_ptr<DS_instruction> ds;
5527    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5528    ds->operands[0] = Operand(address);
5529    ds->operands[1] = Operand(data);
5530    if (num_operands == 4)
5531       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5532    ds->operands[num_operands - 1] = m;
5533    ds->offset0 = offset;
5534    if (return_previous)
5535       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5536    ctx->block->instructions.emplace_back(std::move(ds));
5537 }
5538
5539 Temp get_scratch_resource(isel_context *ctx)
5540 {
5541    Builder bld(ctx->program, ctx->block);
5542    Temp scratch_addr = ctx->program->private_segment_buffer;
5543    if (ctx->stage != compute_cs)
5544       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5545
5546    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5547                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5548
5549    if (ctx->program->chip_class >= GFX10) {
5550       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5551                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5552                    S_008F0C_RESOURCE_LEVEL(1);
5553    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5554       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5555                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5556    }
5557
5558    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5559    if (ctx->program->chip_class <= GFX8)
5560       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5561
5562    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5563 }
5564
5565 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5566    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5567    Builder bld(ctx->program, ctx->block);
5568    Temp rsrc = get_scratch_resource(ctx);
5569    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5570    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5571
5572    aco_opcode op;
5573    switch (dst.size()) {
5574       case 1:
5575          op = aco_opcode::buffer_load_dword;
5576          break;
5577       case 2:
5578          op = aco_opcode::buffer_load_dwordx2;
5579          break;
5580       case 3:
5581          op = aco_opcode::buffer_load_dwordx3;
5582          break;
5583       case 4:
5584          op = aco_opcode::buffer_load_dwordx4;
5585          break;
5586       case 6:
5587       case 8: {
5588          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5589          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5590                                 bld.def(v4), offset, rsrc,
5591                                 ctx->program->scratch_offset, 0, true);
5592          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5593                                                   aco_opcode::buffer_load_dwordx4,
5594                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5595                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5596          emit_split_vector(ctx, lower, 2);
5597          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5598          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5599          if (dst.size() == 8) {
5600             emit_split_vector(ctx, upper, 2);
5601             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5602             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5603          } else {
5604             elems[2] = upper;
5605          }
5606
5607          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5608                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5609          for (unsigned i = 0; i < dst.size() / 2; i++)
5610             vec->operands[i] = Operand(elems[i]);
5611          vec->definitions[0] = Definition(dst);
5612          bld.insert(std::move(vec));
5613          ctx->allocated_vec.emplace(dst.id(), elems);
5614          return;
5615       }
5616       default:
5617          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5618    }
5619
5620    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5621    emit_split_vector(ctx, dst, instr->num_components);
5622 }
5623
5624 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5625    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5626    Builder bld(ctx->program, ctx->block);
5627    Temp rsrc = get_scratch_resource(ctx);
5628    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5629    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5630
5631    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5632    unsigned writemask = nir_intrinsic_write_mask(instr);
5633
5634    while (writemask) {
5635       int start, count;
5636       u_bit_scan_consecutive_range(&writemask, &start, &count);
5637       int num_bytes = count * elem_size_bytes;
5638
5639       if (num_bytes > 16) {
5640          assert(elem_size_bytes == 8);
5641          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5642          count = 2;
5643          num_bytes = 16;
5644       }
5645
5646       // TODO: check alignment of sub-dword stores
5647       // TODO: split 3 bytes. there is no store instruction for that
5648
5649       Temp write_data;
5650       if (count != instr->num_components) {
5651          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5652          for (int i = 0; i < count; i++) {
5653             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5654             vec->operands[i] = Operand(elem);
5655          }
5656          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5657          vec->definitions[0] = Definition(write_data);
5658          ctx->block->instructions.emplace_back(std::move(vec));
5659       } else {
5660          write_data = data;
5661       }
5662
5663       aco_opcode op;
5664       switch (num_bytes) {
5665          case 4:
5666             op = aco_opcode::buffer_store_dword;
5667             break;
5668          case 8:
5669             op = aco_opcode::buffer_store_dwordx2;
5670             break;
5671          case 12:
5672             op = aco_opcode::buffer_store_dwordx3;
5673             break;
5674          case 16:
5675             op = aco_opcode::buffer_store_dwordx4;
5676             break;
5677          default:
5678             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5679       }
5680
5681       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5682    }
5683 }
5684
5685 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5686    uint8_t log2_ps_iter_samples;
5687    if (ctx->program->info->ps.force_persample) {
5688       log2_ps_iter_samples =
5689          util_logbase2(ctx->options->key.fs.num_samples);
5690    } else {
5691       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5692    }
5693
5694    /* The bit pattern matches that used by fixed function fragment
5695     * processing. */
5696    static const unsigned ps_iter_masks[] = {
5697       0xffff, /* not used */
5698       0x5555,
5699       0x1111,
5700       0x0101,
5701       0x0001,
5702    };
5703    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5704
5705    Builder bld(ctx->program, ctx->block);
5706
5707    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5708                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5709    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5710    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5711    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5712    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5713 }
5714
5715 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
5716    Builder bld(ctx->program, ctx->block);
5717
5718    unsigned stream = nir_intrinsic_stream_id(instr);
5719    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5720    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
5721    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
5722
5723    /* get GSVS ring */
5724    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
5725
5726    unsigned num_components =
5727       ctx->program->info->gs.num_stream_output_components[stream];
5728    assert(num_components);
5729
5730    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
5731    unsigned stream_offset = 0;
5732    for (unsigned i = 0; i < stream; i++) {
5733       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
5734       stream_offset += prev_stride * ctx->program->wave_size;
5735    }
5736
5737    /* Limit on the stride field for <= GFX7. */
5738    assert(stride < (1 << 14));
5739
5740    Temp gsvs_dwords[4];
5741    for (unsigned i = 0; i < 4; i++)
5742       gsvs_dwords[i] = bld.tmp(s1);
5743    bld.pseudo(aco_opcode::p_split_vector,
5744               Definition(gsvs_dwords[0]),
5745               Definition(gsvs_dwords[1]),
5746               Definition(gsvs_dwords[2]),
5747               Definition(gsvs_dwords[3]),
5748               gsvs_ring);
5749
5750    if (stream_offset) {
5751       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
5752
5753       Temp carry = bld.tmp(s1);
5754       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
5755       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
5756    }
5757
5758    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
5759    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
5760
5761    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5762                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
5763
5764    unsigned offset = 0;
5765    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
5766       if (ctx->program->info->gs.output_streams[i] != stream)
5767          continue;
5768
5769       for (unsigned j = 0; j < 4; j++) {
5770          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
5771             continue;
5772
5773          if (ctx->outputs.mask[i] & (1 << j)) {
5774             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
5775             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
5776             if (const_offset >= 4096u) {
5777                if (vaddr_offset.isUndefined())
5778                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
5779                else
5780                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
5781                const_offset %= 4096u;
5782             }
5783
5784             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
5785             mtbuf->operands[0] = vaddr_offset;
5786             mtbuf->operands[1] = Operand(gsvs_ring);
5787             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
5788             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
5789             mtbuf->offen = !vaddr_offset.isUndefined();
5790             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
5791             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
5792             mtbuf->offset = const_offset;
5793             mtbuf->glc = true;
5794             mtbuf->slc = true;
5795             mtbuf->barrier = barrier_gs_data;
5796             mtbuf->can_reorder = true;
5797             bld.insert(std::move(mtbuf));
5798          }
5799
5800          offset += ctx->shader->info.gs.vertices_out;
5801       }
5802
5803       /* outputs for the next vertex are undefined and keeping them around can
5804        * create invalid IR with control flow */
5805       ctx->outputs.mask[i] = 0;
5806    }
5807
5808    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
5809 }
5810
5811 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5812 {
5813    Builder bld(ctx->program, ctx->block);
5814
5815    if (cluster_size == 1) {
5816       return src;
5817    } if (op == nir_op_iand && cluster_size == 4) {
5818       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5819       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5820       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5821                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5822    } else if (op == nir_op_ior && cluster_size == 4) {
5823       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5824       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5825                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5826    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5827       //subgroupAnd(val) -> (exec & ~val) == 0
5828       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5829       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5830       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5831    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5832       //subgroupOr(val) -> (val & exec) != 0
5833       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5834       return bool_to_vector_condition(ctx, tmp);
5835    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5836       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5837       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5838       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5839       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5840       return bool_to_vector_condition(ctx, tmp);
5841    } else {
5842       //subgroupClustered{And,Or,Xor}(val, n) ->
5843       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5844       //cluster_offset = ~(n - 1) & lane_id
5845       //cluster_mask = ((1 << n) - 1)
5846       //subgroupClusteredAnd():
5847       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5848       //subgroupClusteredOr():
5849       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5850       //subgroupClusteredXor():
5851       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5852       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5853       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5854
5855       Temp tmp;
5856       if (op == nir_op_iand)
5857          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5858       else
5859          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5860
5861       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5862
5863       if (ctx->program->chip_class <= GFX7)
5864          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5865       else if (ctx->program->wave_size == 64)
5866          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5867       else
5868          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5869       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5870       if (cluster_mask != 0xffffffff)
5871          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5872
5873       Definition cmp_def = Definition();
5874       if (op == nir_op_iand) {
5875          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5876       } else if (op == nir_op_ior) {
5877          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5878       } else if (op == nir_op_ixor) {
5879          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5880                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5881          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5882       }
5883       cmp_def.setHint(vcc);
5884       return cmp_def.getTemp();
5885    }
5886 }
5887
5888 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5889 {
5890    Builder bld(ctx->program, ctx->block);
5891
5892    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5893    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5894    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5895    Temp tmp;
5896    if (op == nir_op_iand)
5897       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5898    else
5899       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5900
5901    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5902    Temp lo = lohi.def(0).getTemp();
5903    Temp hi = lohi.def(1).getTemp();
5904    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5905
5906    Definition cmp_def = Definition();
5907    if (op == nir_op_iand)
5908       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5909    else if (op == nir_op_ior)
5910       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5911    else if (op == nir_op_ixor)
5912       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5913                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5914    cmp_def.setHint(vcc);
5915    return cmp_def.getTemp();
5916 }
5917
5918 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5919 {
5920    Builder bld(ctx->program, ctx->block);
5921
5922    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5923    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5924    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5925    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5926    if (op == nir_op_iand)
5927       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5928    else if (op == nir_op_ior)
5929       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5930    else if (op == nir_op_ixor)
5931       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5932
5933    assert(false);
5934    return Temp();
5935 }
5936
5937 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5938 {
5939    Builder bld(ctx->program, ctx->block);
5940    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5941    if (src.regClass().type() == RegType::vgpr) {
5942       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5943    } else if (src.regClass() == s1) {
5944       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5945    } else if (src.regClass() == s2) {
5946       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5947    } else {
5948       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5949       nir_print_instr(&instr->instr, stderr);
5950       fprintf(stderr, "\n");
5951    }
5952 }
5953
5954 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5955 {
5956    Builder bld(ctx->program, ctx->block);
5957    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5958    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5959    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5960
5961    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5962    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5963    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5964    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5965
5966    /* Build DD X/Y */
5967    if (ctx->program->chip_class >= GFX8) {
5968       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5969       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5970       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5971       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5972       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5973       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5974    } else {
5975       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5976       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5977       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5978       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5979       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5980       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5981       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5982       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5983       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5984       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5985    }
5986
5987    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5988    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5989    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5990    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5991    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5992    Temp wqm1 = bld.tmp(v1);
5993    emit_wqm(ctx, tmp1, wqm1, true);
5994    Temp wqm2 = bld.tmp(v1);
5995    emit_wqm(ctx, tmp2, wqm2, true);
5996    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5997    return;
5998 }
5999
6000 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
6001 {
6002    Builder bld(ctx->program, ctx->block);
6003    switch(instr->intrinsic) {
6004    case nir_intrinsic_load_barycentric_sample:
6005    case nir_intrinsic_load_barycentric_pixel:
6006    case nir_intrinsic_load_barycentric_centroid: {
6007       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
6008       Temp bary = Temp(0, s2);
6009       switch (mode) {
6010       case INTERP_MODE_SMOOTH:
6011       case INTERP_MODE_NONE:
6012          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6013             bary = get_arg(ctx, ctx->args->ac.persp_center);
6014          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6015             bary = ctx->persp_centroid;
6016          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6017             bary = get_arg(ctx, ctx->args->ac.persp_sample);
6018          break;
6019       case INTERP_MODE_NOPERSPECTIVE:
6020          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
6021             bary = get_arg(ctx, ctx->args->ac.linear_center);
6022          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
6023             bary = ctx->linear_centroid;
6024          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
6025             bary = get_arg(ctx, ctx->args->ac.linear_sample);
6026          break;
6027       default:
6028          break;
6029       }
6030       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6031       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
6032       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
6033       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6034                  Operand(p1), Operand(p2));
6035       emit_split_vector(ctx, dst, 2);
6036       break;
6037    }
6038    case nir_intrinsic_load_barycentric_model: {
6039       Temp model = get_arg(ctx, ctx->args->ac.pull_model);
6040
6041       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6042       Temp p1 = emit_extract_vector(ctx, model, 0, v1);
6043       Temp p2 = emit_extract_vector(ctx, model, 1, v1);
6044       Temp p3 = emit_extract_vector(ctx, model, 2, v1);
6045       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6046                  Operand(p1), Operand(p2), Operand(p3));
6047       emit_split_vector(ctx, dst, 3);
6048       break;
6049    }
6050    case nir_intrinsic_load_barycentric_at_sample: {
6051       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
6052       switch (ctx->options->key.fs.num_samples) {
6053          case 2: sample_pos_offset += 1 << 3; break;
6054          case 4: sample_pos_offset += 3 << 3; break;
6055          case 8: sample_pos_offset += 7 << 3; break;
6056          default: break;
6057       }
6058       Temp sample_pos;
6059       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
6060       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
6061       Temp private_segment_buffer = ctx->program->private_segment_buffer;
6062       if (addr.type() == RegType::sgpr) {
6063          Operand offset;
6064          if (const_addr) {
6065             sample_pos_offset += const_addr->u32 << 3;
6066             offset = Operand(sample_pos_offset);
6067          } else if (ctx->options->chip_class >= GFX9) {
6068             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6069          } else {
6070             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
6071             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
6072          }
6073
6074          Operand off = bld.copy(bld.def(s1), Operand(offset));
6075          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
6076
6077       } else if (ctx->options->chip_class >= GFX9) {
6078          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6079          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
6080       } else if (ctx->options->chip_class >= GFX7) {
6081          /* addr += private_segment_buffer + sample_pos_offset */
6082          Temp tmp0 = bld.tmp(s1);
6083          Temp tmp1 = bld.tmp(s1);
6084          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
6085          Definition scc_tmp = bld.def(s1, scc);
6086          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
6087          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
6088          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6089          Temp pck0 = bld.tmp(v1);
6090          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
6091          tmp1 = as_vgpr(ctx, tmp1);
6092          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
6093          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
6094
6095          /* sample_pos = flat_load_dwordx2 addr */
6096          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
6097       } else {
6098          assert(ctx->options->chip_class == GFX6);
6099
6100          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
6101                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
6102          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
6103
6104          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
6105          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
6106
6107          sample_pos = bld.tmp(v2);
6108
6109          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
6110          load->definitions[0] = Definition(sample_pos);
6111          load->operands[0] = Operand(addr);
6112          load->operands[1] = Operand(rsrc);
6113          load->operands[2] = Operand(0u);
6114          load->offset = sample_pos_offset;
6115          load->offen = 0;
6116          load->addr64 = true;
6117          load->glc = false;
6118          load->dlc = false;
6119          load->disable_wqm = false;
6120          load->barrier = barrier_none;
6121          load->can_reorder = true;
6122          ctx->block->instructions.emplace_back(std::move(load));
6123       }
6124
6125       /* sample_pos -= 0.5 */
6126       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6127       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6128       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6129       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6130       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6131
6132       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6133       break;
6134    }
6135    case nir_intrinsic_load_barycentric_at_offset: {
6136       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6137       RegClass rc = RegClass(offset.type(), 1);
6138       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6139       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6140       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6141       break;
6142    }
6143    case nir_intrinsic_load_front_face: {
6144       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6145                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6146       break;
6147    }
6148    case nir_intrinsic_load_view_index:
6149    case nir_intrinsic_load_layer_id: {
6150       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & (sw_vs | sw_gs))) {
6151          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6152          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6153          break;
6154       }
6155
6156       unsigned idx = nir_intrinsic_base(instr);
6157       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6158                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6159       break;
6160    }
6161    case nir_intrinsic_load_frag_coord: {
6162       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6163       break;
6164    }
6165    case nir_intrinsic_load_sample_pos: {
6166       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6167       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6168       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6169                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6170                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6171       break;
6172    }
6173    case nir_intrinsic_load_interpolated_input:
6174       visit_load_interpolated_input(ctx, instr);
6175       break;
6176    case nir_intrinsic_store_output:
6177       visit_store_output(ctx, instr);
6178       break;
6179    case nir_intrinsic_load_input:
6180    case nir_intrinsic_load_input_vertex:
6181       visit_load_input(ctx, instr);
6182       break;
6183    case nir_intrinsic_load_per_vertex_input:
6184       visit_load_per_vertex_input(ctx, instr);
6185       break;
6186    case nir_intrinsic_load_ubo:
6187       visit_load_ubo(ctx, instr);
6188       break;
6189    case nir_intrinsic_load_push_constant:
6190       visit_load_push_constant(ctx, instr);
6191       break;
6192    case nir_intrinsic_load_constant:
6193       visit_load_constant(ctx, instr);
6194       break;
6195    case nir_intrinsic_vulkan_resource_index:
6196       visit_load_resource(ctx, instr);
6197       break;
6198    case nir_intrinsic_discard:
6199       visit_discard(ctx, instr);
6200       break;
6201    case nir_intrinsic_discard_if:
6202       visit_discard_if(ctx, instr);
6203       break;
6204    case nir_intrinsic_load_shared:
6205       visit_load_shared(ctx, instr);
6206       break;
6207    case nir_intrinsic_store_shared:
6208       visit_store_shared(ctx, instr);
6209       break;
6210    case nir_intrinsic_shared_atomic_add:
6211    case nir_intrinsic_shared_atomic_imin:
6212    case nir_intrinsic_shared_atomic_umin:
6213    case nir_intrinsic_shared_atomic_imax:
6214    case nir_intrinsic_shared_atomic_umax:
6215    case nir_intrinsic_shared_atomic_and:
6216    case nir_intrinsic_shared_atomic_or:
6217    case nir_intrinsic_shared_atomic_xor:
6218    case nir_intrinsic_shared_atomic_exchange:
6219    case nir_intrinsic_shared_atomic_comp_swap:
6220       visit_shared_atomic(ctx, instr);
6221       break;
6222    case nir_intrinsic_image_deref_load:
6223       visit_image_load(ctx, instr);
6224       break;
6225    case nir_intrinsic_image_deref_store:
6226       visit_image_store(ctx, instr);
6227       break;
6228    case nir_intrinsic_image_deref_atomic_add:
6229    case nir_intrinsic_image_deref_atomic_umin:
6230    case nir_intrinsic_image_deref_atomic_imin:
6231    case nir_intrinsic_image_deref_atomic_umax:
6232    case nir_intrinsic_image_deref_atomic_imax:
6233    case nir_intrinsic_image_deref_atomic_and:
6234    case nir_intrinsic_image_deref_atomic_or:
6235    case nir_intrinsic_image_deref_atomic_xor:
6236    case nir_intrinsic_image_deref_atomic_exchange:
6237    case nir_intrinsic_image_deref_atomic_comp_swap:
6238       visit_image_atomic(ctx, instr);
6239       break;
6240    case nir_intrinsic_image_deref_size:
6241       visit_image_size(ctx, instr);
6242       break;
6243    case nir_intrinsic_load_ssbo:
6244       visit_load_ssbo(ctx, instr);
6245       break;
6246    case nir_intrinsic_store_ssbo:
6247       visit_store_ssbo(ctx, instr);
6248       break;
6249    case nir_intrinsic_load_global:
6250       visit_load_global(ctx, instr);
6251       break;
6252    case nir_intrinsic_store_global:
6253       visit_store_global(ctx, instr);
6254       break;
6255    case nir_intrinsic_global_atomic_add:
6256    case nir_intrinsic_global_atomic_imin:
6257    case nir_intrinsic_global_atomic_umin:
6258    case nir_intrinsic_global_atomic_imax:
6259    case nir_intrinsic_global_atomic_umax:
6260    case nir_intrinsic_global_atomic_and:
6261    case nir_intrinsic_global_atomic_or:
6262    case nir_intrinsic_global_atomic_xor:
6263    case nir_intrinsic_global_atomic_exchange:
6264    case nir_intrinsic_global_atomic_comp_swap:
6265       visit_global_atomic(ctx, instr);
6266       break;
6267    case nir_intrinsic_ssbo_atomic_add:
6268    case nir_intrinsic_ssbo_atomic_imin:
6269    case nir_intrinsic_ssbo_atomic_umin:
6270    case nir_intrinsic_ssbo_atomic_imax:
6271    case nir_intrinsic_ssbo_atomic_umax:
6272    case nir_intrinsic_ssbo_atomic_and:
6273    case nir_intrinsic_ssbo_atomic_or:
6274    case nir_intrinsic_ssbo_atomic_xor:
6275    case nir_intrinsic_ssbo_atomic_exchange:
6276    case nir_intrinsic_ssbo_atomic_comp_swap:
6277       visit_atomic_ssbo(ctx, instr);
6278       break;
6279    case nir_intrinsic_load_scratch:
6280       visit_load_scratch(ctx, instr);
6281       break;
6282    case nir_intrinsic_store_scratch:
6283       visit_store_scratch(ctx, instr);
6284       break;
6285    case nir_intrinsic_get_buffer_size:
6286       visit_get_buffer_size(ctx, instr);
6287       break;
6288    case nir_intrinsic_control_barrier: {
6289       unsigned* bsize = ctx->program->info->cs.block_size;
6290       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6291       if (workgroup_size > ctx->program->wave_size)
6292          bld.sopp(aco_opcode::s_barrier);
6293       break;
6294    }
6295    case nir_intrinsic_group_memory_barrier:
6296    case nir_intrinsic_memory_barrier:
6297    case nir_intrinsic_memory_barrier_buffer:
6298    case nir_intrinsic_memory_barrier_image:
6299    case nir_intrinsic_memory_barrier_shared:
6300       emit_memory_barrier(ctx, instr);
6301       break;
6302    case nir_intrinsic_memory_barrier_tcs_patch:
6303       break;
6304    case nir_intrinsic_load_num_work_groups: {
6305       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6306       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6307       emit_split_vector(ctx, dst, 3);
6308       break;
6309    }
6310    case nir_intrinsic_load_local_invocation_id: {
6311       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6312       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6313       emit_split_vector(ctx, dst, 3);
6314       break;
6315    }
6316    case nir_intrinsic_load_work_group_id: {
6317       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6318       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6319       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6320                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6321                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6322                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6323       emit_split_vector(ctx, dst, 3);
6324       break;
6325    }
6326    case nir_intrinsic_load_local_invocation_index: {
6327       Temp id = emit_mbcnt(ctx, bld.def(v1));
6328
6329       /* The tg_size bits [6:11] contain the subgroup id,
6330        * we need this multiplied by the wave size, and then OR the thread id to it.
6331        */
6332       if (ctx->program->wave_size == 64) {
6333          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6334          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6335                                 get_arg(ctx, ctx->args->ac.tg_size));
6336          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6337       } else {
6338          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6339          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6340                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6341          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6342       }
6343       break;
6344    }
6345    case nir_intrinsic_load_subgroup_id: {
6346       if (ctx->stage == compute_cs) {
6347          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6348                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6349       } else {
6350          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6351       }
6352       break;
6353    }
6354    case nir_intrinsic_load_subgroup_invocation: {
6355       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6356       break;
6357    }
6358    case nir_intrinsic_load_num_subgroups: {
6359       if (ctx->stage == compute_cs)
6360          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6361                   get_arg(ctx, ctx->args->ac.tg_size));
6362       else
6363          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6364       break;
6365    }
6366    case nir_intrinsic_ballot: {
6367       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6368       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6369       Definition tmp = bld.def(dst.regClass());
6370       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6371       if (instr->src[0].ssa->bit_size == 1) {
6372          assert(src.regClass() == bld.lm);
6373          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6374       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6375          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6376       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6377          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6378       } else {
6379          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6380          nir_print_instr(&instr->instr, stderr);
6381          fprintf(stderr, "\n");
6382       }
6383       if (dst.size() != bld.lm.size()) {
6384          /* Wave32 with ballot size set to 64 */
6385          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6386       }
6387       emit_wqm(ctx, tmp.getTemp(), dst);
6388       break;
6389    }
6390    case nir_intrinsic_shuffle:
6391    case nir_intrinsic_read_invocation: {
6392       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6393       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6394          emit_uniform_subgroup(ctx, instr, src);
6395       } else {
6396          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6397          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6398             tid = bld.as_uniform(tid);
6399          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6400          if (src.regClass() == v1) {
6401             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6402          } else if (src.regClass() == v2) {
6403             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6404             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6405             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6406             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6407             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6408             emit_split_vector(ctx, dst, 2);
6409          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6410             assert(src.regClass() == bld.lm);
6411             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6412             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6413          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6414             assert(src.regClass() == bld.lm);
6415             Temp tmp;
6416             if (ctx->program->chip_class <= GFX7)
6417                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6418             else if (ctx->program->wave_size == 64)
6419                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6420             else
6421                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6422             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6423             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6424             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6425          } else {
6426             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6427             nir_print_instr(&instr->instr, stderr);
6428             fprintf(stderr, "\n");
6429          }
6430       }
6431       break;
6432    }
6433    case nir_intrinsic_load_sample_id: {
6434       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6435                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6436       break;
6437    }
6438    case nir_intrinsic_load_sample_mask_in: {
6439       visit_load_sample_mask_in(ctx, instr);
6440       break;
6441    }
6442    case nir_intrinsic_read_first_invocation: {
6443       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6444       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6445       if (src.regClass() == v1) {
6446          emit_wqm(ctx,
6447                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6448                   dst);
6449       } else if (src.regClass() == v2) {
6450          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6451          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6452          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6453          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6454          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6455          emit_split_vector(ctx, dst, 2);
6456       } else if (instr->dest.ssa.bit_size == 1) {
6457          assert(src.regClass() == bld.lm);
6458          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6459                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6460          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6461       } else if (src.regClass() == s1) {
6462          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6463       } else if (src.regClass() == s2) {
6464          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6465       } else {
6466          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6467          nir_print_instr(&instr->instr, stderr);
6468          fprintf(stderr, "\n");
6469       }
6470       break;
6471    }
6472    case nir_intrinsic_vote_all: {
6473       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6474       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6475       assert(src.regClass() == bld.lm);
6476       assert(dst.regClass() == bld.lm);
6477
6478       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6479       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6480       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6481       break;
6482    }
6483    case nir_intrinsic_vote_any: {
6484       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6485       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6486       assert(src.regClass() == bld.lm);
6487       assert(dst.regClass() == bld.lm);
6488
6489       Temp tmp = bool_to_scalar_condition(ctx, src);
6490       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6491       break;
6492    }
6493    case nir_intrinsic_reduce:
6494    case nir_intrinsic_inclusive_scan:
6495    case nir_intrinsic_exclusive_scan: {
6496       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6497       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6498       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6499       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6500          nir_intrinsic_cluster_size(instr) : 0;
6501       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6502
6503       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6504          emit_uniform_subgroup(ctx, instr, src);
6505       } else if (instr->dest.ssa.bit_size == 1) {
6506          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6507             op = nir_op_iand;
6508          else if (op == nir_op_iadd)
6509             op = nir_op_ixor;
6510          else if (op == nir_op_umax || op == nir_op_imax)
6511             op = nir_op_ior;
6512          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6513
6514          switch (instr->intrinsic) {
6515          case nir_intrinsic_reduce:
6516             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6517             break;
6518          case nir_intrinsic_exclusive_scan:
6519             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6520             break;
6521          case nir_intrinsic_inclusive_scan:
6522             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6523             break;
6524          default:
6525             assert(false);
6526          }
6527       } else if (cluster_size == 1) {
6528          bld.copy(Definition(dst), src);
6529       } else {
6530          src = as_vgpr(ctx, src);
6531
6532          ReduceOp reduce_op;
6533          switch (op) {
6534          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6535             CASE(iadd)
6536             CASE(imul)
6537             CASE(fadd)
6538             CASE(fmul)
6539             CASE(imin)
6540             CASE(umin)
6541             CASE(fmin)
6542             CASE(imax)
6543             CASE(umax)
6544             CASE(fmax)
6545             CASE(iand)
6546             CASE(ior)
6547             CASE(ixor)
6548             default:
6549                unreachable("unknown reduction op");
6550          #undef CASE
6551          }
6552
6553          aco_opcode aco_op;
6554          switch (instr->intrinsic) {
6555             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6556             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6557             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6558             default:
6559                unreachable("unknown reduce intrinsic");
6560          }
6561
6562          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6563          reduce->operands[0] = Operand(src);
6564          // filled in by aco_reduce_assign.cpp, used internally as part of the
6565          // reduce sequence
6566          assert(dst.size() == 1 || dst.size() == 2);
6567          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6568          reduce->operands[2] = Operand(v1.as_linear());
6569
6570          Temp tmp_dst = bld.tmp(dst.regClass());
6571          reduce->definitions[0] = Definition(tmp_dst);
6572          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6573          reduce->definitions[2] = Definition();
6574          reduce->definitions[3] = Definition(scc, s1);
6575          reduce->definitions[4] = Definition();
6576          reduce->reduce_op = reduce_op;
6577          reduce->cluster_size = cluster_size;
6578          ctx->block->instructions.emplace_back(std::move(reduce));
6579
6580          emit_wqm(ctx, tmp_dst, dst);
6581       }
6582       break;
6583    }
6584    case nir_intrinsic_quad_broadcast: {
6585       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6586       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6587          emit_uniform_subgroup(ctx, instr, src);
6588       } else {
6589          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6590          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6591          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6592
6593          if (instr->dest.ssa.bit_size == 1) {
6594             assert(src.regClass() == bld.lm);
6595             assert(dst.regClass() == bld.lm);
6596             uint32_t half_mask = 0x11111111u << lane;
6597             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6598             Temp tmp = bld.tmp(bld.lm);
6599             bld.sop1(Builder::s_wqm, Definition(tmp),
6600                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6601                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6602             emit_wqm(ctx, tmp, dst);
6603          } else if (instr->dest.ssa.bit_size == 32) {
6604             if (ctx->program->chip_class >= GFX8)
6605                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6606             else
6607                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6608          } else if (instr->dest.ssa.bit_size == 64) {
6609             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6610             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6611             if (ctx->program->chip_class >= GFX8) {
6612                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6613                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6614             } else {
6615                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6616                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6617             }
6618             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6619             emit_split_vector(ctx, dst, 2);
6620          } else {
6621             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6622             nir_print_instr(&instr->instr, stderr);
6623             fprintf(stderr, "\n");
6624          }
6625       }
6626       break;
6627    }
6628    case nir_intrinsic_quad_swap_horizontal:
6629    case nir_intrinsic_quad_swap_vertical:
6630    case nir_intrinsic_quad_swap_diagonal:
6631    case nir_intrinsic_quad_swizzle_amd: {
6632       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6633       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6634          emit_uniform_subgroup(ctx, instr, src);
6635          break;
6636       }
6637       uint16_t dpp_ctrl = 0;
6638       switch (instr->intrinsic) {
6639       case nir_intrinsic_quad_swap_horizontal:
6640          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6641          break;
6642       case nir_intrinsic_quad_swap_vertical:
6643          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6644          break;
6645       case nir_intrinsic_quad_swap_diagonal:
6646          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6647          break;
6648       case nir_intrinsic_quad_swizzle_amd:
6649          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6650          break;
6651       default:
6652          break;
6653       }
6654       if (ctx->program->chip_class < GFX8)
6655          dpp_ctrl |= (1 << 15);
6656
6657       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6658       if (instr->dest.ssa.bit_size == 1) {
6659          assert(src.regClass() == bld.lm);
6660          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6661          if (ctx->program->chip_class >= GFX8)
6662             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6663          else
6664             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6665          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6666          emit_wqm(ctx, tmp, dst);
6667       } else if (instr->dest.ssa.bit_size == 32) {
6668          Temp tmp;
6669          if (ctx->program->chip_class >= GFX8)
6670             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6671          else
6672             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6673          emit_wqm(ctx, tmp, dst);
6674       } else if (instr->dest.ssa.bit_size == 64) {
6675          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6676          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6677          if (ctx->program->chip_class >= GFX8) {
6678             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6679             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6680          } else {
6681             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6682             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6683          }
6684          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6685          emit_split_vector(ctx, dst, 2);
6686       } else {
6687          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6688          nir_print_instr(&instr->instr, stderr);
6689          fprintf(stderr, "\n");
6690       }
6691       break;
6692    }
6693    case nir_intrinsic_masked_swizzle_amd: {
6694       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6695       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6696          emit_uniform_subgroup(ctx, instr, src);
6697          break;
6698       }
6699       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6700       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6701       if (dst.regClass() == v1) {
6702          emit_wqm(ctx,
6703                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6704                   dst);
6705       } else if (dst.regClass() == v2) {
6706          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6707          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6708          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6709          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6710          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6711          emit_split_vector(ctx, dst, 2);
6712       } else {
6713          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6714          nir_print_instr(&instr->instr, stderr);
6715          fprintf(stderr, "\n");
6716       }
6717       break;
6718    }
6719    case nir_intrinsic_write_invocation_amd: {
6720       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6721       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6722       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6723       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6724       if (dst.regClass() == v1) {
6725          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6726          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6727       } else if (dst.regClass() == v2) {
6728          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6729          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6730          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6731          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6732          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6733          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6734          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6735          emit_split_vector(ctx, dst, 2);
6736       } else {
6737          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6738          nir_print_instr(&instr->instr, stderr);
6739          fprintf(stderr, "\n");
6740       }
6741       break;
6742    }
6743    case nir_intrinsic_mbcnt_amd: {
6744       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6745       RegClass rc = RegClass(src.type(), 1);
6746       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6747       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6748       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6749       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6750       emit_wqm(ctx, wqm_tmp, dst);
6751       break;
6752    }
6753    case nir_intrinsic_load_helper_invocation: {
6754       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6755       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6756       ctx->block->kind |= block_kind_needs_lowering;
6757       ctx->program->needs_exact = true;
6758       break;
6759    }
6760    case nir_intrinsic_is_helper_invocation: {
6761       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6762       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6763       ctx->block->kind |= block_kind_needs_lowering;
6764       ctx->program->needs_exact = true;
6765       break;
6766    }
6767    case nir_intrinsic_demote:
6768       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6769
6770       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6771          ctx->cf_info.exec_potentially_empty = true;
6772       ctx->block->kind |= block_kind_uses_demote;
6773       ctx->program->needs_exact = true;
6774       break;
6775    case nir_intrinsic_demote_if: {
6776       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6777       assert(src.regClass() == bld.lm);
6778       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6779       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6780
6781       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6782          ctx->cf_info.exec_potentially_empty = true;
6783       ctx->block->kind |= block_kind_uses_demote;
6784       ctx->program->needs_exact = true;
6785       break;
6786    }
6787    case nir_intrinsic_first_invocation: {
6788       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6789                get_ssa_temp(ctx, &instr->dest.ssa));
6790       break;
6791    }
6792    case nir_intrinsic_shader_clock:
6793       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6794       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6795       break;
6796    case nir_intrinsic_load_vertex_id_zero_base: {
6797       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6798       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6799       break;
6800    }
6801    case nir_intrinsic_load_first_vertex: {
6802       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6803       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6804       break;
6805    }
6806    case nir_intrinsic_load_base_instance: {
6807       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6808       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6809       break;
6810    }
6811    case nir_intrinsic_load_instance_id: {
6812       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6813       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6814       break;
6815    }
6816    case nir_intrinsic_load_draw_id: {
6817       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6818       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6819       break;
6820    }
6821    case nir_intrinsic_load_invocation_id: {
6822       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6823       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6824       if (ctx->options->chip_class >= GFX10)
6825          bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6826       else
6827          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6828       break;
6829    }
6830    case nir_intrinsic_load_primitive_id: {
6831       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6832       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6833       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
6834       break;
6835    }
6836    case nir_intrinsic_emit_vertex_with_counter: {
6837       visit_emit_vertex_with_counter(ctx, instr);
6838       break;
6839    }
6840    case nir_intrinsic_end_primitive_with_counter: {
6841       unsigned stream = nir_intrinsic_stream_id(instr);
6842       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
6843       break;
6844    }
6845    case nir_intrinsic_set_vertex_count: {
6846       /* unused, the HW keeps track of this for us */
6847       break;
6848    }
6849    default:
6850       fprintf(stderr, "Unimplemented intrinsic instr: ");
6851       nir_print_instr(&instr->instr, stderr);
6852       fprintf(stderr, "\n");
6853       abort();
6854
6855       break;
6856    }
6857 }
6858
6859
6860 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6861                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6862                     enum glsl_base_type *stype)
6863 {
6864    nir_deref_instr *texture_deref_instr = NULL;
6865    nir_deref_instr *sampler_deref_instr = NULL;
6866    int plane = -1;
6867
6868    for (unsigned i = 0; i < instr->num_srcs; i++) {
6869       switch (instr->src[i].src_type) {
6870       case nir_tex_src_texture_deref:
6871          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6872          break;
6873       case nir_tex_src_sampler_deref:
6874          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6875          break;
6876       case nir_tex_src_plane:
6877          plane = nir_src_as_int(instr->src[i].src);
6878          break;
6879       default:
6880          break;
6881       }
6882    }
6883
6884    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6885
6886    if (!sampler_deref_instr)
6887       sampler_deref_instr = texture_deref_instr;
6888
6889    if (plane >= 0) {
6890       assert(instr->op != nir_texop_txf_ms &&
6891              instr->op != nir_texop_samples_identical);
6892       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6893       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6894    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6895       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6896    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6897       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6898    } else {
6899       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6900    }
6901    if (samp_ptr) {
6902       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6903
6904       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6905          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6906          Builder bld(ctx->program, ctx->block);
6907
6908          /* to avoid unnecessary moves, we split and recombine sampler and image */
6909          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6910                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6911          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6912          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6913                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6914                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6915          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6916                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6917
6918          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6919          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6920                                img[0], img[1], img[2], img[3],
6921                                img[4], img[5], img[6], img[7]);
6922          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6923                                 samp[0], samp[1], samp[2], samp[3]);
6924       }
6925    }
6926    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6927                      instr->op == nir_texop_samples_identical))
6928       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6929 }
6930
6931 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6932                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6933 {
6934    Builder bld(ctx->program, ctx->block);
6935
6936    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6937    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6938    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6939
6940    Operand neg_one(0xbf800000u);
6941    Operand one(0x3f800000u);
6942    Operand two(0x40000000u);
6943    Operand four(0x40800000u);
6944
6945    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6946    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6947    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6948
6949    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6950    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6951    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6952    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6953
6954    // select sc
6955    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6956    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6957                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6958                        one, is_ma_y);
6959    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6960
6961    // select tc
6962    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6963    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6964    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6965
6966    // select ma
6967    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6968                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6969                   deriv_z, is_ma_z);
6970    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6971    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6972 }
6973
6974 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6975 {
6976    Builder bld(ctx->program, ctx->block);
6977    Temp coord_args[4], ma, tc, sc, id;
6978    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6979       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6980
6981    if (is_array) {
6982       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6983
6984       // see comment in ac_prepare_cube_coords()
6985       if (ctx->options->chip_class <= GFX8)
6986          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6987    }
6988
6989    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6990
6991    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6992    vop3a->operands[0] = Operand(ma);
6993    vop3a->abs[0] = true;
6994    Temp invma = bld.tmp(v1);
6995    vop3a->definitions[0] = Definition(invma);
6996    ctx->block->instructions.emplace_back(std::move(vop3a));
6997
6998    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6999    if (!is_deriv)
7000       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
7001
7002    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
7003    if (!is_deriv)
7004       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
7005
7006    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
7007
7008    if (is_deriv) {
7009       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
7010       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
7011
7012       for (unsigned i = 0; i < 2; i++) {
7013          // see comment in ac_prepare_cube_coords()
7014          Temp deriv_ma;
7015          Temp deriv_sc, deriv_tc;
7016          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
7017                            &deriv_ma, &deriv_sc, &deriv_tc);
7018
7019          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
7020
7021          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7022                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
7023                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
7024          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
7025                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
7026                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
7027          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
7028       }
7029
7030       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
7031       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
7032    }
7033
7034    if (is_array)
7035       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
7036    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
7037
7038 }
7039
7040 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
7041 {
7042    Temp coord_vec[3];
7043    for (unsigned i = 0; i < coords.size(); i++)
7044       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
7045
7046    Builder bld(ctx->program, ctx->block);
7047    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
7048
7049    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7050    for (unsigned i = 0; i < coords.size(); i++)
7051       vec->operands[i] = Operand(coord_vec[i]);
7052    Temp res = bld.tmp(RegType::vgpr, coords.size());
7053    vec->definitions[0] = Definition(res);
7054    ctx->block->instructions.emplace_back(std::move(vec));
7055    return res;
7056 }
7057
7058 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
7059 {
7060    if (vec->parent_instr->type != nir_instr_type_alu)
7061       return;
7062    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
7063    if (vec_instr->op != nir_op_vec(vec->num_components))
7064       return;
7065
7066    for (unsigned i = 0; i < vec->num_components; i++) {
7067       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
7068               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
7069    }
7070 }
7071
7072 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
7073 {
7074    Builder bld(ctx->program, ctx->block);
7075    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
7076         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
7077    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
7078         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
7079    nir_const_value *sample_index_cv = NULL;
7080    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
7081    enum glsl_base_type stype;
7082    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
7083
7084    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
7085                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
7086    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
7087                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
7088
7089    for (unsigned i = 0; i < instr->num_srcs; i++) {
7090       switch (instr->src[i].src_type) {
7091       case nir_tex_src_coord:
7092          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
7093          break;
7094       case nir_tex_src_bias:
7095          if (instr->op == nir_texop_txb) {
7096             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
7097             has_bias = true;
7098          }
7099          break;
7100       case nir_tex_src_lod: {
7101          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
7102
7103          if (val && val->f32 <= 0.0) {
7104             level_zero = true;
7105          } else {
7106             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
7107             has_lod = true;
7108          }
7109          break;
7110       }
7111       case nir_tex_src_comparator:
7112          if (instr->is_shadow) {
7113             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
7114             has_compare = true;
7115          }
7116          break;
7117       case nir_tex_src_offset:
7118          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7119          get_const_vec(instr->src[i].src.ssa, const_offset);
7120          has_offset = true;
7121          break;
7122       case nir_tex_src_ddx:
7123          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7124          has_ddx = true;
7125          break;
7126       case nir_tex_src_ddy:
7127          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7128          has_ddy = true;
7129          break;
7130       case nir_tex_src_ms_index:
7131          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7132          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7133          has_sample_index = true;
7134          break;
7135       case nir_tex_src_texture_offset:
7136       case nir_tex_src_sampler_offset:
7137       default:
7138          break;
7139       }
7140    }
7141 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
7142    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7143       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7144
7145    if (instr->op == nir_texop_texture_samples) {
7146       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7147
7148       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7149       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7150       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7151       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7152
7153       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7154                samples, Operand(1u), bld.scc(is_msaa));
7155       return;
7156    }
7157
7158    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7159       aco_ptr<Instruction> tmp_instr;
7160       Temp acc, pack = Temp();
7161
7162       uint32_t pack_const = 0;
7163       for (unsigned i = 0; i < offset.size(); i++) {
7164          if (!const_offset[i])
7165             continue;
7166          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7167       }
7168
7169       if (offset.type() == RegType::sgpr) {
7170          for (unsigned i = 0; i < offset.size(); i++) {
7171             if (const_offset[i])
7172                continue;
7173
7174             acc = emit_extract_vector(ctx, offset, i, s1);
7175             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7176
7177             if (i) {
7178                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7179             }
7180
7181             if (pack == Temp()) {
7182                pack = acc;
7183             } else {
7184                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7185             }
7186          }
7187
7188          if (pack_const && pack != Temp())
7189             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7190       } else {
7191          for (unsigned i = 0; i < offset.size(); i++) {
7192             if (const_offset[i])
7193                continue;
7194
7195             acc = emit_extract_vector(ctx, offset, i, v1);
7196             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7197
7198             if (i) {
7199                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7200             }
7201
7202             if (pack == Temp()) {
7203                pack = acc;
7204             } else {
7205                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7206             }
7207          }
7208
7209          if (pack_const && pack != Temp())
7210             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7211       }
7212       if (pack_const && pack == Temp())
7213          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7214       else if (pack == Temp())
7215          has_offset = false;
7216       else
7217          offset = pack;
7218    }
7219
7220    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7221       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7222
7223    /* pack derivatives */
7224    if (has_ddx || has_ddy) {
7225       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7226          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
7227                              ddx, Operand(0u), ddy, Operand(0u));
7228       } else {
7229          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
7230       }
7231       has_derivs = true;
7232    }
7233
7234    if (instr->coord_components > 1 &&
7235        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7236        instr->is_array &&
7237        instr->op != nir_texop_txf)
7238       coords = apply_round_slice(ctx, coords, 1);
7239
7240    if (instr->coord_components > 2 &&
7241       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7242        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7243        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7244        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7245        instr->is_array &&
7246        instr->op != nir_texop_txf &&
7247        instr->op != nir_texop_txf_ms &&
7248        instr->op != nir_texop_fragment_fetch &&
7249        instr->op != nir_texop_fragment_mask_fetch)
7250       coords = apply_round_slice(ctx, coords, 2);
7251
7252    if (ctx->options->chip_class == GFX9 &&
7253        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7254        instr->op != nir_texop_lod && instr->coord_components) {
7255       assert(coords.size() > 0 && coords.size() < 3);
7256
7257       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
7258       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
7259       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
7260       if (coords.size() > 1)
7261          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
7262       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
7263       vec->definitions[0] = Definition(coords);
7264       ctx->block->instructions.emplace_back(std::move(vec));
7265    }
7266
7267    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7268
7269    if (instr->op == nir_texop_samples_identical)
7270       resource = fmask_ptr;
7271
7272    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7273              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7274             instr->op != nir_texop_txs &&
7275             instr->op != nir_texop_fragment_fetch &&
7276             instr->op != nir_texop_fragment_mask_fetch) {
7277       assert(has_sample_index);
7278       Operand op(sample_index);
7279       if (sample_index_cv)
7280          op = Operand(sample_index_cv->u32);
7281       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7282    }
7283
7284    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7285       Temp split_coords[coords.size()];
7286       emit_split_vector(ctx, coords, coords.size());
7287       for (unsigned i = 0; i < coords.size(); i++)
7288          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
7289
7290       unsigned i = 0;
7291       for (; i < std::min(offset.size(), instr->coord_components); i++) {
7292          Temp off = emit_extract_vector(ctx, offset, i, v1);
7293          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
7294       }
7295
7296       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7297       for (unsigned i = 0; i < coords.size(); i++)
7298          vec->operands[i] = Operand(split_coords[i]);
7299       coords = bld.tmp(coords.regClass());
7300       vec->definitions[0] = Definition(coords);
7301       ctx->block->instructions.emplace_back(std::move(vec));
7302
7303       has_offset = false;
7304    }
7305
7306    /* Build tex instruction */
7307    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7308    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7309                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7310                   : 0;
7311    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7312    Temp tmp_dst = dst;
7313
7314    /* gather4 selects the component by dmask and always returns vec4 */
7315    if (instr->op == nir_texop_tg4) {
7316       assert(instr->dest.ssa.num_components == 4);
7317       if (instr->is_shadow)
7318          dmask = 1;
7319       else
7320          dmask = 1 << instr->component;
7321       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7322          tmp_dst = bld.tmp(v4);
7323    } else if (instr->op == nir_texop_samples_identical) {
7324       tmp_dst = bld.tmp(v1);
7325    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7326       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7327    }
7328
7329    aco_ptr<MIMG_instruction> tex;
7330    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7331       if (!has_lod)
7332          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7333
7334       bool div_by_6 = instr->op == nir_texop_txs &&
7335                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7336                       instr->is_array &&
7337                       (dmask & (1 << 2));
7338       if (tmp_dst.id() == dst.id() && div_by_6)
7339          tmp_dst = bld.tmp(tmp_dst.regClass());
7340
7341       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7342       tex->operands[0] = Operand(as_vgpr(ctx,lod));
7343       tex->operands[1] = Operand(resource);
7344       if (ctx->options->chip_class == GFX9 &&
7345           instr->op == nir_texop_txs &&
7346           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7347           instr->is_array) {
7348          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7349       } else if (instr->op == nir_texop_query_levels) {
7350          tex->dmask = 1 << 3;
7351       } else {
7352          tex->dmask = dmask;
7353       }
7354       tex->da = da;
7355       tex->definitions[0] = Definition(tmp_dst);
7356       tex->dim = dim;
7357       tex->can_reorder = true;
7358       ctx->block->instructions.emplace_back(std::move(tex));
7359
7360       if (div_by_6) {
7361          /* divide 3rd value by 6 by multiplying with magic number */
7362          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7363          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7364          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7365          assert(instr->dest.ssa.num_components == 3);
7366          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7367          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7368                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7369                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7370                               by_6);
7371
7372       }
7373
7374       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7375       return;
7376    }
7377
7378    Temp tg4_compare_cube_wa64 = Temp();
7379
7380    if (tg4_integer_workarounds) {
7381       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7382       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7383       tex->operands[1] = Operand(resource);
7384       tex->dim = dim;
7385       tex->dmask = 0x3;
7386       tex->da = da;
7387       Temp size = bld.tmp(v2);
7388       tex->definitions[0] = Definition(size);
7389       tex->can_reorder = true;
7390       ctx->block->instructions.emplace_back(std::move(tex));
7391       emit_split_vector(ctx, size, size.size());
7392
7393       Temp half_texel[2];
7394       for (unsigned i = 0; i < 2; i++) {
7395          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7396          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7397          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7398          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7399       }
7400
7401       Temp orig_coords[2] = {
7402          emit_extract_vector(ctx, coords, 0, v1),
7403          emit_extract_vector(ctx, coords, 1, v1)};
7404       Temp new_coords[2] = {
7405          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
7406          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
7407       };
7408
7409       if (tg4_integer_cube_workaround) {
7410          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7411          Temp desc[resource.size()];
7412          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7413                                                                            Format::PSEUDO, 1, resource.size())};
7414          split->operands[0] = Operand(resource);
7415          for (unsigned i = 0; i < resource.size(); i++) {
7416             desc[i] = bld.tmp(s1);
7417             split->definitions[i] = Definition(desc[i]);
7418          }
7419          ctx->block->instructions.emplace_back(std::move(split));
7420
7421          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7422          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7423                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7424
7425          Temp nfmt;
7426          if (stype == GLSL_TYPE_UINT) {
7427             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7428                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7429                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7430                             bld.scc(compare_cube_wa));
7431          } else {
7432             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7433                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7434                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7435                             bld.scc(compare_cube_wa));
7436          }
7437          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7438          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7439
7440          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7441
7442          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7443                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7444          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7445
7446          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7447                                                                          Format::PSEUDO, resource.size(), 1)};
7448          for (unsigned i = 0; i < resource.size(); i++)
7449             vec->operands[i] = Operand(desc[i]);
7450          resource = bld.tmp(resource.regClass());
7451          vec->definitions[0] = Definition(resource);
7452          ctx->block->instructions.emplace_back(std::move(vec));
7453
7454          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7455                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
7456          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7457                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
7458       }
7459
7460       if (coords.size() == 3) {
7461          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
7462                              new_coords[0], new_coords[1],
7463                              emit_extract_vector(ctx, coords, 2, v1));
7464       } else {
7465          assert(coords.size() == 2);
7466          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
7467                              new_coords[0], new_coords[1]);
7468       }
7469    }
7470
7471    std::vector<Operand> args;
7472    if (has_offset)
7473       args.emplace_back(Operand(offset));
7474    if (has_bias)
7475       args.emplace_back(Operand(bias));
7476    if (has_compare)
7477       args.emplace_back(Operand(compare));
7478    if (has_derivs)
7479       args.emplace_back(Operand(derivs));
7480    args.emplace_back(Operand(coords));
7481    if (has_sample_index)
7482       args.emplace_back(Operand(sample_index));
7483    if (has_lod)
7484       args.emplace_back(lod);
7485
7486    Temp arg;
7487    if (args.size() > 1) {
7488       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7489       unsigned size = 0;
7490       for (unsigned i = 0; i < args.size(); i++) {
7491          size += args[i].size();
7492          vec->operands[i] = args[i];
7493       }
7494       RegClass rc = RegClass(RegType::vgpr, size);
7495       Temp tmp = bld.tmp(rc);
7496       vec->definitions[0] = Definition(tmp);
7497       ctx->block->instructions.emplace_back(std::move(vec));
7498       arg = tmp;
7499    } else {
7500       assert(args[0].isTemp());
7501       arg = as_vgpr(ctx, args[0].getTemp());
7502    }
7503
7504    /* we don't need the bias, sample index, compare value or offset to be
7505     * computed in WQM but if the p_create_vector copies the coordinates, then it
7506     * needs to be in WQM */
7507    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
7508        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7509        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7510       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7511
7512    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7513       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7514
7515       assert(coords.size() == 1);
7516       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7517       aco_opcode op;
7518       switch (last_bit) {
7519       case 1:
7520          op = aco_opcode::buffer_load_format_x; break;
7521       case 2:
7522          op = aco_opcode::buffer_load_format_xy; break;
7523       case 3:
7524          op = aco_opcode::buffer_load_format_xyz; break;
7525       case 4:
7526          op = aco_opcode::buffer_load_format_xyzw; break;
7527       default:
7528          unreachable("Tex instruction loads more than 4 components.");
7529       }
7530
7531       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7532       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7533          tmp_dst = dst;
7534       else
7535          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7536
7537       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7538       mubuf->operands[0] = Operand(coords);
7539       mubuf->operands[1] = Operand(resource);
7540       mubuf->operands[2] = Operand((uint32_t) 0);
7541       mubuf->definitions[0] = Definition(tmp_dst);
7542       mubuf->idxen = true;
7543       mubuf->can_reorder = true;
7544       ctx->block->instructions.emplace_back(std::move(mubuf));
7545
7546       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7547       return;
7548    }
7549
7550
7551    if (instr->op == nir_texop_txf ||
7552        instr->op == nir_texop_txf_ms ||
7553        instr->op == nir_texop_samples_identical ||
7554        instr->op == nir_texop_fragment_fetch ||
7555        instr->op == nir_texop_fragment_mask_fetch) {
7556       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7557       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
7558       tex->operands[0] = Operand(arg);
7559       tex->operands[1] = Operand(resource);
7560       tex->dim = dim;
7561       tex->dmask = dmask;
7562       tex->unrm = true;
7563       tex->da = da;
7564       tex->definitions[0] = Definition(tmp_dst);
7565       tex->can_reorder = true;
7566       ctx->block->instructions.emplace_back(std::move(tex));
7567
7568       if (instr->op == nir_texop_samples_identical) {
7569          assert(dmask == 1 && dst.regClass() == v1);
7570          assert(dst.id() != tmp_dst.id());
7571
7572          Temp tmp = bld.tmp(bld.lm);
7573          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7574          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7575
7576       } else {
7577          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7578       }
7579       return;
7580    }
7581
7582    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7583    aco_opcode opcode = aco_opcode::image_sample;
7584    if (has_offset) { /* image_sample_*_o */
7585       if (has_compare) {
7586          opcode = aco_opcode::image_sample_c_o;
7587          if (has_derivs)
7588             opcode = aco_opcode::image_sample_c_d_o;
7589          if (has_bias)
7590             opcode = aco_opcode::image_sample_c_b_o;
7591          if (level_zero)
7592             opcode = aco_opcode::image_sample_c_lz_o;
7593          if (has_lod)
7594             opcode = aco_opcode::image_sample_c_l_o;
7595       } else {
7596          opcode = aco_opcode::image_sample_o;
7597          if (has_derivs)
7598             opcode = aco_opcode::image_sample_d_o;
7599          if (has_bias)
7600             opcode = aco_opcode::image_sample_b_o;
7601          if (level_zero)
7602             opcode = aco_opcode::image_sample_lz_o;
7603          if (has_lod)
7604             opcode = aco_opcode::image_sample_l_o;
7605       }
7606    } else { /* no offset */
7607       if (has_compare) {
7608          opcode = aco_opcode::image_sample_c;
7609          if (has_derivs)
7610             opcode = aco_opcode::image_sample_c_d;
7611          if (has_bias)
7612             opcode = aco_opcode::image_sample_c_b;
7613          if (level_zero)
7614             opcode = aco_opcode::image_sample_c_lz;
7615          if (has_lod)
7616             opcode = aco_opcode::image_sample_c_l;
7617       } else {
7618          opcode = aco_opcode::image_sample;
7619          if (has_derivs)
7620             opcode = aco_opcode::image_sample_d;
7621          if (has_bias)
7622             opcode = aco_opcode::image_sample_b;
7623          if (level_zero)
7624             opcode = aco_opcode::image_sample_lz;
7625          if (has_lod)
7626             opcode = aco_opcode::image_sample_l;
7627       }
7628    }
7629
7630    if (instr->op == nir_texop_tg4) {
7631       if (has_offset) {
7632          opcode = aco_opcode::image_gather4_lz_o;
7633          if (has_compare)
7634             opcode = aco_opcode::image_gather4_c_lz_o;
7635       } else {
7636          opcode = aco_opcode::image_gather4_lz;
7637          if (has_compare)
7638             opcode = aco_opcode::image_gather4_c_lz;
7639       }
7640    } else if (instr->op == nir_texop_lod) {
7641       opcode = aco_opcode::image_get_lod;
7642    }
7643
7644    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7645    tex->operands[0] = Operand(arg);
7646    tex->operands[1] = Operand(resource);
7647    tex->operands[2] = Operand(sampler);
7648    tex->dim = dim;
7649    tex->dmask = dmask;
7650    tex->da = da;
7651    tex->definitions[0] = Definition(tmp_dst);
7652    tex->can_reorder = true;
7653    ctx->block->instructions.emplace_back(std::move(tex));
7654
7655    if (tg4_integer_cube_workaround) {
7656       assert(tmp_dst.id() != dst.id());
7657       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7658
7659       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7660       Temp val[4];
7661       for (unsigned i = 0; i < dst.size(); i++) {
7662          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7663          Temp cvt_val;
7664          if (stype == GLSL_TYPE_UINT)
7665             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7666          else
7667             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7668          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7669       }
7670       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7671       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7672                            val[0], val[1], val[2], val[3]);
7673    }
7674    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7675    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7676
7677 }
7678
7679
7680 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7681 {
7682    Temp tmp = get_ssa_temp(ctx, ssa);
7683    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7684       return Operand(tmp.regClass());
7685    else
7686       return Operand(tmp);
7687 }
7688
7689 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7690 {
7691    aco_ptr<Pseudo_instruction> phi;
7692    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7693    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7694
7695    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7696    logical |= ctx->block->kind & block_kind_merge;
7697    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7698
7699    /* we want a sorted list of sources, since the predecessor list is also sorted */
7700    std::map<unsigned, nir_ssa_def*> phi_src;
7701    nir_foreach_phi_src(src, instr)
7702       phi_src[src->pred->index] = src->src.ssa;
7703
7704    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7705    unsigned num_operands = 0;
7706    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7707    unsigned num_defined = 0;
7708    unsigned cur_pred_idx = 0;
7709    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7710       if (cur_pred_idx < preds.size()) {
7711          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7712          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7713          unsigned skipped = 0;
7714          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7715             skipped++;
7716          if (cur_pred_idx + skipped < preds.size()) {
7717             for (unsigned i = 0; i < skipped; i++)
7718                operands[num_operands++] = Operand(dst.regClass());
7719             cur_pred_idx += skipped;
7720          } else {
7721             continue;
7722          }
7723       }
7724       cur_pred_idx++;
7725       Operand op = get_phi_operand(ctx, src.second);
7726       operands[num_operands++] = op;
7727       num_defined += !op.isUndefined();
7728    }
7729    /* handle block_kind_continue_or_break at loop exit blocks */
7730    while (cur_pred_idx++ < preds.size())
7731       operands[num_operands++] = Operand(dst.regClass());
7732
7733    if (num_defined == 0) {
7734       Builder bld(ctx->program, ctx->block);
7735       if (dst.regClass() == s1) {
7736          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7737       } else if (dst.regClass() == v1) {
7738          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7739       } else {
7740          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7741          for (unsigned i = 0; i < dst.size(); i++)
7742             vec->operands[i] = Operand(0u);
7743          vec->definitions[0] = Definition(dst);
7744          ctx->block->instructions.emplace_back(std::move(vec));
7745       }
7746       return;
7747    }
7748
7749    /* we can use a linear phi in some cases if one src is undef */
7750    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7751       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7752
7753       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7754       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7755       assert(invert->kind & block_kind_invert);
7756
7757       unsigned then_block = invert->linear_preds[0];
7758
7759       Block* insert_block = NULL;
7760       for (unsigned i = 0; i < num_operands; i++) {
7761          Operand op = operands[i];
7762          if (op.isUndefined())
7763             continue;
7764          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7765          phi->operands[0] = op;
7766          break;
7767       }
7768       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7769       phi->operands[1] = Operand(dst.regClass());
7770       phi->definitions[0] = Definition(dst);
7771       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7772       return;
7773    }
7774
7775    /* try to scalarize vector phis */
7776    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7777       // TODO: scalarize linear phis on divergent ifs
7778       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7779       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7780       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7781          Operand src = operands[i];
7782          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7783             can_scalarize = false;
7784       }
7785       if (can_scalarize) {
7786          unsigned num_components = instr->dest.ssa.num_components;
7787          assert(dst.size() % num_components == 0);
7788          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7789
7790          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7791          for (unsigned k = 0; k < num_components; k++) {
7792             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7793             for (unsigned i = 0; i < num_operands; i++) {
7794                Operand src = operands[i];
7795                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7796             }
7797             Temp phi_dst = {ctx->program->allocateId(), rc};
7798             phi->definitions[0] = Definition(phi_dst);
7799             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7800             new_vec[k] = phi_dst;
7801             vec->operands[k] = Operand(phi_dst);
7802          }
7803          vec->definitions[0] = Definition(dst);
7804          ctx->block->instructions.emplace_back(std::move(vec));
7805          ctx->allocated_vec.emplace(dst.id(), new_vec);
7806          return;
7807       }
7808    }
7809
7810    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7811    for (unsigned i = 0; i < num_operands; i++)
7812       phi->operands[i] = operands[i];
7813    phi->definitions[0] = Definition(dst);
7814    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7815 }
7816
7817
7818 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7819 {
7820    Temp dst = get_ssa_temp(ctx, &instr->def);
7821
7822    assert(dst.type() == RegType::sgpr);
7823
7824    if (dst.size() == 1) {
7825       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7826    } else {
7827       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7828       for (unsigned i = 0; i < dst.size(); i++)
7829          vec->operands[i] = Operand(0u);
7830       vec->definitions[0] = Definition(dst);
7831       ctx->block->instructions.emplace_back(std::move(vec));
7832    }
7833 }
7834
7835 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7836 {
7837    Builder bld(ctx->program, ctx->block);
7838    Block *logical_target;
7839    append_logical_end(ctx->block);
7840    unsigned idx = ctx->block->index;
7841
7842    switch (instr->type) {
7843    case nir_jump_break:
7844       logical_target = ctx->cf_info.parent_loop.exit;
7845       add_logical_edge(idx, logical_target);
7846       ctx->block->kind |= block_kind_break;
7847
7848       if (!ctx->cf_info.parent_if.is_divergent &&
7849           !ctx->cf_info.parent_loop.has_divergent_continue) {
7850          /* uniform break - directly jump out of the loop */
7851          ctx->block->kind |= block_kind_uniform;
7852          ctx->cf_info.has_branch = true;
7853          bld.branch(aco_opcode::p_branch);
7854          add_linear_edge(idx, logical_target);
7855          return;
7856       }
7857       ctx->cf_info.parent_loop.has_divergent_branch = true;
7858       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7859       break;
7860    case nir_jump_continue:
7861       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7862       add_logical_edge(idx, logical_target);
7863       ctx->block->kind |= block_kind_continue;
7864
7865       if (ctx->cf_info.parent_if.is_divergent) {
7866          /* for potential uniform breaks after this continue,
7867             we must ensure that they are handled correctly */
7868          ctx->cf_info.parent_loop.has_divergent_continue = true;
7869          ctx->cf_info.parent_loop.has_divergent_branch = true;
7870          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7871       } else {
7872          /* uniform continue - directly jump to the loop header */
7873          ctx->block->kind |= block_kind_uniform;
7874          ctx->cf_info.has_branch = true;
7875          bld.branch(aco_opcode::p_branch);
7876          add_linear_edge(idx, logical_target);
7877          return;
7878       }
7879       break;
7880    default:
7881       fprintf(stderr, "Unknown NIR jump instr: ");
7882       nir_print_instr(&instr->instr, stderr);
7883       fprintf(stderr, "\n");
7884       abort();
7885    }
7886
7887    /* remove critical edges from linear CFG */
7888    bld.branch(aco_opcode::p_branch);
7889    Block* break_block = ctx->program->create_and_insert_block();
7890    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7891    break_block->kind |= block_kind_uniform;
7892    add_linear_edge(idx, break_block);
7893    /* the loop_header pointer might be invalidated by this point */
7894    if (instr->type == nir_jump_continue)
7895       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7896    add_linear_edge(break_block->index, logical_target);
7897    bld.reset(break_block);
7898    bld.branch(aco_opcode::p_branch);
7899
7900    Block* continue_block = ctx->program->create_and_insert_block();
7901    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7902    add_linear_edge(idx, continue_block);
7903    append_logical_start(continue_block);
7904    ctx->block = continue_block;
7905    return;
7906 }
7907
7908 void visit_block(isel_context *ctx, nir_block *block)
7909 {
7910    nir_foreach_instr(instr, block) {
7911       switch (instr->type) {
7912       case nir_instr_type_alu:
7913          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7914          break;
7915       case nir_instr_type_load_const:
7916          visit_load_const(ctx, nir_instr_as_load_const(instr));
7917          break;
7918       case nir_instr_type_intrinsic:
7919          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7920          break;
7921       case nir_instr_type_tex:
7922          visit_tex(ctx, nir_instr_as_tex(instr));
7923          break;
7924       case nir_instr_type_phi:
7925          visit_phi(ctx, nir_instr_as_phi(instr));
7926          break;
7927       case nir_instr_type_ssa_undef:
7928          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7929          break;
7930       case nir_instr_type_deref:
7931          break;
7932       case nir_instr_type_jump:
7933          visit_jump(ctx, nir_instr_as_jump(instr));
7934          break;
7935       default:
7936          fprintf(stderr, "Unknown NIR instr type: ");
7937          nir_print_instr(instr, stderr);
7938          fprintf(stderr, "\n");
7939          //abort();
7940       }
7941    }
7942
7943    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7944       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7945 }
7946
7947
7948
7949 static void visit_loop(isel_context *ctx, nir_loop *loop)
7950 {
7951    append_logical_end(ctx->block);
7952    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7953    Builder bld(ctx->program, ctx->block);
7954    bld.branch(aco_opcode::p_branch);
7955    unsigned loop_preheader_idx = ctx->block->index;
7956
7957    Block loop_exit = Block();
7958    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7959    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7960
7961    Block* loop_header = ctx->program->create_and_insert_block();
7962    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7963    loop_header->kind |= block_kind_loop_header;
7964    add_edge(loop_preheader_idx, loop_header);
7965    ctx->block = loop_header;
7966
7967    /* emit loop body */
7968    unsigned loop_header_idx = loop_header->index;
7969    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7970    append_logical_start(ctx->block);
7971    visit_cf_list(ctx, &loop->body);
7972
7973    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7974    if (!ctx->cf_info.has_branch) {
7975       append_logical_end(ctx->block);
7976       if (ctx->cf_info.exec_potentially_empty) {
7977          /* Discards can result in code running with an empty exec mask.
7978           * This would result in divergent breaks not ever being taken. As a
7979           * workaround, break the loop when the loop mask is empty instead of
7980           * always continuing. */
7981          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7982          unsigned block_idx = ctx->block->index;
7983
7984          /* create helper blocks to avoid critical edges */
7985          Block *break_block = ctx->program->create_and_insert_block();
7986          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7987          break_block->kind = block_kind_uniform;
7988          bld.reset(break_block);
7989          bld.branch(aco_opcode::p_branch);
7990          add_linear_edge(block_idx, break_block);
7991          add_linear_edge(break_block->index, &loop_exit);
7992
7993          Block *continue_block = ctx->program->create_and_insert_block();
7994          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7995          continue_block->kind = block_kind_uniform;
7996          bld.reset(continue_block);
7997          bld.branch(aco_opcode::p_branch);
7998          add_linear_edge(block_idx, continue_block);
7999          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
8000
8001          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8002             add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
8003          ctx->block = &ctx->program->blocks[block_idx];
8004       } else {
8005          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
8006          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8007             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8008          else
8009             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
8010       }
8011
8012       bld.reset(ctx->block);
8013       bld.branch(aco_opcode::p_branch);
8014    }
8015
8016    /* fixup phis in loop header from unreachable blocks */
8017    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
8018       bool linear = ctx->cf_info.has_branch;
8019       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
8020       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
8021          if ((logical && instr->opcode == aco_opcode::p_phi) ||
8022              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
8023             /* the last operand should be the one that needs to be removed */
8024             instr->operands.pop_back();
8025          } else if (!is_phi(instr)) {
8026             break;
8027          }
8028       }
8029    }
8030
8031    ctx->cf_info.has_branch = false;
8032
8033    // TODO: if the loop has not a single exit, we must add one °°
8034    /* emit loop successor block */
8035    ctx->block = ctx->program->insert_block(std::move(loop_exit));
8036    append_logical_start(ctx->block);
8037
8038    #if 0
8039    // TODO: check if it is beneficial to not branch on continues
8040    /* trim linear phis in loop header */
8041    for (auto&& instr : loop_entry->instructions) {
8042       if (instr->opcode == aco_opcode::p_linear_phi) {
8043          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
8044          new_phi->definitions[0] = instr->definitions[0];
8045          for (unsigned i = 0; i < new_phi->operands.size(); i++)
8046             new_phi->operands[i] = instr->operands[i];
8047          /* check that the remaining operands are all the same */
8048          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
8049             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
8050          instr.swap(new_phi);
8051       } else if (instr->opcode == aco_opcode::p_phi) {
8052          continue;
8053       } else {
8054          break;
8055       }
8056    }
8057    #endif
8058 }
8059
8060 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
8061 {
8062    ic->cond = cond;
8063
8064    append_logical_end(ctx->block);
8065    ctx->block->kind |= block_kind_branch;
8066
8067    /* branch to linear then block */
8068    assert(cond.regClass() == ctx->program->lane_mask);
8069    aco_ptr<Pseudo_branch_instruction> branch;
8070    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8071    branch->operands[0] = Operand(cond);
8072    ctx->block->instructions.push_back(std::move(branch));
8073
8074    ic->BB_if_idx = ctx->block->index;
8075    ic->BB_invert = Block();
8076    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8077    /* Invert blocks are intentionally not marked as top level because they
8078     * are not part of the logical cfg. */
8079    ic->BB_invert.kind |= block_kind_invert;
8080    ic->BB_endif = Block();
8081    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8082    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
8083
8084    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
8085    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
8086    ctx->cf_info.parent_if.is_divergent = true;
8087    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8088
8089    /** emit logical then block */
8090    Block* BB_then_logical = ctx->program->create_and_insert_block();
8091    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8092    add_edge(ic->BB_if_idx, BB_then_logical);
8093    ctx->block = BB_then_logical;
8094    append_logical_start(BB_then_logical);
8095 }
8096
8097 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
8098 {
8099    Block *BB_then_logical = ctx->block;
8100    append_logical_end(BB_then_logical);
8101     /* branch from logical then block to invert block */
8102    aco_ptr<Pseudo_branch_instruction> branch;
8103    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8104    BB_then_logical->instructions.emplace_back(std::move(branch));
8105    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
8106    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8107       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
8108    BB_then_logical->kind |= block_kind_uniform;
8109    assert(!ctx->cf_info.has_branch);
8110    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8111    ctx->cf_info.parent_loop.has_divergent_branch = false;
8112
8113    /** emit linear then block */
8114    Block* BB_then_linear = ctx->program->create_and_insert_block();
8115    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8116    BB_then_linear->kind |= block_kind_uniform;
8117    add_linear_edge(ic->BB_if_idx, BB_then_linear);
8118    /* branch from linear then block to invert block */
8119    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8120    BB_then_linear->instructions.emplace_back(std::move(branch));
8121    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8122
8123    /** emit invert merge block */
8124    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8125    ic->invert_idx = ctx->block->index;
8126
8127    /* branch to linear else block (skip else) */
8128    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8129    branch->operands[0] = Operand(ic->cond);
8130    ctx->block->instructions.push_back(std::move(branch));
8131
8132    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
8133    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8134
8135    /** emit logical else block */
8136    Block* BB_else_logical = ctx->program->create_and_insert_block();
8137    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8138    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8139    add_linear_edge(ic->invert_idx, BB_else_logical);
8140    ctx->block = BB_else_logical;
8141    append_logical_start(BB_else_logical);
8142 }
8143
8144 static void end_divergent_if(isel_context *ctx, if_context *ic)
8145 {
8146    Block *BB_else_logical = ctx->block;
8147    append_logical_end(BB_else_logical);
8148
8149    /* branch from logical else block to endif block */
8150    aco_ptr<Pseudo_branch_instruction> branch;
8151    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8152    BB_else_logical->instructions.emplace_back(std::move(branch));
8153    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8154    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8155       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8156    BB_else_logical->kind |= block_kind_uniform;
8157
8158    assert(!ctx->cf_info.has_branch);
8159    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8160
8161
8162    /** emit linear else block */
8163    Block* BB_else_linear = ctx->program->create_and_insert_block();
8164    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8165    BB_else_linear->kind |= block_kind_uniform;
8166    add_linear_edge(ic->invert_idx, BB_else_linear);
8167
8168    /* branch from linear else block to endif block */
8169    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8170    BB_else_linear->instructions.emplace_back(std::move(branch));
8171    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8172
8173
8174    /** emit endif merge block */
8175    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8176    append_logical_start(ctx->block);
8177
8178
8179    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8180    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
8181    /* uniform control flow never has an empty exec-mask */
8182    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
8183       ctx->cf_info.exec_potentially_empty = false;
8184 }
8185
8186 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8187 {
8188    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8189    Builder bld(ctx->program, ctx->block);
8190    aco_ptr<Pseudo_branch_instruction> branch;
8191
8192    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8193       /**
8194        * Uniform conditionals are represented in the following way*) :
8195        *
8196        * The linear and logical CFG:
8197        *                        BB_IF
8198        *                        /    \
8199        *       BB_THEN (logical)      BB_ELSE (logical)
8200        *                        \    /
8201        *                        BB_ENDIF
8202        *
8203        * *) Exceptions may be due to break and continue statements within loops
8204        *    If a break/continue happens within uniform control flow, it branches
8205        *    to the loop exit/entry block. Otherwise, it branches to the next
8206        *    merge block.
8207        **/
8208       append_logical_end(ctx->block);
8209       ctx->block->kind |= block_kind_uniform;
8210
8211       /* emit branch */
8212       assert(cond.regClass() == bld.lm);
8213       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8214       cond = bool_to_scalar_condition(ctx, cond);
8215
8216       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8217       branch->operands[0] = Operand(cond);
8218       branch->operands[0].setFixed(scc);
8219       ctx->block->instructions.emplace_back(std::move(branch));
8220
8221       unsigned BB_if_idx = ctx->block->index;
8222       Block BB_endif = Block();
8223       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8224       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8225
8226       /** emit then block */
8227       Block* BB_then = ctx->program->create_and_insert_block();
8228       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8229       add_edge(BB_if_idx, BB_then);
8230       append_logical_start(BB_then);
8231       ctx->block = BB_then;
8232       visit_cf_list(ctx, &if_stmt->then_list);
8233       BB_then = ctx->block;
8234       bool then_branch = ctx->cf_info.has_branch;
8235       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8236
8237       if (!then_branch) {
8238          append_logical_end(BB_then);
8239          /* branch from then block to endif block */
8240          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8241          BB_then->instructions.emplace_back(std::move(branch));
8242          add_linear_edge(BB_then->index, &BB_endif);
8243          if (!then_branch_divergent)
8244             add_logical_edge(BB_then->index, &BB_endif);
8245          BB_then->kind |= block_kind_uniform;
8246       }
8247
8248       ctx->cf_info.has_branch = false;
8249       ctx->cf_info.parent_loop.has_divergent_branch = false;
8250
8251       /** emit else block */
8252       Block* BB_else = ctx->program->create_and_insert_block();
8253       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8254       add_edge(BB_if_idx, BB_else);
8255       append_logical_start(BB_else);
8256       ctx->block = BB_else;
8257       visit_cf_list(ctx, &if_stmt->else_list);
8258       BB_else = ctx->block;
8259
8260       if (!ctx->cf_info.has_branch) {
8261          append_logical_end(BB_else);
8262          /* branch from then block to endif block */
8263          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8264          BB_else->instructions.emplace_back(std::move(branch));
8265          add_linear_edge(BB_else->index, &BB_endif);
8266          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8267             add_logical_edge(BB_else->index, &BB_endif);
8268          BB_else->kind |= block_kind_uniform;
8269       }
8270
8271       ctx->cf_info.has_branch &= then_branch;
8272       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8273
8274       /** emit endif merge block */
8275       if (!ctx->cf_info.has_branch) {
8276          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8277          append_logical_start(ctx->block);
8278       }
8279    } else { /* non-uniform condition */
8280       /**
8281        * To maintain a logical and linear CFG without critical edges,
8282        * non-uniform conditionals are represented in the following way*) :
8283        *
8284        * The linear CFG:
8285        *                        BB_IF
8286        *                        /    \
8287        *       BB_THEN (logical)      BB_THEN (linear)
8288        *                        \    /
8289        *                        BB_INVERT (linear)
8290        *                        /    \
8291        *       BB_ELSE (logical)      BB_ELSE (linear)
8292        *                        \    /
8293        *                        BB_ENDIF
8294        *
8295        * The logical CFG:
8296        *                        BB_IF
8297        *                        /    \
8298        *       BB_THEN (logical)      BB_ELSE (logical)
8299        *                        \    /
8300        *                        BB_ENDIF
8301        *
8302        * *) Exceptions may be due to break and continue statements within loops
8303        **/
8304
8305       if_context ic;
8306
8307       begin_divergent_if_then(ctx, &ic, cond);
8308       visit_cf_list(ctx, &if_stmt->then_list);
8309
8310       begin_divergent_if_else(ctx, &ic);
8311       visit_cf_list(ctx, &if_stmt->else_list);
8312
8313       end_divergent_if(ctx, &ic);
8314    }
8315 }
8316
8317 static void visit_cf_list(isel_context *ctx,
8318                           struct exec_list *list)
8319 {
8320    foreach_list_typed(nir_cf_node, node, node, list) {
8321       switch (node->type) {
8322       case nir_cf_node_block:
8323          visit_block(ctx, nir_cf_node_as_block(node));
8324          break;
8325       case nir_cf_node_if:
8326          visit_if(ctx, nir_cf_node_as_if(node));
8327          break;
8328       case nir_cf_node_loop:
8329          visit_loop(ctx, nir_cf_node_as_loop(node));
8330          break;
8331       default:
8332          unreachable("unimplemented cf list type");
8333       }
8334    }
8335 }
8336
8337 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8338 {
8339    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8340    uint64_t mask = ctx->outputs.mask[slot];
8341    if (!is_pos && !mask)
8342       return;
8343    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8344       return;
8345    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8346    exp->enabled_mask = mask;
8347    for (unsigned i = 0; i < 4; ++i) {
8348       if (mask & (1 << i))
8349          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8350       else
8351          exp->operands[i] = Operand(v1);
8352    }
8353    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8354     * Setting valid_mask=1 prevents it and has no other effect.
8355     */
8356    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8357    exp->done = false;
8358    exp->compressed = false;
8359    if (is_pos)
8360       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8361    else
8362       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8363    ctx->block->instructions.emplace_back(std::move(exp));
8364 }
8365
8366 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8367 {
8368    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8369    exp->enabled_mask = 0;
8370    for (unsigned i = 0; i < 4; ++i)
8371       exp->operands[i] = Operand(v1);
8372    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8373       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8374       exp->enabled_mask |= 0x1;
8375    }
8376    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8377       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8378       exp->enabled_mask |= 0x4;
8379    }
8380    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8381       if (ctx->options->chip_class < GFX9) {
8382          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8383          exp->enabled_mask |= 0x8;
8384       } else {
8385          Builder bld(ctx->program, ctx->block);
8386
8387          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8388                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8389          if (exp->operands[2].isTemp())
8390             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8391
8392          exp->operands[2] = Operand(out);
8393          exp->enabled_mask |= 0x4;
8394       }
8395    }
8396    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8397    exp->done = false;
8398    exp->compressed = false;
8399    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8400    ctx->block->instructions.emplace_back(std::move(exp));
8401 }
8402
8403 static void create_vs_exports(isel_context *ctx)
8404 {
8405    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8406
8407    if (outinfo->export_prim_id) {
8408       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8409       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8410    }
8411
8412    if (ctx->options->key.has_multiview_view_index) {
8413       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8414       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8415    }
8416
8417    /* the order these position exports are created is important */
8418    int next_pos = 0;
8419    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8420    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8421       export_vs_psiz_layer_viewport(ctx, &next_pos);
8422    }
8423    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8424       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8425    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8426       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8427
8428    if (ctx->export_clip_dists) {
8429       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8430          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8431       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8432          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8433    }
8434
8435    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8436       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8437           i != VARYING_SLOT_PRIMITIVE_ID)
8438          continue;
8439
8440       export_vs_varying(ctx, i, false, NULL);
8441    }
8442 }
8443
8444 static void export_fs_mrt_z(isel_context *ctx)
8445 {
8446    Builder bld(ctx->program, ctx->block);
8447    unsigned enabled_channels = 0;
8448    bool compr = false;
8449    Operand values[4];
8450
8451    for (unsigned i = 0; i < 4; ++i) {
8452       values[i] = Operand(v1);
8453    }
8454
8455    /* Both stencil and sample mask only need 16-bits. */
8456    if (!ctx->program->info->ps.writes_z &&
8457        (ctx->program->info->ps.writes_stencil ||
8458         ctx->program->info->ps.writes_sample_mask)) {
8459       compr = true; /* COMPR flag */
8460
8461       if (ctx->program->info->ps.writes_stencil) {
8462          /* Stencil should be in X[23:16]. */
8463          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8464          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8465          enabled_channels |= 0x3;
8466       }
8467
8468       if (ctx->program->info->ps.writes_sample_mask) {
8469          /* SampleMask should be in Y[15:0]. */
8470          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8471          enabled_channels |= 0xc;
8472      }
8473    } else {
8474       if (ctx->program->info->ps.writes_z) {
8475          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8476          enabled_channels |= 0x1;
8477       }
8478
8479       if (ctx->program->info->ps.writes_stencil) {
8480          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8481          enabled_channels |= 0x2;
8482       }
8483
8484       if (ctx->program->info->ps.writes_sample_mask) {
8485          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8486          enabled_channels |= 0x4;
8487       }
8488    }
8489
8490    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8491     * writemask component.
8492     */
8493    if (ctx->options->chip_class == GFX6 &&
8494        ctx->options->family != CHIP_OLAND &&
8495        ctx->options->family != CHIP_HAINAN) {
8496             enabled_channels |= 0x1;
8497    }
8498
8499    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8500            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8501 }
8502
8503 static void export_fs_mrt_color(isel_context *ctx, int slot)
8504 {
8505    Builder bld(ctx->program, ctx->block);
8506    unsigned write_mask = ctx->outputs.mask[slot];
8507    Operand values[4];
8508
8509    for (unsigned i = 0; i < 4; ++i) {
8510       if (write_mask & (1 << i)) {
8511          values[i] = Operand(ctx->outputs.outputs[slot][i]);
8512       } else {
8513          values[i] = Operand(v1);
8514       }
8515    }
8516
8517    unsigned target, col_format;
8518    unsigned enabled_channels = 0;
8519    aco_opcode compr_op = (aco_opcode)0;
8520
8521    slot -= FRAG_RESULT_DATA0;
8522    target = V_008DFC_SQ_EXP_MRT + slot;
8523    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
8524
8525    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
8526    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
8527
8528    switch (col_format)
8529    {
8530    case V_028714_SPI_SHADER_ZERO:
8531       enabled_channels = 0; /* writemask */
8532       target = V_008DFC_SQ_EXP_NULL;
8533       break;
8534
8535    case V_028714_SPI_SHADER_32_R:
8536       enabled_channels = 1;
8537       break;
8538
8539    case V_028714_SPI_SHADER_32_GR:
8540       enabled_channels = 0x3;
8541       break;
8542
8543    case V_028714_SPI_SHADER_32_AR:
8544       if (ctx->options->chip_class >= GFX10) {
8545          /* Special case: on GFX10, the outputs are different for 32_AR */
8546          enabled_channels = 0x3;
8547          values[1] = values[3];
8548          values[3] = Operand(v1);
8549       } else {
8550          enabled_channels = 0x9;
8551       }
8552       break;
8553
8554    case V_028714_SPI_SHADER_FP16_ABGR:
8555       enabled_channels = 0x5;
8556       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
8557       break;
8558
8559    case V_028714_SPI_SHADER_UNORM16_ABGR:
8560       enabled_channels = 0x5;
8561       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
8562       break;
8563
8564    case V_028714_SPI_SHADER_SNORM16_ABGR:
8565       enabled_channels = 0x5;
8566       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
8567       break;
8568
8569    case V_028714_SPI_SHADER_UINT16_ABGR: {
8570       enabled_channels = 0x5;
8571       compr_op = aco_opcode::v_cvt_pk_u16_u32;
8572       if (is_int8 || is_int10) {
8573          /* clamp */
8574          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
8575          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8576
8577          for (unsigned i = 0; i < 4; i++) {
8578             if ((write_mask >> i) & 1) {
8579                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
8580                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
8581                                     values[i]);
8582             }
8583          }
8584       }
8585       break;
8586    }
8587
8588    case V_028714_SPI_SHADER_SINT16_ABGR:
8589       enabled_channels = 0x5;
8590       compr_op = aco_opcode::v_cvt_pk_i16_i32;
8591       if (is_int8 || is_int10) {
8592          /* clamp */
8593          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
8594          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
8595          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8596          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
8597
8598          for (unsigned i = 0; i < 4; i++) {
8599             if ((write_mask >> i) & 1) {
8600                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
8601                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
8602                                     values[i]);
8603                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
8604                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
8605                                     values[i]);
8606             }
8607          }
8608       }
8609       break;
8610
8611    case V_028714_SPI_SHADER_32_ABGR:
8612       enabled_channels = 0xF;
8613       break;
8614
8615    default:
8616       break;
8617    }
8618
8619    if (target == V_008DFC_SQ_EXP_NULL)
8620       return;
8621
8622    if ((bool) compr_op) {
8623       for (int i = 0; i < 2; i++) {
8624          /* check if at least one of the values to be compressed is enabled */
8625          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
8626          if (enabled) {
8627             enabled_channels |= enabled << (i*2);
8628             values[i] = bld.vop3(compr_op, bld.def(v1),
8629                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
8630                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
8631          } else {
8632             values[i] = Operand(v1);
8633          }
8634       }
8635       values[2] = Operand(v1);
8636       values[3] = Operand(v1);
8637    } else {
8638       for (int i = 0; i < 4; i++)
8639          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
8640    }
8641
8642    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8643            enabled_channels, target, (bool) compr_op);
8644 }
8645
8646 static void create_fs_exports(isel_context *ctx)
8647 {
8648    /* Export depth, stencil and sample mask. */
8649    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
8650        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
8651        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
8652       export_fs_mrt_z(ctx);
8653    }
8654
8655    /* Export all color render targets. */
8656    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
8657       if (ctx->outputs.mask[i])
8658          export_fs_mrt_color(ctx, i);
8659    }
8660 }
8661
8662 static void emit_stream_output(isel_context *ctx,
8663                                Temp const *so_buffers,
8664                                Temp const *so_write_offset,
8665                                const struct radv_stream_output *output)
8666 {
8667    unsigned num_comps = util_bitcount(output->component_mask);
8668    unsigned writemask = (1 << num_comps) - 1;
8669    unsigned loc = output->location;
8670    unsigned buf = output->buffer;
8671
8672    assert(num_comps && num_comps <= 4);
8673    if (!num_comps || num_comps > 4)
8674       return;
8675
8676    unsigned start = ffs(output->component_mask) - 1;
8677
8678    Temp out[4];
8679    bool all_undef = true;
8680    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
8681    for (unsigned i = 0; i < num_comps; i++) {
8682       out[i] = ctx->outputs.outputs[loc][start + i];
8683       all_undef = all_undef && !out[i].id();
8684    }
8685    if (all_undef)
8686       return;
8687
8688    while (writemask) {
8689       int start, count;
8690       u_bit_scan_consecutive_range(&writemask, &start, &count);
8691       if (count == 3 && ctx->options->chip_class == GFX6) {
8692          /* GFX6 doesn't support storing vec3, split it. */
8693          writemask |= 1u << (start + 2);
8694          count = 2;
8695       }
8696
8697       unsigned offset = output->offset + start * 4;
8698
8699       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8700       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8701       for (int i = 0; i < count; ++i)
8702          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8703       vec->definitions[0] = Definition(write_data);
8704       ctx->block->instructions.emplace_back(std::move(vec));
8705
8706       aco_opcode opcode;
8707       switch (count) {
8708       case 1:
8709          opcode = aco_opcode::buffer_store_dword;
8710          break;
8711       case 2:
8712          opcode = aco_opcode::buffer_store_dwordx2;
8713          break;
8714       case 3:
8715          opcode = aco_opcode::buffer_store_dwordx3;
8716          break;
8717       case 4:
8718          opcode = aco_opcode::buffer_store_dwordx4;
8719          break;
8720       default:
8721          unreachable("Unsupported dword count.");
8722       }
8723
8724       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8725       store->operands[0] = Operand(so_write_offset[buf]);
8726       store->operands[1] = Operand(so_buffers[buf]);
8727       store->operands[2] = Operand((uint32_t) 0);
8728       store->operands[3] = Operand(write_data);
8729       if (offset > 4095) {
8730          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8731          Builder bld(ctx->program, ctx->block);
8732          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8733       } else {
8734          store->offset = offset;
8735       }
8736       store->offen = true;
8737       store->glc = true;
8738       store->dlc = false;
8739       store->slc = true;
8740       store->can_reorder = true;
8741       ctx->block->instructions.emplace_back(std::move(store));
8742    }
8743 }
8744
8745 static void emit_streamout(isel_context *ctx, unsigned stream)
8746 {
8747    Builder bld(ctx->program, ctx->block);
8748
8749    Temp so_buffers[4];
8750    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8751    for (unsigned i = 0; i < 4; i++) {
8752       unsigned stride = ctx->program->info->so.strides[i];
8753       if (!stride)
8754          continue;
8755
8756       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
8757       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
8758    }
8759
8760    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8761                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8762
8763    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8764
8765    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8766
8767    if_context ic;
8768    begin_divergent_if_then(ctx, &ic, can_emit);
8769
8770    bld.reset(ctx->block);
8771
8772    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8773
8774    Temp so_write_offset[4];
8775
8776    for (unsigned i = 0; i < 4; i++) {
8777       unsigned stride = ctx->program->info->so.strides[i];
8778       if (!stride)
8779          continue;
8780
8781       if (stride == 1) {
8782          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8783                                 get_arg(ctx, ctx->args->streamout_write_idx),
8784                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8785          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8786
8787          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8788       } else {
8789          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8790          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8791                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8792          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8793       }
8794    }
8795
8796    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8797       struct radv_stream_output *output =
8798          &ctx->program->info->so.outputs[i];
8799       if (stream != output->stream)
8800          continue;
8801
8802       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8803    }
8804
8805    begin_divergent_if_else(ctx, &ic);
8806    end_divergent_if(ctx, &ic);
8807 }
8808
8809 } /* end namespace */
8810
8811 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8812 {
8813    /* Split all arguments except for the first (ring_offsets) and the last
8814     * (exec) so that the dead channels don't stay live throughout the program.
8815     */
8816    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8817       if (startpgm->definitions[i].regClass().size() > 1) {
8818          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8819                            startpgm->definitions[i].regClass().size());
8820       }
8821    }
8822 }
8823
8824 void handle_bc_optimize(isel_context *ctx)
8825 {
8826    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8827    Builder bld(ctx->program, ctx->block);
8828    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8829    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8830    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8831    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8832    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8833    if (uses_center && uses_centroid) {
8834       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8835                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8836
8837       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8838          Temp new_coord[2];
8839          for (unsigned i = 0; i < 2; i++) {
8840             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8841             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8842             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8843                                     persp_centroid, persp_center, sel);
8844          }
8845          ctx->persp_centroid = bld.tmp(v2);
8846          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8847                     Operand(new_coord[0]), Operand(new_coord[1]));
8848          emit_split_vector(ctx, ctx->persp_centroid, 2);
8849       }
8850
8851       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8852          Temp new_coord[2];
8853          for (unsigned i = 0; i < 2; i++) {
8854             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8855             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8856             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8857                                     linear_centroid, linear_center, sel);
8858          }
8859          ctx->linear_centroid = bld.tmp(v2);
8860          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8861                     Operand(new_coord[0]), Operand(new_coord[1]));
8862          emit_split_vector(ctx, ctx->linear_centroid, 2);
8863       }
8864    }
8865 }
8866
8867 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8868 {
8869    Program *program = ctx->program;
8870
8871    unsigned float_controls = shader->info.float_controls_execution_mode;
8872
8873    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8874       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8875    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8876       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8877                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8878
8879    program->next_fp_mode.must_flush_denorms32 =
8880       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8881    program->next_fp_mode.must_flush_denorms16_64 =
8882       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8883                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8884
8885    program->next_fp_mode.care_about_round32 =
8886       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8887
8888    program->next_fp_mode.care_about_round16_64 =
8889       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8890                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8891
8892    /* default to preserving fp16 and fp64 denorms, since it's free */
8893    if (program->next_fp_mode.must_flush_denorms16_64)
8894       program->next_fp_mode.denorm16_64 = 0;
8895    else
8896       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8897
8898    /* preserving fp32 denorms is expensive, so only do it if asked */
8899    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8900       program->next_fp_mode.denorm32 = fp_denorm_keep;
8901    else
8902       program->next_fp_mode.denorm32 = 0;
8903
8904    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8905       program->next_fp_mode.round32 = fp_round_tz;
8906    else
8907       program->next_fp_mode.round32 = fp_round_ne;
8908
8909    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8910       program->next_fp_mode.round16_64 = fp_round_tz;
8911    else
8912       program->next_fp_mode.round16_64 = fp_round_ne;
8913
8914    ctx->block->fp_mode = program->next_fp_mode;
8915 }
8916
8917 void cleanup_cfg(Program *program)
8918 {
8919    /* create linear_succs/logical_succs */
8920    for (Block& BB : program->blocks) {
8921       for (unsigned idx : BB.linear_preds)
8922          program->blocks[idx].linear_succs.emplace_back(BB.index);
8923       for (unsigned idx : BB.logical_preds)
8924          program->blocks[idx].logical_succs.emplace_back(BB.index);
8925    }
8926 }
8927
8928 void select_program(Program *program,
8929                     unsigned shader_count,
8930                     struct nir_shader *const *shaders,
8931                     ac_shader_config* config,
8932                     struct radv_shader_args *args)
8933 {
8934    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
8935
8936    for (unsigned i = 0; i < shader_count; i++) {
8937       nir_shader *nir = shaders[i];
8938       init_context(&ctx, nir);
8939
8940       setup_fp_mode(&ctx, nir);
8941
8942       if (!i) {
8943          /* needs to be after init_context() for FS */
8944          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8945          append_logical_start(ctx.block);
8946          split_arguments(&ctx, startpgm);
8947       }
8948
8949       if_context ic;
8950       if (shader_count >= 2) {
8951          Builder bld(ctx.program, ctx.block);
8952          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
8953          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8954          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8955
8956          begin_divergent_if_then(&ctx, &ic, cond);
8957       }
8958
8959       if (i) {
8960          Builder bld(ctx.program, ctx.block);
8961          assert(ctx.stage == vertex_geometry_gs);
8962          bld.barrier(aco_opcode::p_memory_barrier_shared);
8963          bld.sopp(aco_opcode::s_barrier);
8964
8965          ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
8966       } else if (ctx.stage == geometry_gs)
8967          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
8968
8969       if (ctx.stage == fragment_fs)
8970          handle_bc_optimize(&ctx);
8971
8972       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8973       visit_cf_list(&ctx, &func->body);
8974
8975       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
8976          emit_streamout(&ctx, 0);
8977
8978       if (ctx.stage == vertex_vs) {
8979          create_vs_exports(&ctx);
8980       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
8981          Builder bld(ctx.program, ctx.block);
8982          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
8983          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
8984       }
8985
8986       if (ctx.stage == fragment_fs)
8987          create_fs_exports(&ctx);
8988
8989       if (shader_count >= 2) {
8990          begin_divergent_if_else(&ctx, &ic);
8991          end_divergent_if(&ctx, &ic);
8992       }
8993
8994       ralloc_free(ctx.divergent_vals);
8995    }
8996
8997    program->config->float_mode = program->blocks[0].fp_mode.val;
8998
8999    append_logical_end(ctx.block);
9000    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
9001    Builder bld(ctx.program, ctx.block);
9002    if (ctx.program->wb_smem_l1_on_end)
9003       bld.smem(aco_opcode::s_dcache_wb, false);
9004    bld.sopp(aco_opcode::s_endpgm);
9005
9006    cleanup_cfg(program);
9007 }
9008
9009 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
9010                            ac_shader_config* config,
9011                            struct radv_shader_args *args)
9012 {
9013    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
9014
9015    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
9016    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
9017    program->next_fp_mode.must_flush_denorms32 = false;
9018    program->next_fp_mode.must_flush_denorms16_64 = false;
9019    program->next_fp_mode.care_about_round32 = false;
9020    program->next_fp_mode.care_about_round16_64 = false;
9021    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
9022    program->next_fp_mode.denorm32 = 0;
9023    program->next_fp_mode.round32 = fp_round_ne;
9024    program->next_fp_mode.round16_64 = fp_round_ne;
9025    ctx.block->fp_mode = program->next_fp_mode;
9026
9027    add_startpgm(&ctx);
9028    append_logical_start(ctx.block);
9029
9030    Builder bld(ctx.program, ctx.block);
9031
9032    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
9033
9034    Operand stream_id(0u);
9035    if (args->shader_info->so.num_outputs)
9036       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
9037                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
9038
9039    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
9040
9041    std::stack<Block> endif_blocks;
9042
9043    for (unsigned stream = 0; stream < 4; stream++) {
9044       if (stream_id.isConstant() && stream != stream_id.constantValue())
9045          continue;
9046
9047       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
9048       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
9049          continue;
9050
9051       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
9052
9053       unsigned BB_if_idx = ctx.block->index;
9054       Block BB_endif = Block();
9055       if (!stream_id.isConstant()) {
9056          /* begin IF */
9057          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
9058          append_logical_end(ctx.block);
9059          ctx.block->kind |= block_kind_uniform;
9060          bld.branch(aco_opcode::p_cbranch_z, cond);
9061
9062          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
9063
9064          ctx.block = ctx.program->create_and_insert_block();
9065          add_edge(BB_if_idx, ctx.block);
9066          bld.reset(ctx.block);
9067          append_logical_start(ctx.block);
9068       }
9069
9070       unsigned offset = 0;
9071       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
9072          if (args->shader_info->gs.output_streams[i] != stream)
9073             continue;
9074
9075          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
9076          unsigned length = util_last_bit(output_usage_mask);
9077          for (unsigned j = 0; j < length; ++j) {
9078             if (!(output_usage_mask & (1 << j)))
9079                continue;
9080
9081             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
9082             Temp voffset = vtx_offset;
9083             if (const_offset >= 4096u) {
9084                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
9085                const_offset %= 4096u;
9086             }
9087
9088             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
9089             mubuf->definitions[0] = bld.def(v1);
9090             mubuf->operands[0] = Operand(voffset);
9091             mubuf->operands[1] = Operand(gsvs_ring);
9092             mubuf->operands[2] = Operand(0u);
9093             mubuf->offen = true;
9094             mubuf->offset = const_offset;
9095             mubuf->glc = true;
9096             mubuf->slc = true;
9097             mubuf->dlc = args->options->chip_class >= GFX10;
9098             mubuf->barrier = barrier_none;
9099             mubuf->can_reorder = true;
9100
9101             ctx.outputs.mask[i] |= 1 << j;
9102             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
9103
9104             bld.insert(std::move(mubuf));
9105
9106             offset++;
9107          }
9108       }
9109
9110       if (args->shader_info->so.num_outputs) {
9111          emit_streamout(&ctx, stream);
9112          bld.reset(ctx.block);
9113       }
9114
9115       if (stream == 0) {
9116          create_vs_exports(&ctx);
9117          ctx.block->kind |= block_kind_export_end;
9118       }
9119
9120       if (!stream_id.isConstant()) {
9121          append_logical_end(ctx.block);
9122
9123          /* branch from then block to endif block */
9124          bld.branch(aco_opcode::p_branch);
9125          add_edge(ctx.block->index, &BB_endif);
9126          ctx.block->kind |= block_kind_uniform;
9127
9128          /* emit else block */
9129          ctx.block = ctx.program->create_and_insert_block();
9130          add_edge(BB_if_idx, ctx.block);
9131          bld.reset(ctx.block);
9132          append_logical_start(ctx.block);
9133
9134          endif_blocks.push(std::move(BB_endif));
9135       }
9136    }
9137
9138    while (!endif_blocks.empty()) {
9139       Block BB_endif = std::move(endif_blocks.top());
9140       endif_blocks.pop();
9141
9142       Block *BB_else = ctx.block;
9143
9144       append_logical_end(BB_else);
9145       /* branch from else block to endif block */
9146       bld.branch(aco_opcode::p_branch);
9147       add_edge(BB_else->index, &BB_endif);
9148       BB_else->kind |= block_kind_uniform;
9149
9150       /** emit endif merge block */
9151       ctx.block = program->insert_block(std::move(BB_endif));
9152       bld.reset(ctx.block);
9153       append_logical_start(ctx.block);
9154    }
9155
9156    program->config->float_mode = program->blocks[0].fp_mode.val;
9157
9158    append_logical_end(ctx.block);
9159    ctx.block->kind |= block_kind_uniform;
9160    bld.sopp(aco_opcode::s_endpgm);
9161
9162    cleanup_cfg(program);
9163 }
9164 }