src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <stack>
  29 #include <map>
  30
  31 #include "ac_shader_util.h"
  32 #include "aco_ir.h"
  33 #include "aco_builder.h"
  34 #include "aco_interface.h"
  35 #include "aco_instruction_selection_setup.cpp"
  36 #include "util/fast_idiv_by_const.h"
  37
  38 namespace aco {
  39 namespace {
  40
  41 class loop_info_RAII {
  42    isel_context* ctx;
  43    unsigned header_idx_old;
  44    Block* exit_old;
  45    bool divergent_cont_old;
  46    bool divergent_branch_old;
  47    bool divergent_if_old;
  48
  49 public:
  50    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  51       : ctx(ctx),
  52         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  53         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  54         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  55         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  56    {
  57       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  58       ctx->cf_info.parent_loop.exit = loop_exit;
  59       ctx->cf_info.parent_loop.has_divergent_continue = false;
  60       ctx->cf_info.parent_loop.has_divergent_branch = false;
  61       ctx->cf_info.parent_if.is_divergent = false;
  62       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  63    }
  64
  65    ~loop_info_RAII()
  66    {
  67       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  68       ctx->cf_info.parent_loop.exit = exit_old;
  69       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  70       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  71       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  72       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  73       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  74          ctx->cf_info.exec_potentially_empty = false;
  75    }
  76 };
  77
  78 struct if_context {
  79    Temp cond;
  80
  81    bool divergent_old;
  82    bool exec_potentially_empty_old;
  83
  84    unsigned BB_if_idx;
  85    unsigned invert_idx;
  86    bool then_branch_divergent;
  87    Block BB_invert;
  88    Block BB_endif;
  89 };
  90
  91 static void visit_cf_list(struct isel_context *ctx,
  92                           struct exec_list *list);
  93
  94 static void add_logical_edge(unsigned pred_idx, Block *succ)
  95 {
  96    succ->logical_preds.emplace_back(pred_idx);
  97 }
  98
  99
 100 static void add_linear_edge(unsigned pred_idx, Block *succ)
 101 {
 102    succ->linear_preds.emplace_back(pred_idx);
 103 }
 104
 105 static void add_edge(unsigned pred_idx, Block *succ)
 106 {
 107    add_logical_edge(pred_idx, succ);
 108    add_linear_edge(pred_idx, succ);
 109 }
 110
 111 static void append_logical_start(Block *b)
 112 {
 113    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 114 }
 115
 116 static void append_logical_end(Block *b)
 117 {
 118    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 119 }
 120
 121 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 122 {
 123    assert(ctx->allocated[def->index].id());
 124    return ctx->allocated[def->index];
 125 }
 126
 127 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 128                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 129 {
 130    Builder bld(ctx->program, ctx->block);
 131    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 132    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 133
 134    if (ctx->program->wave_size == 32) {
 135       return thread_id_lo;
 136    } else {
 137       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 138       return thread_id_hi;
 139    }
 140 }
 141
 142 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 143 {
 144    Builder bld(ctx->program, ctx->block);
 145
 146    if (!dst.id())
 147       dst = bld.tmp(src.regClass());
 148
 149    assert(src.size() == dst.size());
 150
 151    if (ctx->stage != fragment_fs) {
 152       if (!dst.id())
 153          return src;
 154
 155       bld.copy(Definition(dst), src);
 156       return dst;
 157    }
 158
 159    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 160    ctx->program->needs_wqm |= program_needs_wqm;
 161    return dst;
 162 }
 163
 164 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 165 {
 166    if (index.regClass() == s1)
 167       return bld.readlane(bld.def(s1), data, index);
 168
 169    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 170
 171    /* Currently not implemented on GFX6-7 */
 172    assert(ctx->options->chip_class >= GFX8);
 173
 174    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 175       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 176    }
 177
 178    /* GFX10, wave64 mode:
 179     * The bpermute instruction is limited to half-wave operation, which means that it can't
 180     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 181     * emulate it here.
 182     */
 183    if (!ctx->has_gfx10_wave64_bpermute) {
 184       ctx->has_gfx10_wave64_bpermute = true;
 185       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 186       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 187    }
 188
 189    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 190    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 191    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 192    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi);
 193
 194    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 195                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 196 }
 197
 198 Temp as_vgpr(isel_context *ctx, Temp val)
 199 {
 200    if (val.type() == RegType::sgpr) {
 201       Builder bld(ctx->program, ctx->block);
 202       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 203    }
 204    assert(val.type() == RegType::vgpr);
 205    return val;
 206 }
 207
 208 //assumes a != 0xffffffff
 209 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 210 {
 211    assert(b != 0);
 212    Builder bld(ctx->program, ctx->block);
 213
 214    if (util_is_power_of_two_or_zero(b)) {
 215       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 216       return;
 217    }
 218
 219    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 220
 221    assert(info.multiplier <= 0xffffffff);
 222
 223    bool pre_shift = info.pre_shift != 0;
 224    bool increment = info.increment != 0;
 225    bool multiply = true;
 226    bool post_shift = info.post_shift != 0;
 227
 228    if (!pre_shift && !increment && !multiply && !post_shift) {
 229       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 230       return;
 231    }
 232
 233    Temp pre_shift_dst = a;
 234    if (pre_shift) {
 235       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 237    }
 238
 239    Temp increment_dst = pre_shift_dst;
 240    if (increment) {
 241       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 242       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 243    }
 244
 245    Temp multiply_dst = increment_dst;
 246    if (multiply) {
 247       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 248       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 249                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 250    }
 251
 252    if (post_shift) {
 253       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 254    }
 255 }
 256
 257 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 258 {
 259    Builder bld(ctx->program, ctx->block);
 260    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 261 }
 262
 263
 264 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 265 {
 266    /* no need to extract the whole vector */
 267    if (src.regClass() == dst_rc) {
 268       assert(idx == 0);
 269       return src;
 270    }
 271    assert(src.size() > idx);
 272    Builder bld(ctx->program, ctx->block);
 273    auto it = ctx->allocated_vec.find(src.id());
 274    /* the size check needs to be early because elements other than 0 may be garbage */
 275    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 276       if (it->second[idx].regClass() == dst_rc) {
 277          return it->second[idx];
 278       } else {
 279          assert(dst_rc.size() == it->second[idx].regClass().size());
 280          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 281          return bld.copy(bld.def(dst_rc), it->second[idx]);
 282       }
 283    }
 284
 285    if (src.size() == dst_rc.size()) {
 286       assert(idx == 0);
 287       return bld.copy(bld.def(dst_rc), src);
 288    } else {
 289       Temp dst = bld.tmp(dst_rc);
 290       emit_extract_vector(ctx, src, idx, dst);
 291       return dst;
 292    }
 293 }
 294
 295 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 296 {
 297    if (num_components == 1)
 298       return;
 299    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 300       return;
 301    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 302    split->operands[0] = Operand(vec_src);
 303    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 304    for (unsigned i = 0; i < num_components; i++) {
 305       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 306       split->definitions[i] = Definition(elems[i]);
 307    }
 308    ctx->block->instructions.emplace_back(std::move(split));
 309    ctx->allocated_vec.emplace(vec_src.id(), elems);
 310 }
 311
 312 /* This vector expansion uses a mask to determine which elements in the new vector
 313  * come from the original vector. The other elements are undefined. */
 314 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 315 {
 316    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 317
 318    if (vec_src == dst)
 319       return;
 320
 321    Builder bld(ctx->program, ctx->block);
 322    if (num_components == 1) {
 323       if (dst.type() == RegType::sgpr)
 324          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 325       else
 326          bld.copy(Definition(dst), vec_src);
 327       return;
 328    }
 329
 330    unsigned component_size = dst.size() / num_components;
 331    std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 332
 333    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 334    vec->definitions[0] = Definition(dst);
 335    unsigned k = 0;
 336    for (unsigned i = 0; i < num_components; i++) {
 337       if (mask & (1 << i)) {
 338          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 339          if (dst.type() == RegType::sgpr)
 340             src = bld.as_uniform(src);
 341          vec->operands[i] = Operand(src);
 342       } else {
 343          vec->operands[i] = Operand(0u);
 344       }
 345       elems[i] = vec->operands[i].getTemp();
 346    }
 347    ctx->block->instructions.emplace_back(std::move(vec));
 348    ctx->allocated_vec.emplace(dst.id(), elems);
 349 }
 350
 351 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 352 {
 353    Builder bld(ctx->program, ctx->block);
 354    if (!dst.id())
 355       dst = bld.tmp(bld.lm);
 356
 357    assert(val.regClass() == s1);
 358    assert(dst.regClass() == bld.lm);
 359
 360    return bld.sop2(Builder::s_cselect, Definition(dst), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 361 }
 362
 363 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 364 {
 365    Builder bld(ctx->program, ctx->block);
 366    if (!dst.id())
 367       dst = bld.tmp(s1);
 368
 369    assert(val.regClass() == bld.lm);
 370    assert(dst.regClass() == s1);
 371
 372    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 373    Temp tmp = bld.tmp(s1);
 374    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 375    return emit_wqm(ctx, tmp, dst);
 376 }
 377
 378 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 379 {
 380    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 381       return get_ssa_temp(ctx, src.src.ssa);
 382
 383    if (src.src.ssa->num_components == size) {
 384       bool identity_swizzle = true;
 385       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 386          if (src.swizzle[i] != i)
 387             identity_swizzle = false;
 388       }
 389       if (identity_swizzle)
 390          return get_ssa_temp(ctx, src.src.ssa);
 391    }
 392
 393    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 394    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 395    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 396    assert(vec.size() % elem_size == 0);
 397
 398    RegClass elem_rc = RegClass(vec.type(), elem_size);
 399    if (size == 1) {
 400       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 401    } else {
 402       assert(size <= 4);
 403       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 404       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 405       for (unsigned i = 0; i < size; ++i) {
 406          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 407          vec_instr->operands[i] = Operand{elems[i]};
 408       }
 409       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 410       vec_instr->definitions[0] = Definition(dst);
 411       ctx->block->instructions.emplace_back(std::move(vec_instr));
 412       ctx->allocated_vec.emplace(dst.id(), elems);
 413       return dst;
 414    }
 415 }
 416
 417 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 418 {
 419    if (ptr.size() == 2)
 420       return ptr;
 421    Builder bld(ctx->program, ctx->block);
 422    if (ptr.type() == RegType::vgpr)
 423       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 424    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 425                      ptr, Operand((unsigned)ctx->options->address32_hi));
 426 }
 427
 428 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 429 {
 430    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 431    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 432    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 433    sop2->definitions[0] = Definition(dst);
 434    if (writes_scc)
 435       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 436    ctx->block->instructions.emplace_back(std::move(sop2));
 437 }
 438
 439 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 440                            bool commutative, bool swap_srcs=false, bool flush_denorms = false)
 441 {
 442    Builder bld(ctx->program, ctx->block);
 443    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 444    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 445    if (src1.type() == RegType::sgpr) {
 446       if (commutative && src0.type() == RegType::vgpr) {
 447          Temp t = src0;
 448          src0 = src1;
 449          src1 = t;
 450       } else if (src0.type() == RegType::vgpr &&
 451                  op != aco_opcode::v_madmk_f32 &&
 452                  op != aco_opcode::v_madak_f32 &&
 453                  op != aco_opcode::v_madmk_f16 &&
 454                  op != aco_opcode::v_madak_f16) {
 455          /* If the instruction is not commutative, we emit a VOP3A instruction */
 456          bld.vop2_e64(op, Definition(dst), src0, src1);
 457          return;
 458       } else {
 459          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 460       }
 461    }
 462
 463    if (flush_denorms && ctx->program->chip_class < GFX9) {
 464       assert(dst.size() == 1);
 465       Temp tmp = bld.vop2(op, bld.def(v1), src0, src1);
 466       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 467    } else {
 468       bld.vop2(op, Definition(dst), src0, src1);
 469    }
 470 }
 471
 472 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst,
 473                             bool flush_denorms = false)
 474 {
 475    Temp src0 = get_alu_src(ctx, instr->src[0]);
 476    Temp src1 = get_alu_src(ctx, instr->src[1]);
 477    Temp src2 = get_alu_src(ctx, instr->src[2]);
 478
 479    /* ensure that the instruction has at most 1 sgpr operand
 480     * The optimizer will inline constants for us */
 481    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 482       src0 = as_vgpr(ctx, src0);
 483    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 484       src1 = as_vgpr(ctx, src1);
 485    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 486       src2 = as_vgpr(ctx, src2);
 487
 488    Builder bld(ctx->program, ctx->block);
 489    if (flush_denorms && ctx->program->chip_class < GFX9) {
 490       assert(dst.size() == 1);
 491       Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2);
 492       bld.vop2(aco_opcode::v_mul_f32, Definition(dst), Operand(0x3f800000u), tmp);
 493    } else {
 494       bld.vop3(op, Definition(dst), src0, src1, src2);
 495    }
 496 }
 497
 498 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 499 {
 500    Builder bld(ctx->program, ctx->block);
 501    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 502 }
 503
 504 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 505 {
 506    Temp src0 = get_alu_src(ctx, instr->src[0]);
 507    Temp src1 = get_alu_src(ctx, instr->src[1]);
 508    assert(src0.size() == src1.size());
 509
 510    aco_ptr<Instruction> vopc;
 511    if (src1.type() == RegType::sgpr) {
 512       if (src0.type() == RegType::vgpr) {
 513          /* to swap the operands, we might also have to change the opcode */
 514          switch (op) {
 515             case aco_opcode::v_cmp_lt_f32:
 516                op = aco_opcode::v_cmp_gt_f32;
 517                break;
 518             case aco_opcode::v_cmp_ge_f32:
 519                op = aco_opcode::v_cmp_le_f32;
 520                break;
 521             case aco_opcode::v_cmp_lt_i32:
 522                op = aco_opcode::v_cmp_gt_i32;
 523                break;
 524             case aco_opcode::v_cmp_ge_i32:
 525                op = aco_opcode::v_cmp_le_i32;
 526                break;
 527             case aco_opcode::v_cmp_lt_u32:
 528                op = aco_opcode::v_cmp_gt_u32;
 529                break;
 530             case aco_opcode::v_cmp_ge_u32:
 531                op = aco_opcode::v_cmp_le_u32;
 532                break;
 533             case aco_opcode::v_cmp_lt_f64:
 534                op = aco_opcode::v_cmp_gt_f64;
 535                break;
 536             case aco_opcode::v_cmp_ge_f64:
 537                op = aco_opcode::v_cmp_le_f64;
 538                break;
 539             case aco_opcode::v_cmp_lt_i64:
 540                op = aco_opcode::v_cmp_gt_i64;
 541                break;
 542             case aco_opcode::v_cmp_ge_i64:
 543                op = aco_opcode::v_cmp_le_i64;
 544                break;
 545             case aco_opcode::v_cmp_lt_u64:
 546                op = aco_opcode::v_cmp_gt_u64;
 547                break;
 548             case aco_opcode::v_cmp_ge_u64:
 549                op = aco_opcode::v_cmp_le_u64;
 550                break;
 551             default: /* eq and ne are commutative */
 552                break;
 553          }
 554          Temp t = src0;
 555          src0 = src1;
 556          src1 = t;
 557       } else {
 558          src1 = as_vgpr(ctx, src1);
 559       }
 560    }
 561
 562    Builder bld(ctx->program, ctx->block);
 563    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 564 }
 565
 566 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 567 {
 568    Temp src0 = get_alu_src(ctx, instr->src[0]);
 569    Temp src1 = get_alu_src(ctx, instr->src[1]);
 570    Builder bld(ctx->program, ctx->block);
 571
 572    assert(dst.regClass() == bld.lm);
 573    assert(src0.type() == RegType::sgpr);
 574    assert(src1.type() == RegType::sgpr);
 575    assert(src0.regClass() == src1.regClass());
 576
 577    /* Emit the SALU comparison instruction */
 578    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 579    /* Turn the result into a per-lane bool */
 580    bool_to_vector_condition(ctx, cmp, dst);
 581 }
 582
 583 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 584                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 585 {
 586    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 587    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 588    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 589    bool use_valu = s_op == aco_opcode::num_opcodes ||
 590                    divergent_vals ||
 591                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 592                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 593    aco_opcode op = use_valu ? v_op : s_op;
 594    assert(op != aco_opcode::num_opcodes);
 595    assert(dst.regClass() == ctx->program->lane_mask);
 596
 597    if (use_valu)
 598       emit_vopc_instruction(ctx, instr, op, dst);
 599    else
 600       emit_sopc_instruction(ctx, instr, op, dst);
 601 }
 602
 603 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 604 {
 605    Builder bld(ctx->program, ctx->block);
 606    Temp src0 = get_alu_src(ctx, instr->src[0]);
 607    Temp src1 = get_alu_src(ctx, instr->src[1]);
 608
 609    assert(dst.regClass() == bld.lm);
 610    assert(src0.regClass() == bld.lm);
 611    assert(src1.regClass() == bld.lm);
 612
 613    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 614 }
 615
 616 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 617 {
 618    Builder bld(ctx->program, ctx->block);
 619    Temp cond = get_alu_src(ctx, instr->src[0]);
 620    Temp then = get_alu_src(ctx, instr->src[1]);
 621    Temp els = get_alu_src(ctx, instr->src[2]);
 622
 623    assert(cond.regClass() == bld.lm);
 624
 625    if (dst.type() == RegType::vgpr) {
 626       aco_ptr<Instruction> bcsel;
 627       if (dst.size() == 1) {
 628          then = as_vgpr(ctx, then);
 629          els = as_vgpr(ctx, els);
 630
 631          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 632       } else if (dst.size() == 2) {
 633          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 634          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 635          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 636          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 637
 638          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 639          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 640
 641          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 642       } else {
 643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 644          nir_print_instr(&instr->instr, stderr);
 645          fprintf(stderr, "\n");
 646       }
 647       return;
 648    }
 649
 650    if (instr->dest.dest.ssa.bit_size == 1) {
 651       assert(dst.regClass() == bld.lm);
 652       assert(then.regClass() == bld.lm);
 653       assert(els.regClass() == bld.lm);
 654    }
 655
 656    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 657       if (dst.regClass() == s1 || dst.regClass() == s2) {
 658          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 659          assert(dst.size() == then.size());
 660          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 661          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 662       } else {
 663          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 664          nir_print_instr(&instr->instr, stderr);
 665          fprintf(stderr, "\n");
 666       }
 667       return;
 668    }
 669
 670    /* divergent boolean bcsel
 671     * this implements bcsel on bools: dst = s0 ? s1 : s2
 672     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 673    assert(instr->dest.dest.ssa.bit_size == 1);
 674
 675    if (cond.id() != then.id())
 676       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 677
 678    if (cond.id() == els.id())
 679       bld.sop1(Builder::s_mov, Definition(dst), then);
 680    else
 681       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 682                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 683 }
 684
 685 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 686                     aco_opcode op, uint32_t undo)
 687 {
 688    /* multiply by 16777216 to handle denormals */
 689    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 690                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 691    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 692    scaled = bld.vop1(op, bld.def(v1), scaled);
 693    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 694
 695    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 696
 697    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 698 }
 699
 700 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 701 {
 702    if (ctx->block->fp_mode.denorm32 == 0) {
 703       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 704       return;
 705    }
 706
 707    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 708 }
 709
 710 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 711 {
 712    if (ctx->block->fp_mode.denorm32 == 0) {
 713       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 714       return;
 715    }
 716
 717    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 718 }
 719
 720 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 721 {
 722    if (ctx->block->fp_mode.denorm32 == 0) {
 723       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 724       return;
 725    }
 726
 727    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 728 }
 729
 730 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 731 {
 732    if (ctx->block->fp_mode.denorm32 == 0) {
 733       bld.vop1(aco_opcode::v_log_f32, dst, val);
 734       return;
 735    }
 736
 737    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 738 }
 739
 740 Temp emit_trunc_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 741 {
 742    if (ctx->options->chip_class >= GFX7)
 743       return bld.vop1(aco_opcode::v_trunc_f64, Definition(dst), val);
 744
 745    /* GFX6 doesn't support V_TRUNC_F64, lower it. */
 746    /* TODO: create more efficient code! */
 747    if (val.type() == RegType::sgpr)
 748       val = as_vgpr(ctx, val);
 749
 750    /* Split the input value. */
 751    Temp val_lo = bld.tmp(v1), val_hi = bld.tmp(v1);
 752    bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
 753
 754    /* Extract the exponent and compute the unbiased value. */
 755    Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f64, bld.def(v1), val);
 756
 757    /* Extract the fractional part. */
 758    Temp fract_mask = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x000fffffu));
 759    fract_mask = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), fract_mask, exponent);
 760
 761    Temp fract_mask_lo = bld.tmp(v1), fract_mask_hi = bld.tmp(v1);
 762    bld.pseudo(aco_opcode::p_split_vector, Definition(fract_mask_lo), Definition(fract_mask_hi), fract_mask);
 763
 764    Temp fract_lo = bld.tmp(v1), fract_hi = bld.tmp(v1);
 765    Temp tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_lo);
 766    fract_lo = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_lo, tmp);
 767    tmp = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), fract_mask_hi);
 768    fract_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), val_hi, tmp);
 769
 770    /* Get the sign bit. */
 771    Temp sign = bld.vop2(aco_opcode::v_ashr_i32, bld.def(v1), Operand(31u), val_hi);
 772
 773    /* Decide the operation to apply depending on the unbiased exponent. */
 774    Temp exp_lt0 = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)), exponent, Operand(0u));
 775    Temp dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_lo, bld.copy(bld.def(v1), Operand(0u)), exp_lt0);
 776    Temp dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), fract_hi, sign, exp_lt0);
 777    Temp exp_gt51 = bld.vopc_e64(aco_opcode::v_cmp_gt_i32, bld.def(s2), exponent, Operand(51u));
 778    dst_lo = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_lo, val_lo, exp_gt51);
 779    dst_hi = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), dst_hi, val_hi, exp_gt51);
 780
 781    return bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst_lo, dst_hi);
 782 }
 783
 784 Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 785 {
 786    if (ctx->options->chip_class >= GFX7)
 787       return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val);
 788
 789    /* GFX6 doesn't support V_FLOOR_F64, lower it. */
 790    Temp src0 = as_vgpr(ctx, val);
 791
 792    Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */
 793    Temp min_val = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(-1u), Operand(0x3fefffffu));
 794
 795    Temp isnan = bld.vopc_e64(aco_opcode::v_cmp_class_f64, bld.hint_vcc(bld.def(bld.lm)), src0, mask);
 796    Temp fract = bld.vop1(aco_opcode::v_fract_f64, bld.def(v2), src0);
 797    Temp min = bld.vop3(aco_opcode::v_min_f64, bld.def(v2), fract, min_val);
 798
 799    Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 800    bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), src0);
 801    Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 802    bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), min);
 803
 804    Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, isnan);
 805    Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, isnan);
 806
 807    Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), dst0, dst1);
 808
 809    Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src0, v);
 810    static_cast<VOP3A_instruction*>(add)->neg[1] = true;
 811
 812    return add->definitions[0].getTemp();
 813 }
 814
 815 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 816 {
 817    if (!instr->dest.dest.is_ssa) {
 818       fprintf(stderr, "nir alu dst not in ssa: ");
 819       nir_print_instr(&instr->instr, stderr);
 820       fprintf(stderr, "\n");
 821       abort();
 822    }
 823    Builder bld(ctx->program, ctx->block);
 824    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 825    switch(instr->op) {
 826    case nir_op_vec2:
 827    case nir_op_vec3:
 828    case nir_op_vec4: {
 829       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
 830       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 831       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 832          elems[i] = get_alu_src(ctx, instr->src[i]);
 833          vec->operands[i] = Operand{elems[i]};
 834       }
 835       vec->definitions[0] = Definition(dst);
 836       ctx->block->instructions.emplace_back(std::move(vec));
 837       ctx->allocated_vec.emplace(dst.id(), elems);
 838       break;
 839    }
 840    case nir_op_mov: {
 841       Temp src = get_alu_src(ctx, instr->src[0]);
 842       aco_ptr<Instruction> mov;
 843       if (dst.type() == RegType::sgpr) {
 844          if (src.type() == RegType::vgpr)
 845             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 846          else if (src.regClass() == s1)
 847             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 848          else if (src.regClass() == s2)
 849             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 850          else
 851             unreachable("wrong src register class for nir_op_imov");
 852       } else if (dst.regClass() == v1) {
 853          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 854       } else if (dst.regClass() == v2) {
 855          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 856       } else {
 857          nir_print_instr(&instr->instr, stderr);
 858          unreachable("Should have been lowered to scalar.");
 859       }
 860       break;
 861    }
 862    case nir_op_inot: {
 863       Temp src = get_alu_src(ctx, instr->src[0]);
 864       if (instr->dest.dest.ssa.bit_size == 1) {
 865          assert(src.regClass() == bld.lm);
 866          assert(dst.regClass() == bld.lm);
 867          /* Don't use s_andn2 here, this allows the optimizer to make a better decision */
 868          Temp tmp = bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), src);
 869          bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm));
 870       } else if (dst.regClass() == v1) {
 871          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 872       } else if (dst.type() == RegType::sgpr) {
 873          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 874          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_ineg: {
 883       Temp src = get_alu_src(ctx, instr->src[0]);
 884       if (dst.regClass() == v1) {
 885          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 886       } else if (dst.regClass() == s1) {
 887          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 888       } else if (dst.size() == 2) {
 889          Temp src0 = bld.tmp(dst.type(), 1);
 890          Temp src1 = bld.tmp(dst.type(), 1);
 891          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 892
 893          if (dst.regClass() == s2) {
 894             Temp carry = bld.tmp(s1);
 895             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 896             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 897             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 898          } else {
 899             Temp lower = bld.tmp(v1);
 900             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 901             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 902             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 903          }
 904       } else {
 905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 906          nir_print_instr(&instr->instr, stderr);
 907          fprintf(stderr, "\n");
 908       }
 909       break;
 910    }
 911    case nir_op_iabs: {
 912       if (dst.regClass() == s1) {
 913          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 914       } else if (dst.regClass() == v1) {
 915          Temp src = get_alu_src(ctx, instr->src[0]);
 916          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 917       } else {
 918          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 919          nir_print_instr(&instr->instr, stderr);
 920          fprintf(stderr, "\n");
 921       }
 922       break;
 923    }
 924    case nir_op_isign: {
 925       Temp src = get_alu_src(ctx, instr->src[0]);
 926       if (dst.regClass() == s1) {
 927          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 928          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 929          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 930       } else if (dst.regClass() == s2) {
 931          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 932          Temp neqz;
 933          if (ctx->program->chip_class >= GFX8)
 934             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 935          else
 936             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 937          /* SCC gets zero-extended to 64 bit */
 938          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 939       } else if (dst.regClass() == v1) {
 940          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 941          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 942          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 943       } else if (dst.regClass() == v2) {
 944          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 945          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 946          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 947          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 948          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 949          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 950       } else {
 951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 952          nir_print_instr(&instr->instr, stderr);
 953          fprintf(stderr, "\n");
 954       }
 955       break;
 956    }
 957    case nir_op_imax: {
 958       if (dst.regClass() == v1) {
 959          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 960       } else if (dst.regClass() == s1) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 962       } else {
 963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 964          nir_print_instr(&instr->instr, stderr);
 965          fprintf(stderr, "\n");
 966       }
 967       break;
 968    }
 969    case nir_op_umax: {
 970       if (dst.regClass() == v1) {
 971          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 972       } else if (dst.regClass() == s1) {
 973          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 974       } else {
 975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 976          nir_print_instr(&instr->instr, stderr);
 977          fprintf(stderr, "\n");
 978       }
 979       break;
 980    }
 981    case nir_op_imin: {
 982       if (dst.regClass() == v1) {
 983          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 984       } else if (dst.regClass() == s1) {
 985          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 986       } else {
 987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 988          nir_print_instr(&instr->instr, stderr);
 989          fprintf(stderr, "\n");
 990       }
 991       break;
 992    }
 993    case nir_op_umin: {
 994       if (dst.regClass() == v1) {
 995          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 996       } else if (dst.regClass() == s1) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_ior: {
1006       if (instr->dest.dest.ssa.bit_size == 1) {
1007          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
1008       } else if (dst.regClass() == v1) {
1009          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
1010       } else if (dst.regClass() == s1) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
1012       } else if (dst.regClass() == s2) {
1013          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
1014       } else {
1015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1016          nir_print_instr(&instr->instr, stderr);
1017          fprintf(stderr, "\n");
1018       }
1019       break;
1020    }
1021    case nir_op_iand: {
1022       if (instr->dest.dest.ssa.bit_size == 1) {
1023          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
1024       } else if (dst.regClass() == v1) {
1025          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
1026       } else if (dst.regClass() == s1) {
1027          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
1028       } else if (dst.regClass() == s2) {
1029          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
1030       } else {
1031          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1032          nir_print_instr(&instr->instr, stderr);
1033          fprintf(stderr, "\n");
1034       }
1035       break;
1036    }
1037    case nir_op_ixor: {
1038       if (instr->dest.dest.ssa.bit_size == 1) {
1039          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
1040       } else if (dst.regClass() == v1) {
1041          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
1042       } else if (dst.regClass() == s1) {
1043          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
1044       } else if (dst.regClass() == s2) {
1045          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
1046       } else {
1047          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1048          nir_print_instr(&instr->instr, stderr);
1049          fprintf(stderr, "\n");
1050       }
1051       break;
1052    }
1053    case nir_op_ushr: {
1054       if (dst.regClass() == v1) {
1055          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
1056       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1057          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
1058                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1059       } else if (dst.regClass() == v2) {
1060          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
1061                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1062       } else if (dst.regClass() == s2) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
1064       } else if (dst.regClass() == s1) {
1065          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
1066       } else {
1067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1068          nir_print_instr(&instr->instr, stderr);
1069          fprintf(stderr, "\n");
1070       }
1071       break;
1072    }
1073    case nir_op_ishl: {
1074       if (dst.regClass() == v1) {
1075          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
1076       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1077          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
1078                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1079       } else if (dst.regClass() == v2) {
1080          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
1081                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1082       } else if (dst.regClass() == s1) {
1083          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
1084       } else if (dst.regClass() == s2) {
1085          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_ishr: {
1094       if (dst.regClass() == v1) {
1095          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1096       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1097          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1098                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1099       } else if (dst.regClass() == v2) {
1100          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1101                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1102       } else if (dst.regClass() == s1) {
1103          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1104       } else if (dst.regClass() == s2) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1106       } else {
1107          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1108          nir_print_instr(&instr->instr, stderr);
1109          fprintf(stderr, "\n");
1110       }
1111       break;
1112    }
1113    case nir_op_find_lsb: {
1114       Temp src = get_alu_src(ctx, instr->src[0]);
1115       if (src.regClass() == s1) {
1116          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1117       } else if (src.regClass() == v1) {
1118          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1119       } else if (src.regClass() == s2) {
1120          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1121       } else {
1122          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1123          nir_print_instr(&instr->instr, stderr);
1124          fprintf(stderr, "\n");
1125       }
1126       break;
1127    }
1128    case nir_op_ufind_msb:
1129    case nir_op_ifind_msb: {
1130       Temp src = get_alu_src(ctx, instr->src[0]);
1131       if (src.regClass() == s1 || src.regClass() == s2) {
1132          aco_opcode op = src.regClass() == s2 ?
1133                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1134                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1135          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1136
1137          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1138                                         Operand(src.size() * 32u - 1u), msb_rev);
1139          Temp msb = sub.def(0).getTemp();
1140          Temp carry = sub.def(1).getTemp();
1141
1142          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1143       } else if (src.regClass() == v1) {
1144          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1145          Temp msb_rev = bld.tmp(v1);
1146          emit_vop1_instruction(ctx, instr, op, msb_rev);
1147          Temp msb = bld.tmp(v1);
1148          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1149          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1150       } else {
1151          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1152          nir_print_instr(&instr->instr, stderr);
1153          fprintf(stderr, "\n");
1154       }
1155       break;
1156    }
1157    case nir_op_bitfield_reverse: {
1158       if (dst.regClass() == s1) {
1159          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1160       } else if (dst.regClass() == v1) {
1161          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1162       } else {
1163          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1164          nir_print_instr(&instr->instr, stderr);
1165          fprintf(stderr, "\n");
1166       }
1167       break;
1168    }
1169    case nir_op_iadd: {
1170       if (dst.regClass() == s1) {
1171          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1172          break;
1173       }
1174
1175       Temp src0 = get_alu_src(ctx, instr->src[0]);
1176       Temp src1 = get_alu_src(ctx, instr->src[1]);
1177       if (dst.regClass() == v1) {
1178          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1179          break;
1180       }
1181
1182       assert(src0.size() == 2 && src1.size() == 2);
1183       Temp src00 = bld.tmp(src0.type(), 1);
1184       Temp src01 = bld.tmp(dst.type(), 1);
1185       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1186       Temp src10 = bld.tmp(src1.type(), 1);
1187       Temp src11 = bld.tmp(dst.type(), 1);
1188       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1189
1190       if (dst.regClass() == s2) {
1191          Temp carry = bld.tmp(s1);
1192          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1193          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1195       } else if (dst.regClass() == v2) {
1196          Temp dst0 = bld.tmp(v1);
1197          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1198          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1199          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1200       } else {
1201          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1202          nir_print_instr(&instr->instr, stderr);
1203          fprintf(stderr, "\n");
1204       }
1205       break;
1206    }
1207    case nir_op_uadd_sat: {
1208       Temp src0 = get_alu_src(ctx, instr->src[0]);
1209       Temp src1 = get_alu_src(ctx, instr->src[1]);
1210       if (dst.regClass() == s1) {
1211          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1212          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1213                   src0, src1);
1214          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1215       } else if (dst.regClass() == v1) {
1216          if (ctx->options->chip_class >= GFX9) {
1217             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1218             add->operands[0] = Operand(src0);
1219             add->operands[1] = Operand(src1);
1220             add->definitions[0] = Definition(dst);
1221             add->clamp = 1;
1222             ctx->block->instructions.emplace_back(std::move(add));
1223          } else {
1224             if (src1.regClass() != v1)
1225                std::swap(src0, src1);
1226             assert(src1.regClass() == v1);
1227             Temp tmp = bld.tmp(v1);
1228             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1229             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1230          }
1231       } else {
1232          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1233          nir_print_instr(&instr->instr, stderr);
1234          fprintf(stderr, "\n");
1235       }
1236       break;
1237    }
1238    case nir_op_uadd_carry: {
1239       Temp src0 = get_alu_src(ctx, instr->src[0]);
1240       Temp src1 = get_alu_src(ctx, instr->src[1]);
1241       if (dst.regClass() == s1) {
1242          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1243          break;
1244       }
1245       if (dst.regClass() == v1) {
1246          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1247          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1248          break;
1249       }
1250
1251       Temp src00 = bld.tmp(src0.type(), 1);
1252       Temp src01 = bld.tmp(dst.type(), 1);
1253       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1254       Temp src10 = bld.tmp(src1.type(), 1);
1255       Temp src11 = bld.tmp(dst.type(), 1);
1256       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1257       if (dst.regClass() == s2) {
1258          Temp carry = bld.tmp(s1);
1259          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1260          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1261          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1262       } else if (dst.regClass() == v2) {
1263          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1264          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1265          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1266          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1267       } else {
1268          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1269          nir_print_instr(&instr->instr, stderr);
1270          fprintf(stderr, "\n");
1271       }
1272       break;
1273    }
1274    case nir_op_isub: {
1275       if (dst.regClass() == s1) {
1276          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1277          break;
1278       }
1279
1280       Temp src0 = get_alu_src(ctx, instr->src[0]);
1281       Temp src1 = get_alu_src(ctx, instr->src[1]);
1282       if (dst.regClass() == v1) {
1283          bld.vsub32(Definition(dst), src0, src1);
1284          break;
1285       }
1286
1287       Temp src00 = bld.tmp(src0.type(), 1);
1288       Temp src01 = bld.tmp(dst.type(), 1);
1289       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1290       Temp src10 = bld.tmp(src1.type(), 1);
1291       Temp src11 = bld.tmp(dst.type(), 1);
1292       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1293       if (dst.regClass() == s2) {
1294          Temp carry = bld.tmp(s1);
1295          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1296          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1297          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1298       } else if (dst.regClass() == v2) {
1299          Temp lower = bld.tmp(v1);
1300          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1301          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1302          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1303       } else {
1304          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1305          nir_print_instr(&instr->instr, stderr);
1306          fprintf(stderr, "\n");
1307       }
1308       break;
1309    }
1310    case nir_op_usub_borrow: {
1311       Temp src0 = get_alu_src(ctx, instr->src[0]);
1312       Temp src1 = get_alu_src(ctx, instr->src[1]);
1313       if (dst.regClass() == s1) {
1314          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1315          break;
1316       } else if (dst.regClass() == v1) {
1317          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1318          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1319          break;
1320       }
1321
1322       Temp src00 = bld.tmp(src0.type(), 1);
1323       Temp src01 = bld.tmp(dst.type(), 1);
1324       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1325       Temp src10 = bld.tmp(src1.type(), 1);
1326       Temp src11 = bld.tmp(dst.type(), 1);
1327       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1328       if (dst.regClass() == s2) {
1329          Temp borrow = bld.tmp(s1);
1330          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1331          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1332          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1333       } else if (dst.regClass() == v2) {
1334          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1335          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1336          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1337          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_imul: {
1346       if (dst.regClass() == v1) {
1347          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1349       } else if (dst.regClass() == s1) {
1350          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1351       } else {
1352          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1353          nir_print_instr(&instr->instr, stderr);
1354          fprintf(stderr, "\n");
1355       }
1356       break;
1357    }
1358    case nir_op_umul_high: {
1359       if (dst.regClass() == v1) {
1360          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1361       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1362          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1363       } else if (dst.regClass() == s1) {
1364          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1365                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1366          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_imul_high: {
1375       if (dst.regClass() == v1) {
1376          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1377       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1378          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1379       } else if (dst.regClass() == s1) {
1380          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1381                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1382          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1383       } else {
1384          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1385          nir_print_instr(&instr->instr, stderr);
1386          fprintf(stderr, "\n");
1387       }
1388       break;
1389    }
1390    case nir_op_fmul: {
1391       if (dst.size() == 1) {
1392          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1393       } else if (dst.size() == 2) {
1394          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1395                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1396       } else {
1397          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1398          nir_print_instr(&instr->instr, stderr);
1399          fprintf(stderr, "\n");
1400       }
1401       break;
1402    }
1403    case nir_op_fadd: {
1404       if (dst.size() == 1) {
1405          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1406       } else if (dst.size() == 2) {
1407          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1408                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1409       } else {
1410          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1411          nir_print_instr(&instr->instr, stderr);
1412          fprintf(stderr, "\n");
1413       }
1414       break;
1415    }
1416    case nir_op_fsub: {
1417       Temp src0 = get_alu_src(ctx, instr->src[0]);
1418       Temp src1 = get_alu_src(ctx, instr->src[1]);
1419       if (dst.size() == 1) {
1420          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1421             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1422          else
1423             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1424       } else if (dst.size() == 2) {
1425          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1426                                      get_alu_src(ctx, instr->src[0]),
1427                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1428          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1429          sub->neg[1] = true;
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_fmax: {
1438       if (dst.size() == 1) {
1439          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1440       } else if (dst.size() == 2) {
1441          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1442             Temp tmp = bld.vop3(aco_opcode::v_max_f64, bld.def(v2),
1443                                 get_alu_src(ctx, instr->src[0]),
1444                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1445             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1446          } else {
1447             bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1448                      get_alu_src(ctx, instr->src[0]),
1449                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1450          }
1451       } else {
1452          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1453          nir_print_instr(&instr->instr, stderr);
1454          fprintf(stderr, "\n");
1455       }
1456       break;
1457    }
1458    case nir_op_fmin: {
1459       if (dst.size() == 1) {
1460          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true, false, ctx->block->fp_mode.must_flush_denorms32);
1461       } else if (dst.size() == 2) {
1462          if (ctx->block->fp_mode.must_flush_denorms16_64 && ctx->program->chip_class < GFX9) {
1463             Temp tmp = bld.vop3(aco_opcode::v_min_f64, bld.def(v2),
1464                                 get_alu_src(ctx, instr->src[0]),
1465                                 as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1466             bld.vop3(aco_opcode::v_mul_f64, Definition(dst), Operand(0x3FF0000000000000lu), tmp);
1467          } else {
1468             bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1469                      get_alu_src(ctx, instr->src[0]),
1470                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1471          }
1472       } else {
1473          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1474          nir_print_instr(&instr->instr, stderr);
1475          fprintf(stderr, "\n");
1476       }
1477       break;
1478    }
1479    case nir_op_fmax3: {
1480       if (dst.size() == 1) {
1481          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1482       } else {
1483          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1484          nir_print_instr(&instr->instr, stderr);
1485          fprintf(stderr, "\n");
1486       }
1487       break;
1488    }
1489    case nir_op_fmin3: {
1490       if (dst.size() == 1) {
1491          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fmed3: {
1500       if (dst.size() == 1) {
1501          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst, ctx->block->fp_mode.must_flush_denorms32);
1502       } else {
1503          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1504          nir_print_instr(&instr->instr, stderr);
1505          fprintf(stderr, "\n");
1506       }
1507       break;
1508    }
1509    case nir_op_umax3: {
1510       if (dst.size() == 1) {
1511          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1512       } else {
1513          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1514          nir_print_instr(&instr->instr, stderr);
1515          fprintf(stderr, "\n");
1516       }
1517       break;
1518    }
1519    case nir_op_umin3: {
1520       if (dst.size() == 1) {
1521          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1522       } else {
1523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1524          nir_print_instr(&instr->instr, stderr);
1525          fprintf(stderr, "\n");
1526       }
1527       break;
1528    }
1529    case nir_op_umed3: {
1530       if (dst.size() == 1) {
1531          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_imax3: {
1540       if (dst.size() == 1) {
1541          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_imin3: {
1550       if (dst.size() == 1) {
1551          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_imed3: {
1560       if (dst.size() == 1) {
1561          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1562       } else {
1563          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1564          nir_print_instr(&instr->instr, stderr);
1565          fprintf(stderr, "\n");
1566       }
1567       break;
1568    }
1569    case nir_op_cube_face_coord: {
1570       Temp in = get_alu_src(ctx, instr->src[0], 3);
1571       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1572                       emit_extract_vector(ctx, in, 1, v1),
1573                       emit_extract_vector(ctx, in, 2, v1) };
1574       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1575       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1576       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1577       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1578       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1579       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1580       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1581       break;
1582    }
1583    case nir_op_cube_face_index: {
1584       Temp in = get_alu_src(ctx, instr->src[0], 3);
1585       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1586                       emit_extract_vector(ctx, in, 1, v1),
1587                       emit_extract_vector(ctx, in, 2, v1) };
1588       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1589       break;
1590    }
1591    case nir_op_bcsel: {
1592       emit_bcsel(ctx, instr, dst);
1593       break;
1594    }
1595    case nir_op_frsq: {
1596       if (dst.size() == 1) {
1597          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1598       } else if (dst.size() == 2) {
1599          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1600       } else {
1601          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1602          nir_print_instr(&instr->instr, stderr);
1603          fprintf(stderr, "\n");
1604       }
1605       break;
1606    }
1607    case nir_op_fneg: {
1608       Temp src = get_alu_src(ctx, instr->src[0]);
1609       if (dst.size() == 1) {
1610          if (ctx->block->fp_mode.must_flush_denorms32)
1611             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1612          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1613       } else if (dst.size() == 2) {
1614          if (ctx->block->fp_mode.must_flush_denorms16_64)
1615             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1616          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1617          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1618          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1619          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1620       } else {
1621          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1622          nir_print_instr(&instr->instr, stderr);
1623          fprintf(stderr, "\n");
1624       }
1625       break;
1626    }
1627    case nir_op_fabs: {
1628       Temp src = get_alu_src(ctx, instr->src[0]);
1629       if (dst.size() == 1) {
1630          if (ctx->block->fp_mode.must_flush_denorms32)
1631             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1632          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1633       } else if (dst.size() == 2) {
1634          if (ctx->block->fp_mode.must_flush_denorms16_64)
1635             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1636          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1637          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1638          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1640       } else {
1641          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1642          nir_print_instr(&instr->instr, stderr);
1643          fprintf(stderr, "\n");
1644       }
1645       break;
1646    }
1647    case nir_op_fsat: {
1648       Temp src = get_alu_src(ctx, instr->src[0]);
1649       if (dst.size() == 1) {
1650          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1651          /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */
1652          // TODO: confirm that this holds under any circumstances
1653       } else if (dst.size() == 2) {
1654          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1655          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1656          vop3->clamp = true;
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_flog2: {
1665       if (dst.size() == 1) {
1666          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_frcp: {
1675       if (dst.size() == 1) {
1676          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1677       } else if (dst.size() == 2) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1679       } else {
1680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1681          nir_print_instr(&instr->instr, stderr);
1682          fprintf(stderr, "\n");
1683       }
1684       break;
1685    }
1686    case nir_op_fexp2: {
1687       if (dst.size() == 1) {
1688          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1689       } else {
1690          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1691          nir_print_instr(&instr->instr, stderr);
1692          fprintf(stderr, "\n");
1693       }
1694       break;
1695    }
1696    case nir_op_fsqrt: {
1697       if (dst.size() == 1) {
1698          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1699       } else if (dst.size() == 2) {
1700          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1701       } else {
1702          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1703          nir_print_instr(&instr->instr, stderr);
1704          fprintf(stderr, "\n");
1705       }
1706       break;
1707    }
1708    case nir_op_ffract: {
1709       if (dst.size() == 1) {
1710          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1711       } else if (dst.size() == 2) {
1712          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1713       } else {
1714          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1715          nir_print_instr(&instr->instr, stderr);
1716          fprintf(stderr, "\n");
1717       }
1718       break;
1719    }
1720    case nir_op_ffloor: {
1721       if (dst.size() == 1) {
1722          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1723       } else if (dst.size() == 2) {
1724          emit_floor_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1725       } else {
1726          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1727          nir_print_instr(&instr->instr, stderr);
1728          fprintf(stderr, "\n");
1729       }
1730       break;
1731    }
1732    case nir_op_fceil: {
1733       if (dst.size() == 1) {
1734          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1735       } else if (dst.size() == 2) {
1736          if (ctx->options->chip_class >= GFX7) {
1737             emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1738          } else {
1739             /* GFX6 doesn't support V_CEIL_F64, lower it. */
1740             Temp src0 = get_alu_src(ctx, instr->src[0]);
1741
1742             /* trunc = trunc(src0)
1743              * if (src0 > 0.0 && src0 != trunc)
1744              *    trunc += 1.0
1745              */
1746             Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src0);
1747             Temp tmp0 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.def(bld.lm), src0, Operand(0u));
1748             Temp tmp1 = bld.vopc(aco_opcode::v_cmp_lg_f64, bld.hint_vcc(bld.def(bld.lm)), src0, trunc);
1749             Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), tmp0, tmp1);
1750             Temp add = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), bld.copy(bld.def(v1), Operand(0u)), bld.copy(bld.def(v1), Operand(0x3ff00000u)), cond);
1751             add = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), bld.copy(bld.def(v1), Operand(0u)), add);
1752             bld.vop3(aco_opcode::v_add_f64, Definition(dst), trunc, add);
1753          }
1754       } else {
1755          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1756          nir_print_instr(&instr->instr, stderr);
1757          fprintf(stderr, "\n");
1758       }
1759       break;
1760    }
1761    case nir_op_ftrunc: {
1762       if (dst.size() == 1) {
1763          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1764       } else if (dst.size() == 2) {
1765          emit_trunc_f64(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1766       } else {
1767          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1768          nir_print_instr(&instr->instr, stderr);
1769          fprintf(stderr, "\n");
1770       }
1771       break;
1772    }
1773    case nir_op_fround_even: {
1774       if (dst.size() == 1) {
1775          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1776       } else if (dst.size() == 2) {
1777          if (ctx->options->chip_class >= GFX7) {
1778             emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1779          } else {
1780             /* GFX6 doesn't support V_RNDNE_F64, lower it. */
1781             Temp src0 = get_alu_src(ctx, instr->src[0]);
1782
1783             Temp src0_lo = bld.tmp(v1), src0_hi = bld.tmp(v1);
1784             bld.pseudo(aco_opcode::p_split_vector, Definition(src0_lo), Definition(src0_hi), src0);
1785
1786             Temp bitmask = bld.sop1(aco_opcode::s_brev_b32, bld.def(s1), bld.copy(bld.def(s1), Operand(-2u)));
1787             Temp bfi = bld.vop3(aco_opcode::v_bfi_b32, bld.def(v1), bitmask, bld.copy(bld.def(v1), Operand(0x43300000u)), as_vgpr(ctx, src0_hi));
1788             Temp tmp = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), src0, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1789             Instruction *sub = bld.vop3(aco_opcode::v_add_f64, bld.def(v2), tmp, bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), bfi));
1790             static_cast<VOP3A_instruction*>(sub)->neg[1] = true;
1791             tmp = sub->definitions[0].getTemp();
1792
1793             Temp v = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(-1u), Operand(0x432fffffu));
1794             Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_gt_f64, bld.hint_vcc(bld.def(bld.lm)), src0, v);
1795             static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
1796             Temp cond = vop3->definitions[0].getTemp();
1797
1798             Temp tmp_lo = bld.tmp(v1), tmp_hi = bld.tmp(v1);
1799             bld.pseudo(aco_opcode::p_split_vector, Definition(tmp_lo), Definition(tmp_hi), tmp);
1800             Temp dst0 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_lo, as_vgpr(ctx, src0_lo), cond);
1801             Temp dst1 = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp_hi, as_vgpr(ctx, src0_hi), cond);
1802
1803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1804          }
1805       } else {
1806          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1807          nir_print_instr(&instr->instr, stderr);
1808          fprintf(stderr, "\n");
1809       }
1810       break;
1811    }
1812    case nir_op_fsin:
1813    case nir_op_fcos: {
1814       Temp src = get_alu_src(ctx, instr->src[0]);
1815       aco_ptr<Instruction> norm;
1816       if (dst.size() == 1) {
1817          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1818          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1819
1820          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1821          if (ctx->options->chip_class < GFX9)
1822             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1823
1824          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1825          bld.vop1(opcode, Definition(dst), tmp);
1826       } else {
1827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1828          nir_print_instr(&instr->instr, stderr);
1829          fprintf(stderr, "\n");
1830       }
1831       break;
1832    }
1833    case nir_op_ldexp: {
1834       if (dst.size() == 1) {
1835          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1836                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1837                   get_alu_src(ctx, instr->src[1]));
1838       } else if (dst.size() == 2) {
1839          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1840                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1841                   get_alu_src(ctx, instr->src[1]));
1842       } else {
1843          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1844          nir_print_instr(&instr->instr, stderr);
1845          fprintf(stderr, "\n");
1846       }
1847       break;
1848    }
1849    case nir_op_frexp_sig: {
1850       if (dst.size() == 1) {
1851          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1852                   get_alu_src(ctx, instr->src[0]));
1853       } else if (dst.size() == 2) {
1854          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1855                   get_alu_src(ctx, instr->src[0]));
1856       } else {
1857          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1858          nir_print_instr(&instr->instr, stderr);
1859          fprintf(stderr, "\n");
1860       }
1861       break;
1862    }
1863    case nir_op_frexp_exp: {
1864       if (instr->src[0].src.ssa->bit_size == 32) {
1865          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1866                   get_alu_src(ctx, instr->src[0]));
1867       } else if (instr->src[0].src.ssa->bit_size == 64) {
1868          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1869                   get_alu_src(ctx, instr->src[0]));
1870       } else {
1871          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1872          nir_print_instr(&instr->instr, stderr);
1873          fprintf(stderr, "\n");
1874       }
1875       break;
1876    }
1877    case nir_op_fsign: {
1878       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1879       if (dst.size() == 1) {
1880          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1881          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1882          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1883          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1884       } else if (dst.size() == 2) {
1885          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1886          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1887          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1888
1889          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1890          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1891          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1892
1893          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1894       } else {
1895          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1896          nir_print_instr(&instr->instr, stderr);
1897          fprintf(stderr, "\n");
1898       }
1899       break;
1900    }
1901    case nir_op_f2f32: {
1902       if (instr->src[0].src.ssa->bit_size == 64) {
1903          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_f2f64: {
1912       if (instr->src[0].src.ssa->bit_size == 32) {
1913          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1914       } else {
1915          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1916          nir_print_instr(&instr->instr, stderr);
1917          fprintf(stderr, "\n");
1918       }
1919       break;
1920    }
1921    case nir_op_i2f32: {
1922       assert(dst.size() == 1);
1923       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1924       break;
1925    }
1926    case nir_op_i2f64: {
1927       if (instr->src[0].src.ssa->bit_size == 32) {
1928          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1929       } else if (instr->src[0].src.ssa->bit_size == 64) {
1930          Temp src = get_alu_src(ctx, instr->src[0]);
1931          RegClass rc = RegClass(src.type(), 1);
1932          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1933          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1934          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1935          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1936          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1937          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1938
1939       } else {
1940          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1941          nir_print_instr(&instr->instr, stderr);
1942          fprintf(stderr, "\n");
1943       }
1944       break;
1945    }
1946    case nir_op_u2f32: {
1947       assert(dst.size() == 1);
1948       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1949       break;
1950    }
1951    case nir_op_u2f64: {
1952       if (instr->src[0].src.ssa->bit_size == 32) {
1953          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1954       } else if (instr->src[0].src.ssa->bit_size == 64) {
1955          Temp src = get_alu_src(ctx, instr->src[0]);
1956          RegClass rc = RegClass(src.type(), 1);
1957          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1958          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1959          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1960          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1961          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1962          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_f2i32: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32) {
1973          if (dst.type() == RegType::vgpr)
1974             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1975          else
1976             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1977                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1978
1979       } else if (instr->src[0].src.ssa->bit_size == 64) {
1980          if (dst.type() == RegType::vgpr)
1981             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1982          else
1983             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1984                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1985
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_f2u32: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       if (instr->src[0].src.ssa->bit_size == 32) {
1996          if (dst.type() == RegType::vgpr)
1997             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1998          else
1999             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2000                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
2001
2002       } else if (instr->src[0].src.ssa->bit_size == 64) {
2003          if (dst.type() == RegType::vgpr)
2004             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
2005          else
2006             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
2007                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
2008
2009       } else {
2010          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2011          nir_print_instr(&instr->instr, stderr);
2012          fprintf(stderr, "\n");
2013       }
2014       break;
2015    }
2016    case nir_op_f2i64: {
2017       Temp src = get_alu_src(ctx, instr->src[0]);
2018       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2019          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2020          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
2021          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2022          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2023          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2024          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
2025          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2026          Temp new_exponent = bld.tmp(v1);
2027          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
2028          if (ctx->program->chip_class >= GFX8)
2029             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
2030          else
2031             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
2032          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
2033          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2034          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2035          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
2036          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
2037          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
2038          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
2039          Temp new_lower = bld.tmp(v1);
2040          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
2041          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
2042          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
2043
2044       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2045          if (src.type() == RegType::vgpr)
2046             src = bld.as_uniform(src);
2047          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2048          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2049          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2050          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
2051          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2052          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2053          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2054          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
2055          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2056          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
2057          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
2058          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
2059          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
2060          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
2061          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2062          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2063          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
2064          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
2065          Temp borrow = bld.tmp(s1);
2066          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
2067          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
2068          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2069
2070       } else if (instr->src[0].src.ssa->bit_size == 64) {
2071          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2072          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2073          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2074          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2075          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2076          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2077          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2078          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
2079          if (dst.type() == RegType::sgpr) {
2080             lower = bld.as_uniform(lower);
2081             upper = bld.as_uniform(upper);
2082          }
2083          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2084
2085       } else {
2086          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2087          nir_print_instr(&instr->instr, stderr);
2088          fprintf(stderr, "\n");
2089       }
2090       break;
2091    }
2092    case nir_op_f2u64: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
2095          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
2096          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
2097          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
2098          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
2099          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
2100          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
2101          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
2102          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
2103          Temp new_exponent = bld.tmp(v1);
2104          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
2105          if (ctx->program->chip_class >= GFX8)
2106             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
2107          else
2108             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
2109          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
2110          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2111          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
2112          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
2113          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
2114          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
2115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2116
2117       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
2118          if (src.type() == RegType::vgpr)
2119             src = bld.as_uniform(src);
2120          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
2121          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
2122          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
2123          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
2124          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
2125          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
2126          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
2127          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
2128          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
2129          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
2130          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
2131          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
2132          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
2133          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
2134          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
2135          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
2136          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
2137          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2138
2139       } else if (instr->src[0].src.ssa->bit_size == 64) {
2140          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
2141          Temp trunc = emit_trunc_f64(ctx, bld, bld.def(v2), src);
2142          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
2143          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
2144          Temp floor = emit_floor_f64(ctx, bld, bld.def(v2), mul);
2145          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
2146          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
2147          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
2148          if (dst.type() == RegType::sgpr) {
2149             lower = bld.as_uniform(lower);
2150             upper = bld.as_uniform(upper);
2151          }
2152          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
2153
2154       } else {
2155          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2156          nir_print_instr(&instr->instr, stderr);
2157          fprintf(stderr, "\n");
2158       }
2159       break;
2160    }
2161    case nir_op_b2f32: {
2162       Temp src = get_alu_src(ctx, instr->src[0]);
2163       assert(src.regClass() == bld.lm);
2164
2165       if (dst.regClass() == s1) {
2166          src = bool_to_scalar_condition(ctx, src);
2167          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2168       } else if (dst.regClass() == v1) {
2169          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2170       } else {
2171          unreachable("Wrong destination register class for nir_op_b2f32.");
2172       }
2173       break;
2174    }
2175    case nir_op_b2f64: {
2176       Temp src = get_alu_src(ctx, instr->src[0]);
2177       assert(src.regClass() == bld.lm);
2178
2179       if (dst.regClass() == s2) {
2180          src = bool_to_scalar_condition(ctx, src);
2181          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2182       } else if (dst.regClass() == v2) {
2183          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2184          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2185          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2186       } else {
2187          unreachable("Wrong destination register class for nir_op_b2f64.");
2188       }
2189       break;
2190    }
2191    case nir_op_i2i32: {
2192       Temp src = get_alu_src(ctx, instr->src[0]);
2193       if (instr->src[0].src.ssa->bit_size == 64) {
2194          /* we can actually just say dst = src, as it would map the lower register */
2195          emit_extract_vector(ctx, src, 0, dst);
2196       } else {
2197          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2198          nir_print_instr(&instr->instr, stderr);
2199          fprintf(stderr, "\n");
2200       }
2201       break;
2202    }
2203    case nir_op_u2u32: {
2204       Temp src = get_alu_src(ctx, instr->src[0]);
2205       if (instr->src[0].src.ssa->bit_size == 16) {
2206          if (dst.regClass() == s1) {
2207             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2208          } else {
2209             // TODO: do better with SDWA
2210             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2211          }
2212       } else if (instr->src[0].src.ssa->bit_size == 64) {
2213          /* we can actually just say dst = src, as it would map the lower register */
2214          emit_extract_vector(ctx, src, 0, dst);
2215       } else {
2216          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2217          nir_print_instr(&instr->instr, stderr);
2218          fprintf(stderr, "\n");
2219       }
2220       break;
2221    }
2222    case nir_op_i2i64: {
2223       Temp src = get_alu_src(ctx, instr->src[0]);
2224       if (src.regClass() == s1) {
2225          Temp high = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
2226          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2227       } else if (src.regClass() == v1) {
2228          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2230       } else {
2231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2232          nir_print_instr(&instr->instr, stderr);
2233          fprintf(stderr, "\n");
2234       }
2235       break;
2236    }
2237    case nir_op_u2u64: {
2238       Temp src = get_alu_src(ctx, instr->src[0]);
2239       if (instr->src[0].src.ssa->bit_size == 32) {
2240          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2241       } else {
2242          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2243          nir_print_instr(&instr->instr, stderr);
2244          fprintf(stderr, "\n");
2245       }
2246       break;
2247    }
2248    case nir_op_b2i32: {
2249       Temp src = get_alu_src(ctx, instr->src[0]);
2250       assert(src.regClass() == bld.lm);
2251
2252       if (dst.regClass() == s1) {
2253          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2254          bool_to_scalar_condition(ctx, src, dst);
2255       } else if (dst.regClass() == v1) {
2256          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2257       } else {
2258          unreachable("Invalid register class for b2i32");
2259       }
2260       break;
2261    }
2262    case nir_op_i2b1: {
2263       Temp src = get_alu_src(ctx, instr->src[0]);
2264       assert(dst.regClass() == bld.lm);
2265
2266       if (src.type() == RegType::vgpr) {
2267          assert(src.regClass() == v1 || src.regClass() == v2);
2268          assert(dst.regClass() == bld.lm);
2269          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2270                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2271       } else {
2272          assert(src.regClass() == s1 || src.regClass() == s2);
2273          Temp tmp;
2274          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2275             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2276          } else {
2277             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2278                            bld.scc(bld.def(s1)), Operand(0u), src);
2279          }
2280          bool_to_vector_condition(ctx, tmp, dst);
2281       }
2282       break;
2283    }
2284    case nir_op_pack_64_2x32_split: {
2285       Temp src0 = get_alu_src(ctx, instr->src[0]);
2286       Temp src1 = get_alu_src(ctx, instr->src[1]);
2287
2288       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2289       break;
2290    }
2291    case nir_op_unpack_64_2x32_split_x:
2292       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2293       break;
2294    case nir_op_unpack_64_2x32_split_y:
2295       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2296       break;
2297    case nir_op_pack_half_2x16: {
2298       Temp src = get_alu_src(ctx, instr->src[0], 2);
2299
2300       if (dst.regClass() == v1) {
2301          Temp src0 = bld.tmp(v1);
2302          Temp src1 = bld.tmp(v1);
2303          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2304          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2305             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2306          else
2307             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2308                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2309                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2310       } else {
2311          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2312          nir_print_instr(&instr->instr, stderr);
2313          fprintf(stderr, "\n");
2314       }
2315       break;
2316    }
2317    case nir_op_unpack_half_2x16_split_x: {
2318       if (dst.regClass() == v1) {
2319          Builder bld(ctx->program, ctx->block);
2320          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_unpack_half_2x16_split_y: {
2329       if (dst.regClass() == v1) {
2330          Builder bld(ctx->program, ctx->block);
2331          /* TODO: use SDWA here */
2332          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2333                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2334       } else {
2335          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2336          nir_print_instr(&instr->instr, stderr);
2337          fprintf(stderr, "\n");
2338       }
2339       break;
2340    }
2341    case nir_op_fquantize2f16: {
2342       Temp src = get_alu_src(ctx, instr->src[0]);
2343       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2344       Temp f32, cmp_res;
2345
2346       if (ctx->program->chip_class >= GFX8) {
2347          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2348          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2349          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2350       } else {
2351          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2352           * so compare the result and flush to 0 if it's smaller.
2353           */
2354          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2355          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2356          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), f32, smallest);
2357          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2358          cmp_res = vop3->definitions[0].getTemp();
2359       }
2360
2361       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2362          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2363          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2364       } else {
2365          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2366       }
2367       break;
2368    }
2369    case nir_op_bfm: {
2370       Temp bits = get_alu_src(ctx, instr->src[0]);
2371       Temp offset = get_alu_src(ctx, instr->src[1]);
2372
2373       if (dst.regClass() == s1) {
2374          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2375       } else if (dst.regClass() == v1) {
2376          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2377       } else {
2378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2379          nir_print_instr(&instr->instr, stderr);
2380          fprintf(stderr, "\n");
2381       }
2382       break;
2383    }
2384    case nir_op_bitfield_select: {
2385       /* (mask & insert) | (~mask & base) */
2386       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2387       Temp insert = get_alu_src(ctx, instr->src[1]);
2388       Temp base = get_alu_src(ctx, instr->src[2]);
2389
2390       /* dst = (insert & bitmask) | (base & ~bitmask) */
2391       if (dst.regClass() == s1) {
2392          aco_ptr<Instruction> sop2;
2393          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2394          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2395          Operand lhs;
2396          if (const_insert && const_bitmask) {
2397             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2398          } else {
2399             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2400             lhs = Operand(insert);
2401          }
2402
2403          Operand rhs;
2404          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2405          if (const_base && const_bitmask) {
2406             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2407          } else {
2408             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2409             rhs = Operand(base);
2410          }
2411
2412          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2413
2414       } else if (dst.regClass() == v1) {
2415          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2416             base = as_vgpr(ctx, base);
2417          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2418             insert = as_vgpr(ctx, insert);
2419
2420          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2421
2422       } else {
2423          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2424          nir_print_instr(&instr->instr, stderr);
2425          fprintf(stderr, "\n");
2426       }
2427       break;
2428    }
2429    case nir_op_ubfe:
2430    case nir_op_ibfe: {
2431       Temp base = get_alu_src(ctx, instr->src[0]);
2432       Temp offset = get_alu_src(ctx, instr->src[1]);
2433       Temp bits = get_alu_src(ctx, instr->src[2]);
2434
2435       if (dst.type() == RegType::sgpr) {
2436          Operand extract;
2437          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2438          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2439          if (const_offset && const_bits) {
2440             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2441             extract = Operand(const_extract);
2442          } else {
2443             Operand width;
2444             if (const_bits) {
2445                width = Operand(const_bits->u32 << 16);
2446             } else {
2447                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2448             }
2449             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2450          }
2451
2452          aco_opcode opcode;
2453          if (dst.regClass() == s1) {
2454             if (instr->op == nir_op_ubfe)
2455                opcode = aco_opcode::s_bfe_u32;
2456             else
2457                opcode = aco_opcode::s_bfe_i32;
2458          } else if (dst.regClass() == s2) {
2459             if (instr->op == nir_op_ubfe)
2460                opcode = aco_opcode::s_bfe_u64;
2461             else
2462                opcode = aco_opcode::s_bfe_i64;
2463          } else {
2464             unreachable("Unsupported BFE bit size");
2465          }
2466
2467          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2468
2469       } else {
2470          aco_opcode opcode;
2471          if (dst.regClass() == v1) {
2472             if (instr->op == nir_op_ubfe)
2473                opcode = aco_opcode::v_bfe_u32;
2474             else
2475                opcode = aco_opcode::v_bfe_i32;
2476          } else {
2477             unreachable("Unsupported BFE bit size");
2478          }
2479
2480          emit_vop3a_instruction(ctx, instr, opcode, dst);
2481       }
2482       break;
2483    }
2484    case nir_op_bit_count: {
2485       Temp src = get_alu_src(ctx, instr->src[0]);
2486       if (src.regClass() == s1) {
2487          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2488       } else if (src.regClass() == v1) {
2489          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2490       } else if (src.regClass() == v2) {
2491          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2492                   emit_extract_vector(ctx, src, 1, v1),
2493                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2494                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2495       } else if (src.regClass() == s2) {
2496          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2497       } else {
2498          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2499          nir_print_instr(&instr->instr, stderr);
2500          fprintf(stderr, "\n");
2501       }
2502       break;
2503    }
2504    case nir_op_flt: {
2505       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2506       break;
2507    }
2508    case nir_op_fge: {
2509       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2510       break;
2511    }
2512    case nir_op_feq: {
2513       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2514       break;
2515    }
2516    case nir_op_fne: {
2517       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2518       break;
2519    }
2520    case nir_op_ilt: {
2521       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2522       break;
2523    }
2524    case nir_op_ige: {
2525       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2526       break;
2527    }
2528    case nir_op_ieq: {
2529       if (instr->src[0].src.ssa->bit_size == 1)
2530          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2531       else
2532          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2533                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2534       break;
2535    }
2536    case nir_op_ine: {
2537       if (instr->src[0].src.ssa->bit_size == 1)
2538          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2539       else
2540          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2541                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2542       break;
2543    }
2544    case nir_op_ult: {
2545       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2546       break;
2547    }
2548    case nir_op_uge: {
2549       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2550       break;
2551    }
2552    case nir_op_fddx:
2553    case nir_op_fddy:
2554    case nir_op_fddx_fine:
2555    case nir_op_fddy_fine:
2556    case nir_op_fddx_coarse:
2557    case nir_op_fddy_coarse: {
2558       Temp src = get_alu_src(ctx, instr->src[0]);
2559       uint16_t dpp_ctrl1, dpp_ctrl2;
2560       if (instr->op == nir_op_fddx_fine) {
2561          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2562          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2563       } else if (instr->op == nir_op_fddy_fine) {
2564          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2565          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2566       } else {
2567          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2568          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2569             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2570          else
2571             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2572       }
2573
2574       Temp tmp;
2575       if (ctx->program->chip_class >= GFX8) {
2576          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2577          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2578       } else {
2579          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2580          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2581          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2582       }
2583       emit_wqm(ctx, tmp, dst, true);
2584       break;
2585    }
2586    default:
2587       fprintf(stderr, "Unknown NIR ALU instr: ");
2588       nir_print_instr(&instr->instr, stderr);
2589       fprintf(stderr, "\n");
2590    }
2591 }
2592
2593 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2594 {
2595    Temp dst = get_ssa_temp(ctx, &instr->def);
2596
2597    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2598    // which get truncated the lsb if double and msb if int
2599    // for now, we only use s_mov_b64 with 64bit inline constants
2600    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2601    assert(dst.type() == RegType::sgpr);
2602
2603    Builder bld(ctx->program, ctx->block);
2604
2605    if (instr->def.bit_size == 1) {
2606       assert(dst.regClass() == bld.lm);
2607       int val = instr->value[0].b ? -1 : 0;
2608       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2609       bld.sop1(Builder::s_mov, Definition(dst), op);
2610    } else if (dst.size() == 1) {
2611       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2612    } else {
2613       assert(dst.size() != 1);
2614       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2615       if (instr->def.bit_size == 64)
2616          for (unsigned i = 0; i < dst.size(); i++)
2617             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2618       else {
2619          for (unsigned i = 0; i < dst.size(); i++)
2620             vec->operands[i] = Operand{instr->value[i].u32};
2621       }
2622       vec->definitions[0] = Definition(dst);
2623       ctx->block->instructions.emplace_back(std::move(vec));
2624    }
2625 }
2626
2627 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2628 {
2629    uint32_t new_mask = 0;
2630    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2631       if (mask & (1u << i))
2632          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2633    return new_mask;
2634 }
2635
2636 Operand load_lds_size_m0(isel_context *ctx)
2637 {
2638    /* TODO: m0 does not need to be initialized on GFX9+ */
2639    Builder bld(ctx->program, ctx->block);
2640    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2641 }
2642
2643 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2644               Temp address, unsigned base_offset, unsigned align)
2645 {
2646    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2647
2648    Builder bld(ctx->program, ctx->block);
2649
2650    Operand m = load_lds_size_m0(ctx);
2651
2652    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2653    unsigned bytes_read = 0;
2654    unsigned result_size = 0;
2655    unsigned total_bytes = num_components * elem_size_bytes;
2656    std::array<Temp, NIR_MAX_VEC_COMPONENTS> result;
2657    bool large_ds_read = ctx->options->chip_class >= GFX7;
2658
2659    while (bytes_read < total_bytes) {
2660       unsigned todo = total_bytes - bytes_read;
2661       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2662       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2663
2664       aco_opcode op = aco_opcode::last_opcode;
2665       bool read2 = false;
2666       if (todo >= 16 && aligned16 && large_ds_read) {
2667          op = aco_opcode::ds_read_b128;
2668          todo = 16;
2669       } else if (todo >= 16 && aligned8) {
2670          op = aco_opcode::ds_read2_b64;
2671          read2 = true;
2672          todo = 16;
2673       } else if (todo >= 12 && aligned16 && large_ds_read) {
2674          op = aco_opcode::ds_read_b96;
2675          todo = 12;
2676       } else if (todo >= 8 && aligned8) {
2677          op = aco_opcode::ds_read_b64;
2678          todo = 8;
2679       } else if (todo >= 8) {
2680          op = aco_opcode::ds_read2_b32;
2681          read2 = true;
2682          todo = 8;
2683       } else if (todo >= 4) {
2684          op = aco_opcode::ds_read_b32;
2685          todo = 4;
2686       } else {
2687          assert(false);
2688       }
2689       assert(todo % elem_size_bytes == 0);
2690       unsigned num_elements = todo / elem_size_bytes;
2691       unsigned offset = base_offset + bytes_read;
2692       unsigned max_offset = read2 ? 1019 : 65535;
2693
2694       Temp address_offset = address;
2695       if (offset > max_offset) {
2696          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2697          offset = bytes_read;
2698       }
2699       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2700
2701       Temp res;
2702       if (num_components == 1 && dst.type() == RegType::vgpr)
2703          res = dst;
2704       else
2705          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2706
2707       if (read2)
2708          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2709       else
2710          res = bld.ds(op, Definition(res), address_offset, m, offset);
2711
2712       if (num_components == 1) {
2713          assert(todo == total_bytes);
2714          if (dst.type() == RegType::sgpr)
2715             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2716          return;
2717       }
2718
2719       if (dst.type() == RegType::sgpr) {
2720          Temp new_res = bld.tmp(RegType::sgpr, res.size());
2721          expand_vector(ctx, res, new_res, res.size(), (1 << res.size()) - 1);
2722          res = new_res;
2723       }
2724
2725       if (num_elements == 1) {
2726          result[result_size++] = res;
2727       } else {
2728          assert(res != dst && res.size() % num_elements == 0);
2729          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2730          split->operands[0] = Operand(res);
2731          for (unsigned i = 0; i < num_elements; i++)
2732             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2733          ctx->block->instructions.emplace_back(std::move(split));
2734       }
2735
2736       bytes_read += todo;
2737    }
2738
2739    assert(result_size == num_components && result_size > 1);
2740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2741    for (unsigned i = 0; i < result_size; i++)
2742       vec->operands[i] = Operand(result[i]);
2743    vec->definitions[0] = Definition(dst);
2744    ctx->block->instructions.emplace_back(std::move(vec));
2745    ctx->allocated_vec.emplace(dst.id(), result);
2746 }
2747
2748 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2749 {
2750    if (start == 0 && size == data.size())
2751       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2752
2753    unsigned size_hint = 1;
2754    auto it = ctx->allocated_vec.find(data.id());
2755    if (it != ctx->allocated_vec.end())
2756       size_hint = it->second[0].size();
2757    if (size % size_hint || start % size_hint)
2758       size_hint = 1;
2759
2760    start /= size_hint;
2761    size /= size_hint;
2762
2763    Temp elems[size];
2764    for (unsigned i = 0; i < size; i++)
2765       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2766
2767    if (size == 1)
2768       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2769
2770    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2771    for (unsigned i = 0; i < size; i++)
2772       vec->operands[i] = Operand(elems[i]);
2773    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2774    vec->definitions[0] = Definition(res);
2775    ctx->block->instructions.emplace_back(std::move(vec));
2776    return res;
2777 }
2778
2779 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2780 {
2781    Builder bld(ctx->program, ctx->block);
2782    unsigned bytes_written = 0;
2783    bool large_ds_write = ctx->options->chip_class >= GFX7;
2784
2785    while (bytes_written < total_size * 4) {
2786       unsigned todo = total_size * 4 - bytes_written;
2787       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2788       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2789
2790       aco_opcode op = aco_opcode::last_opcode;
2791       bool write2 = false;
2792       unsigned size = 0;
2793       if (todo >= 16 && aligned16 && large_ds_write) {
2794          op = aco_opcode::ds_write_b128;
2795          size = 4;
2796       } else if (todo >= 16 && aligned8) {
2797          op = aco_opcode::ds_write2_b64;
2798          write2 = true;
2799          size = 4;
2800       } else if (todo >= 12 && aligned16 && large_ds_write) {
2801          op = aco_opcode::ds_write_b96;
2802          size = 3;
2803       } else if (todo >= 8 && aligned8) {
2804          op = aco_opcode::ds_write_b64;
2805          size = 2;
2806       } else if (todo >= 8) {
2807          op = aco_opcode::ds_write2_b32;
2808          write2 = true;
2809          size = 2;
2810       } else if (todo >= 4) {
2811          op = aco_opcode::ds_write_b32;
2812          size = 1;
2813       } else {
2814          assert(false);
2815       }
2816
2817       unsigned offset = offset0 + offset1 + bytes_written;
2818       unsigned max_offset = write2 ? 1020 : 65535;
2819       Temp address_offset = address;
2820       if (offset > max_offset) {
2821          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2822          offset = offset1 + bytes_written;
2823       }
2824       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2825
2826       if (write2) {
2827          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2828          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2829          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2830       } else {
2831          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2832          bld.ds(op, address_offset, val, m, offset);
2833       }
2834
2835       bytes_written += size * 4;
2836    }
2837 }
2838
2839 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2840                Temp address, unsigned base_offset, unsigned align)
2841 {
2842    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2843
2844    Operand m = load_lds_size_m0(ctx);
2845
2846    /* we need at most two stores for 32bit variables */
2847    int start[2], count[2];
2848    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2849    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2850    assert(wrmask == 0);
2851
2852    /* one combined store is sufficient */
2853    if (count[0] == count[1]) {
2854       Builder bld(ctx->program, ctx->block);
2855
2856       Temp address_offset = address;
2857       if ((base_offset >> 2) + start[1] > 255) {
2858          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2859          base_offset = 0;
2860       }
2861
2862       assert(count[0] == 1);
2863       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2864       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2865       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2866       base_offset = base_offset / elem_size_bytes;
2867       bld.ds(op, address_offset, val0, val1, m,
2868              base_offset + start[0], base_offset + start[1]);
2869       return;
2870    }
2871
2872    for (unsigned i = 0; i < 2; i++) {
2873       if (count[i] == 0)
2874          continue;
2875
2876       unsigned elem_size_words = elem_size_bytes / 4;
2877       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2878                       base_offset, start[i] * elem_size_bytes, align);
2879    }
2880    return;
2881 }
2882
2883 void visit_store_vsgs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2884 {
2885    unsigned write_mask = nir_intrinsic_write_mask(instr);
2886    unsigned component = nir_intrinsic_component(instr);
2887    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2888    unsigned idx = (nir_intrinsic_base(instr) + component) * 4u;
2889    Operand offset(s1);
2890    Builder bld(ctx->program, ctx->block);
2891
2892    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2893    if (off_instr->type != nir_instr_type_load_const)
2894       offset = bld.v_mul24_imm(bld.def(v1), get_ssa_temp(ctx, instr->src[1].ssa), 16u);
2895    else
2896       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 16u;
2897
2898    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8u;
2899    if (ctx->stage == vertex_es) {
2900       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u));
2901
2902       Temp elems[NIR_MAX_VEC_COMPONENTS * 2];
2903       if (elem_size_bytes == 8) {
2904          for (unsigned i = 0; i < src.size() / 2; i++) {
2905             Temp elem = emit_extract_vector(ctx, src, i, v2);
2906             elems[i*2] = bld.tmp(v1);
2907             elems[i*2+1] = bld.tmp(v1);
2908             bld.pseudo(aco_opcode::p_split_vector, Definition(elems[i*2]), Definition(elems[i*2+1]), elem);
2909          }
2910          write_mask = widen_mask(write_mask, 2);
2911          elem_size_bytes /= 2u;
2912       } else {
2913          for (unsigned i = 0; i < src.size(); i++)
2914             elems[i] = emit_extract_vector(ctx, src, i, v1);
2915       }
2916
2917       while (write_mask) {
2918          unsigned index = u_bit_scan(&write_mask);
2919          unsigned offset = index * elem_size_bytes;
2920          Temp elem = emit_extract_vector(ctx, src, index, RegClass(RegType::vgpr, elem_size_bytes / 4));
2921
2922          Operand vaddr_offset(v1);
2923          unsigned const_offset = idx + offset;
2924          if (const_offset >= 4096u) {
2925             vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
2926             const_offset %= 4096u;
2927          }
2928
2929          aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
2930          mtbuf->operands[0] = vaddr_offset;
2931          mtbuf->operands[1] = Operand(esgs_ring);
2932          mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->es2gs_offset));
2933          mtbuf->operands[3] = Operand(elem);
2934          mtbuf->offen = !vaddr_offset.isUndefined();
2935          mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
2936          mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
2937          mtbuf->offset = const_offset;
2938          mtbuf->glc = true;
2939          mtbuf->slc = true;
2940          mtbuf->barrier = barrier_none;
2941          mtbuf->can_reorder = true;
2942          bld.insert(std::move(mtbuf));
2943       }
2944    } else {
2945       unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
2946
2947       Temp vertex_idx = emit_mbcnt(ctx, bld.def(v1));
2948       Temp wave_idx = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(ctx, ctx->args->merged_wave_info), Operand(4u << 16 | 24));
2949       vertex_idx = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), vertex_idx,
2950                             bld.v_mul24_imm(bld.def(v1), as_vgpr(ctx, wave_idx), ctx->program->wave_size));
2951
2952       Temp lds_base = bld.v_mul24_imm(bld.def(v1), vertex_idx, itemsize);
2953       if (!offset.isUndefined())
2954          lds_base = bld.vadd32(bld.def(v1), offset, lds_base);
2955
2956       unsigned align = 1 << (ffs(itemsize) - 1);
2957       if (idx)
2958          align = std::min(align, 1u << (ffs(idx) - 1));
2959
2960       store_lds(ctx, elem_size_bytes, src, write_mask, lds_base, idx, align);
2961    }
2962 }
2963
2964 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2965 {
2966    if (ctx->stage == vertex_vs ||
2967        ctx->stage == fragment_fs ||
2968        ctx->shader->info.stage == MESA_SHADER_GEOMETRY) {
2969       unsigned write_mask = nir_intrinsic_write_mask(instr);
2970       unsigned component = nir_intrinsic_component(instr);
2971       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2972       unsigned idx = nir_intrinsic_base(instr) + component;
2973
2974       nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2975       if (off_instr->type != nir_instr_type_load_const) {
2976          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2977          nir_print_instr(off_instr, stderr);
2978          fprintf(stderr, "\n");
2979       }
2980       idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2981
2982       if (instr->src[0].ssa->bit_size == 64)
2983          write_mask = widen_mask(write_mask, 2);
2984
2985       for (unsigned i = 0; i < 8; ++i) {
2986          if (write_mask & (1 << i)) {
2987             ctx->outputs.mask[idx / 4u] |= 1 << (idx % 4u);
2988             ctx->outputs.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2989          }
2990          idx++;
2991       }
2992    } else if (ctx->stage == vertex_es ||
2993               (ctx->stage == vertex_geometry_gs && ctx->shader->info.stage == MESA_SHADER_VERTEX)) {
2994       visit_store_vsgs_output(ctx, instr);
2995    } else {
2996       unreachable("Shader stage not implemented");
2997    }
2998 }
2999
3000 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
3001 {
3002    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
3003    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
3004
3005    Builder bld(ctx->program, ctx->block);
3006    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
3007    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
3008 }
3009
3010 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
3011 {
3012    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
3013    for (unsigned i = 0; i < num_components; i++)
3014       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
3015    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
3016       assert(num_components == 4);
3017       Builder bld(ctx->program, ctx->block);
3018       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
3019    }
3020
3021    for (Operand& op : vec->operands)
3022       op = op.isUndefined() ? Operand(0u) : op;
3023
3024    vec->definitions[0] = Definition(dst);
3025    ctx->block->instructions.emplace_back(std::move(vec));
3026    emit_split_vector(ctx, dst, num_components);
3027    return;
3028 }
3029
3030 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
3031 {
3032    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3033    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
3034    unsigned idx = nir_intrinsic_base(instr);
3035    unsigned component = nir_intrinsic_component(instr);
3036    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3037
3038    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
3039    if (offset) {
3040       assert(offset->u32 == 0);
3041    } else {
3042       /* the lower 15bit of the prim_mask contain the offset into LDS
3043        * while the upper bits contain the number of prims */
3044       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3045       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3046       Builder bld(ctx->program, ctx->block);
3047       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3048       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3049       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3050       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3051       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3052    }
3053
3054    if (instr->dest.ssa.num_components == 1) {
3055       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3056    } else {
3057       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3058       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3059       {
3060          Temp tmp = {ctx->program->allocateId(), v1};
3061          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3062          vec->operands[i] = Operand(tmp);
3063       }
3064       vec->definitions[0] = Definition(dst);
3065       ctx->block->instructions.emplace_back(std::move(vec));
3066    }
3067 }
3068
3069 unsigned get_num_channels_from_data_format(unsigned data_format)
3070 {
3071    switch (data_format) {
3072    case V_008F0C_BUF_DATA_FORMAT_8:
3073    case V_008F0C_BUF_DATA_FORMAT_16:
3074    case V_008F0C_BUF_DATA_FORMAT_32:
3075       return 1;
3076    case V_008F0C_BUF_DATA_FORMAT_8_8:
3077    case V_008F0C_BUF_DATA_FORMAT_16_16:
3078    case V_008F0C_BUF_DATA_FORMAT_32_32:
3079       return 2;
3080    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3081    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3082    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3083       return 3;
3084    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3085    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3086    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3087    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3088    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3089       return 4;
3090    default:
3091       break;
3092    }
3093
3094    return 4;
3095 }
3096
3097 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3098  * so we may need to fix it up. */
3099 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3100 {
3101    Builder bld(ctx->program, ctx->block);
3102
3103    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3104       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3105
3106    /* For the integer-like cases, do a natural sign extension.
3107     *
3108     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3109     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3110     * exponent.
3111     */
3112    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3113    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3114
3115    /* Convert back to the right type. */
3116    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3117       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3118       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3119       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3120    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3121       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3122    }
3123
3124    return alpha;
3125 }
3126
3127 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3128 {
3129    Builder bld(ctx->program, ctx->block);
3130    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3131    if (ctx->stage & sw_vs) {
3132
3133       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3134       if (off_instr->type != nir_instr_type_load_const) {
3135          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3136          nir_print_instr(off_instr, stderr);
3137          fprintf(stderr, "\n");
3138       }
3139       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3140
3141       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3142
3143       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3144       unsigned component = nir_intrinsic_component(instr);
3145       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3146       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3147       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3148       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3149
3150       unsigned dfmt = attrib_format & 0xf;
3151
3152       unsigned nfmt = (attrib_format >> 4) & 0x7;
3153       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3154       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3155       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3156       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3157       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3158       if (post_shuffle)
3159          num_channels = MAX2(num_channels, 3);
3160
3161       Operand off = bld.copy(bld.def(s1), Operand(attrib_binding * 16u));
3162       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, off);
3163
3164       Temp index;
3165       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3166          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3167          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3168          if (divisor) {
3169             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3170             if (divisor != 1) {
3171                Temp divided = bld.tmp(v1);
3172                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3173                index = bld.vadd32(bld.def(v1), start_instance, divided);
3174             } else {
3175                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3176             }
3177          } else {
3178             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3179          }
3180       } else {
3181          index = bld.vadd32(bld.def(v1),
3182                             get_arg(ctx, ctx->args->ac.base_vertex),
3183                             get_arg(ctx, ctx->args->ac.vertex_id));
3184       }
3185
3186       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3187          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3188          attrib_offset = attrib_offset % attrib_stride;
3189       }
3190
3191       Operand soffset(0u);
3192       if (attrib_offset >= 4096) {
3193          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3194          attrib_offset = 0;
3195       }
3196
3197       aco_opcode opcode;
3198       switch (num_channels) {
3199       case 1:
3200          opcode = aco_opcode::tbuffer_load_format_x;
3201          break;
3202       case 2:
3203          opcode = aco_opcode::tbuffer_load_format_xy;
3204          break;
3205       case 3:
3206          opcode = aco_opcode::tbuffer_load_format_xyz;
3207          break;
3208       case 4:
3209          opcode = aco_opcode::tbuffer_load_format_xyzw;
3210          break;
3211       default:
3212          unreachable("Unimplemented load_input vector size");
3213       }
3214
3215       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3216
3217       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3218       mubuf->operands[0] = Operand(index);
3219       mubuf->operands[1] = Operand(list);
3220       mubuf->operands[2] = soffset;
3221       mubuf->definitions[0] = Definition(tmp);
3222       mubuf->idxen = true;
3223       mubuf->can_reorder = true;
3224       mubuf->dfmt = dfmt;
3225       mubuf->nfmt = nfmt;
3226       assert(attrib_offset < 4096);
3227       mubuf->offset = attrib_offset;
3228       ctx->block->instructions.emplace_back(std::move(mubuf));
3229
3230       emit_split_vector(ctx, tmp, tmp.size());
3231
3232       if (tmp.id() != dst.id()) {
3233          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3234                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3235
3236          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3237          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3238          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3239
3240          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3241          for (unsigned i = 0; i < dst.size(); i++) {
3242             unsigned idx = i + component;
3243             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3244                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3245                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3246             } else if (idx < num_channels) {
3247                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3248             } else if (is_float && idx == 3) {
3249                vec->operands[i] = Operand(0x3f800000u);
3250             } else if (!is_float && idx == 3) {
3251                vec->operands[i] = Operand(1u);
3252             } else {
3253                vec->operands[i] = Operand(0u);
3254             }
3255          }
3256          vec->definitions[0] = Definition(dst);
3257          ctx->block->instructions.emplace_back(std::move(vec));
3258          emit_split_vector(ctx, dst, dst.size());
3259       }
3260
3261    } else if (ctx->stage == fragment_fs) {
3262       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3263       if (off_instr->type != nir_instr_type_load_const ||
3264           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3265          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3266          nir_print_instr(off_instr, stderr);
3267          fprintf(stderr, "\n");
3268       }
3269
3270       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3271       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3272       if (offset) {
3273          assert(offset->u32 == 0);
3274       } else {
3275          /* the lower 15bit of the prim_mask contain the offset into LDS
3276           * while the upper bits contain the number of prims */
3277          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3278          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3279          Builder bld(ctx->program, ctx->block);
3280          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3281          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3282          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3283          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3284          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3285       }
3286
3287       unsigned idx = nir_intrinsic_base(instr);
3288       unsigned component = nir_intrinsic_component(instr);
3289
3290       if (dst.size() == 1) {
3291          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3292       } else {
3293          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3294          for (unsigned i = 0; i < dst.size(); i++)
3295             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3296          vec->definitions[0] = Definition(dst);
3297          bld.insert(std::move(vec));
3298       }
3299
3300    } else {
3301       unreachable("Shader stage not implemented");
3302    }
3303 }
3304
3305 void visit_load_per_vertex_input(isel_context *ctx, nir_intrinsic_instr *instr)
3306 {
3307    assert(ctx->stage == vertex_geometry_gs || ctx->stage == geometry_gs);
3308    assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
3309
3310    Builder bld(ctx->program, ctx->block);
3311    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3312
3313    Temp offset = Temp();
3314    if (instr->src[0].ssa->parent_instr->type != nir_instr_type_load_const) {
3315       /* better code could be created, but this case probably doesn't happen
3316        * much in practice */
3317       Temp indirect_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
3318       for (unsigned i = 0; i < ctx->shader->info.gs.vertices_in; i++) {
3319          Temp elem;
3320          if (ctx->stage == vertex_geometry_gs) {
3321             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i / 2u * 2u]);
3322             if (i % 2u)
3323                elem = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), elem);
3324          } else {
3325             elem = get_arg(ctx, ctx->args->gs_vtx_offset[i]);
3326          }
3327          if (offset.id()) {
3328             Temp cond = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.hint_vcc(bld.def(s2)),
3329                                  Operand(i), indirect_vertex);
3330             offset = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), offset, elem, cond);
3331          } else {
3332             offset = elem;
3333          }
3334       }
3335       if (ctx->stage == vertex_geometry_gs)
3336          offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0xffffu), offset);
3337    } else {
3338       unsigned vertex = nir_src_as_uint(instr->src[0]);
3339       if (ctx->stage == vertex_geometry_gs)
3340          offset = bld.vop3(
3341          aco_opcode::v_bfe_u32, bld.def(v1), get_arg(ctx, ctx->args->gs_vtx_offset[vertex / 2u * 2u]),
3342             Operand((vertex % 2u) * 16u), Operand(16u));
3343       else
3344          offset = get_arg(ctx, ctx->args->gs_vtx_offset[vertex]);
3345    }
3346
3347    unsigned const_offset = nir_intrinsic_base(instr);
3348    const_offset += nir_intrinsic_component(instr);
3349
3350    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
3351    if (off_instr->type != nir_instr_type_load_const) {
3352       Temp indirect_offset = get_ssa_temp(ctx, instr->src[1].ssa);
3353       offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u),
3354                         bld.vadd32(bld.def(v1), indirect_offset, offset));
3355    } else {
3356       const_offset += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
3357    }
3358    const_offset *= 4u;
3359
3360    offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), offset);
3361
3362    unsigned itemsize = ctx->program->info->vs.es_info.esgs_itemsize;
3363
3364    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
3365    if (ctx->stage == geometry_gs) {
3366       Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_GS * 16u));
3367
3368       const_offset *= ctx->program->wave_size;
3369
3370       std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3371       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(
3372          aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1)};
3373       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++) {
3374          Temp subelems[2];
3375          for (unsigned j = 0; j < elem_size_bytes / 4; j++) {
3376             Operand soffset(0u);
3377             if (const_offset >= 4096u)
3378                soffset = bld.copy(bld.def(s1), Operand(const_offset / 4096u * 4096u));
3379
3380             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
3381             mubuf->definitions[0] = bld.def(v1);
3382             subelems[j] = mubuf->definitions[0].getTemp();
3383             mubuf->operands[0] = Operand(offset);
3384             mubuf->operands[1] = Operand(esgs_ring);
3385             mubuf->operands[2] = Operand(soffset);
3386             mubuf->offen = true;
3387             mubuf->offset = const_offset % 4096u;
3388             mubuf->glc = true;
3389             mubuf->dlc = ctx->options->chip_class >= GFX10;
3390             mubuf->barrier = barrier_none;
3391             mubuf->can_reorder = true;
3392             bld.insert(std::move(mubuf));
3393
3394             const_offset += ctx->program->wave_size * 4u;
3395          }
3396
3397          if (elem_size_bytes == 4)
3398             elems[i] = subelems[0];
3399          else
3400             elems[i] = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), subelems[0], subelems[1]);
3401          vec->operands[i] = Operand(elems[i]);
3402       }
3403       vec->definitions[0] = Definition(dst);
3404       ctx->block->instructions.emplace_back(std::move(vec));
3405       ctx->allocated_vec.emplace(dst.id(), elems);
3406    } else {
3407       unsigned align = 16; /* alignment of indirect offset */
3408       align = std::min(align, 1u << (ffs(itemsize) - 1));
3409       if (const_offset)
3410          align = std::min(align, 1u << (ffs(const_offset) - 1));
3411
3412       load_lds(ctx, elem_size_bytes, dst, offset, const_offset, align);
3413    }
3414 }
3415
3416 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3417 {
3418    if (ctx->program->info->need_indirect_descriptor_sets) {
3419       Builder bld(ctx->program, ctx->block);
3420       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3421       Operand off = bld.copy(bld.def(s1), Operand(desc_set << 2));
3422       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, off);//, false, false, false);
3423    }
3424
3425    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3426 }
3427
3428
3429 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3430 {
3431    Builder bld(ctx->program, ctx->block);
3432    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3433    if (!ctx->divergent_vals[instr->dest.ssa.index])
3434       index = bld.as_uniform(index);
3435    unsigned desc_set = nir_intrinsic_desc_set(instr);
3436    unsigned binding = nir_intrinsic_binding(instr);
3437
3438    Temp desc_ptr;
3439    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3440    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3441    unsigned offset = layout->binding[binding].offset;
3442    unsigned stride;
3443    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3444        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3445       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3446       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3447       offset = pipeline_layout->push_constant_size + 16 * idx;
3448       stride = 16;
3449    } else {
3450       desc_ptr = load_desc_ptr(ctx, desc_set);
3451       stride = layout->binding[binding].size;
3452    }
3453
3454    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3455    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3456    if (stride != 1) {
3457       if (nir_const_index) {
3458          const_index = const_index * stride;
3459       } else if (index.type() == RegType::vgpr) {
3460          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3461          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3462       } else {
3463          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3464       }
3465    }
3466    if (offset) {
3467       if (nir_const_index) {
3468          const_index = const_index + offset;
3469       } else if (index.type() == RegType::vgpr) {
3470          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3471       } else {
3472          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3473       }
3474    }
3475
3476    if (nir_const_index && const_index == 0) {
3477       index = desc_ptr;
3478    } else if (index.type() == RegType::vgpr) {
3479       index = bld.vadd32(bld.def(v1),
3480                          nir_const_index ? Operand(const_index) : Operand(index),
3481                          Operand(desc_ptr));
3482    } else {
3483       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3484                        nir_const_index ? Operand(const_index) : Operand(index),
3485                        Operand(desc_ptr));
3486    }
3487
3488    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3489 }
3490
3491 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3492                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3493 {
3494    Builder bld(ctx->program, ctx->block);
3495
3496    unsigned num_bytes = dst.size() * 4;
3497    bool dlc = glc && ctx->options->chip_class >= GFX10;
3498
3499    aco_opcode op;
3500    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3501       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3502       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3503       unsigned const_offset = 0;
3504
3505       Temp lower = Temp();
3506       if (num_bytes > 16) {
3507          assert(num_components == 3 || num_components == 4);
3508          op = aco_opcode::buffer_load_dwordx4;
3509          lower = bld.tmp(v4);
3510          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3511          mubuf->definitions[0] = Definition(lower);
3512          mubuf->operands[0] = vaddr;
3513          mubuf->operands[1] = Operand(rsrc);
3514          mubuf->operands[2] = soffset;
3515          mubuf->offen = (offset.type() == RegType::vgpr);
3516          mubuf->glc = glc;
3517          mubuf->dlc = dlc;
3518          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3519          mubuf->can_reorder = readonly;
3520          bld.insert(std::move(mubuf));
3521          emit_split_vector(ctx, lower, 2);
3522          num_bytes -= 16;
3523          const_offset = 16;
3524       } else if (num_bytes == 12 && ctx->options->chip_class == GFX6) {
3525          /* GFX6 doesn't support loading vec3, expand to vec4. */
3526          num_bytes = 16;
3527       }
3528
3529       switch (num_bytes) {
3530          case 4:
3531             op = aco_opcode::buffer_load_dword;
3532             break;
3533          case 8:
3534             op = aco_opcode::buffer_load_dwordx2;
3535             break;
3536          case 12:
3537             assert(ctx->options->chip_class > GFX6);
3538             op = aco_opcode::buffer_load_dwordx3;
3539             break;
3540          case 16:
3541             op = aco_opcode::buffer_load_dwordx4;
3542             break;
3543          default:
3544             unreachable("Load SSBO not implemented for this size.");
3545       }
3546       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3547       mubuf->operands[0] = vaddr;
3548       mubuf->operands[1] = Operand(rsrc);
3549       mubuf->operands[2] = soffset;
3550       mubuf->offen = (offset.type() == RegType::vgpr);
3551       mubuf->glc = glc;
3552       mubuf->dlc = dlc;
3553       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3554       mubuf->can_reorder = readonly;
3555       mubuf->offset = const_offset;
3556       aco_ptr<Instruction> instr = std::move(mubuf);
3557
3558       if (dst.size() > 4) {
3559          assert(lower != Temp());
3560          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3561          instr->definitions[0] = Definition(upper);
3562          bld.insert(std::move(instr));
3563          if (dst.size() == 8)
3564             emit_split_vector(ctx, upper, 2);
3565          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3566          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3567          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3568          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3569          if (dst.size() == 8)
3570             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3571       } else if (dst.size() == 3 && ctx->options->chip_class == GFX6) {
3572          Temp vec = bld.tmp(v4);
3573          instr->definitions[0] = Definition(vec);
3574          bld.insert(std::move(instr));
3575          emit_split_vector(ctx, vec, 4);
3576
3577          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
3578          instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
3579          instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
3580          instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
3581       }
3582
3583       if (dst.type() == RegType::sgpr) {
3584          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3585          instr->definitions[0] = Definition(vec);
3586          bld.insert(std::move(instr));
3587          expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
3588       } else {
3589          instr->definitions[0] = Definition(dst);
3590          bld.insert(std::move(instr));
3591          emit_split_vector(ctx, dst, num_components);
3592       }
3593    } else {
3594       switch (num_bytes) {
3595          case 4:
3596             op = aco_opcode::s_buffer_load_dword;
3597             break;
3598          case 8:
3599             op = aco_opcode::s_buffer_load_dwordx2;
3600             break;
3601          case 12:
3602          case 16:
3603             op = aco_opcode::s_buffer_load_dwordx4;
3604             break;
3605          case 24:
3606          case 32:
3607             op = aco_opcode::s_buffer_load_dwordx8;
3608             break;
3609          default:
3610             unreachable("Load SSBO not implemented for this size.");
3611       }
3612       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3613       load->operands[0] = Operand(rsrc);
3614       load->operands[1] = Operand(bld.as_uniform(offset));
3615       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3616       load->definitions[0] = Definition(dst);
3617       load->glc = glc;
3618       load->dlc = dlc;
3619       load->barrier = readonly ? barrier_none : barrier_buffer;
3620       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3621       assert(ctx->options->chip_class >= GFX8 || !glc);
3622
3623       /* trim vector */
3624       if (dst.size() == 3) {
3625          Temp vec = bld.tmp(s4);
3626          load->definitions[0] = Definition(vec);
3627          bld.insert(std::move(load));
3628          emit_split_vector(ctx, vec, 4);
3629
3630          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3631                     emit_extract_vector(ctx, vec, 0, s1),
3632                     emit_extract_vector(ctx, vec, 1, s1),
3633                     emit_extract_vector(ctx, vec, 2, s1));
3634       } else if (dst.size() == 6) {
3635          Temp vec = bld.tmp(s8);
3636          load->definitions[0] = Definition(vec);
3637          bld.insert(std::move(load));
3638          emit_split_vector(ctx, vec, 4);
3639
3640          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3641                     emit_extract_vector(ctx, vec, 0, s2),
3642                     emit_extract_vector(ctx, vec, 1, s2),
3643                     emit_extract_vector(ctx, vec, 2, s2));
3644       } else {
3645          bld.insert(std::move(load));
3646       }
3647       emit_split_vector(ctx, dst, num_components);
3648    }
3649 }
3650
3651 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3652 {
3653    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3654    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3655
3656    Builder bld(ctx->program, ctx->block);
3657
3658    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3659    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3660    unsigned binding = nir_intrinsic_binding(idx_instr);
3661    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3662
3663    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3664       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3665                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3666                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3667                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3668       if (ctx->options->chip_class >= GFX10) {
3669          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3670                       S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3671                       S_008F0C_RESOURCE_LEVEL(1);
3672       } else {
3673          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3674                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3675       }
3676       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3677                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3678                                      Operand(0xFFFFFFFFu),
3679                                      Operand(desc_type));
3680       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3681                         rsrc, upper_dwords);
3682    } else {
3683       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3684       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3685    }
3686
3687    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3688 }
3689
3690 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3691 {
3692    Builder bld(ctx->program, ctx->block);
3693    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3694
3695    unsigned offset = nir_intrinsic_base(instr);
3696    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3697    if (index_cv && instr->dest.ssa.bit_size == 32) {
3698
3699       unsigned count = instr->dest.ssa.num_components;
3700       unsigned start = (offset + index_cv->u32) / 4u;
3701       start -= ctx->args->ac.base_inline_push_consts;
3702       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3703          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3704          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3705          for (unsigned i = 0; i < count; ++i) {
3706             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3707             vec->operands[i] = Operand{elems[i]};
3708          }
3709          vec->definitions[0] = Definition(dst);
3710          ctx->block->instructions.emplace_back(std::move(vec));
3711          ctx->allocated_vec.emplace(dst.id(), elems);
3712          return;
3713       }
3714    }
3715
3716    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3717    if (offset != 0) // TODO check if index != 0 as well
3718       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3719    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3720    Temp vec = dst;
3721    bool trim = false;
3722    aco_opcode op;
3723
3724    switch (dst.size()) {
3725    case 1:
3726       op = aco_opcode::s_load_dword;
3727       break;
3728    case 2:
3729       op = aco_opcode::s_load_dwordx2;
3730       break;
3731    case 3:
3732       vec = bld.tmp(s4);
3733       trim = true;
3734    case 4:
3735       op = aco_opcode::s_load_dwordx4;
3736       break;
3737    case 6:
3738       vec = bld.tmp(s8);
3739       trim = true;
3740    case 8:
3741       op = aco_opcode::s_load_dwordx8;
3742       break;
3743    default:
3744       unreachable("unimplemented or forbidden load_push_constant.");
3745    }
3746
3747    bld.smem(op, Definition(vec), ptr, index);
3748
3749    if (trim) {
3750       emit_split_vector(ctx, vec, 4);
3751       RegClass rc = dst.size() == 3 ? s1 : s2;
3752       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3753                  emit_extract_vector(ctx, vec, 0, rc),
3754                  emit_extract_vector(ctx, vec, 1, rc),
3755                  emit_extract_vector(ctx, vec, 2, rc));
3756
3757    }
3758    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3759 }
3760
3761 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3762 {
3763    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3764
3765    Builder bld(ctx->program, ctx->block);
3766
3767    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3768                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3769                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3770                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3771    if (ctx->options->chip_class >= GFX10) {
3772       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3773                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
3774                    S_008F0C_RESOURCE_LEVEL(1);
3775    } else {
3776       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3777                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3778    }
3779
3780    unsigned base = nir_intrinsic_base(instr);
3781    unsigned range = nir_intrinsic_range(instr);
3782
3783    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3784    if (base && offset.type() == RegType::sgpr)
3785       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3786    else if (base && offset.type() == RegType::vgpr)
3787       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3788
3789    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3790                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3791                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3792                           Operand(desc_type));
3793
3794    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3795 }
3796
3797 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3798 {
3799    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3800       ctx->cf_info.exec_potentially_empty = true;
3801
3802    ctx->program->needs_exact = true;
3803
3804    // TODO: optimize uniform conditions
3805    Builder bld(ctx->program, ctx->block);
3806    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3807    assert(src.regClass() == bld.lm);
3808    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3809    bld.pseudo(aco_opcode::p_discard_if, src);
3810    ctx->block->kind |= block_kind_uses_discard_if;
3811    return;
3812 }
3813
3814 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3815 {
3816    Builder bld(ctx->program, ctx->block);
3817
3818    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3819       ctx->cf_info.exec_potentially_empty = true;
3820
3821    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3822                     ctx->cf_info.parent_loop.has_divergent_continue;
3823
3824    if (ctx->block->loop_nest_depth &&
3825        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3826       /* we handle discards the same way as jump instructions */
3827       append_logical_end(ctx->block);
3828
3829       /* in loops, discard behaves like break */
3830       Block *linear_target = ctx->cf_info.parent_loop.exit;
3831       ctx->block->kind |= block_kind_discard;
3832
3833       if (!divergent) {
3834          /* uniform discard - loop ends here */
3835          assert(nir_instr_is_last(&instr->instr));
3836          ctx->block->kind |= block_kind_uniform;
3837          ctx->cf_info.has_branch = true;
3838          bld.branch(aco_opcode::p_branch);
3839          add_linear_edge(ctx->block->index, linear_target);
3840          return;
3841       }
3842
3843       /* we add a break right behind the discard() instructions */
3844       ctx->block->kind |= block_kind_break;
3845       unsigned idx = ctx->block->index;
3846
3847       /* remove critical edges from linear CFG */
3848       bld.branch(aco_opcode::p_branch);
3849       Block* break_block = ctx->program->create_and_insert_block();
3850       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3851       break_block->kind |= block_kind_uniform;
3852       add_linear_edge(idx, break_block);
3853       add_linear_edge(break_block->index, linear_target);
3854       bld.reset(break_block);
3855       bld.branch(aco_opcode::p_branch);
3856
3857       Block* continue_block = ctx->program->create_and_insert_block();
3858       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3859       add_linear_edge(idx, continue_block);
3860       append_logical_start(continue_block);
3861       ctx->block = continue_block;
3862
3863       return;
3864    }
3865
3866    /* it can currently happen that NIR doesn't remove the unreachable code */
3867    if (!nir_instr_is_last(&instr->instr)) {
3868       ctx->program->needs_exact = true;
3869       /* save exec somewhere temporarily so that it doesn't get
3870        * overwritten before the discard from outer exec masks */
3871       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3872       bld.pseudo(aco_opcode::p_discard_if, cond);
3873       ctx->block->kind |= block_kind_uses_discard_if;
3874       return;
3875    }
3876
3877    /* This condition is incorrect for uniformly branched discards in a loop
3878     * predicated by a divergent condition, but the above code catches that case
3879     * and the discard would end up turning into a discard_if.
3880     * For example:
3881     * if (divergent) {
3882     *    while (...) {
3883     *       if (uniform) {
3884     *          discard;
3885     *       }
3886     *    }
3887     * }
3888     */
3889    if (!ctx->cf_info.parent_if.is_divergent) {
3890       /* program just ends here */
3891       ctx->block->kind |= block_kind_uniform;
3892       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3893               0 /* enabled mask */, 9 /* dest */,
3894               false /* compressed */, true/* done */, true /* valid mask */);
3895       bld.sopp(aco_opcode::s_endpgm);
3896       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3897    } else {
3898       ctx->block->kind |= block_kind_discard;
3899       /* branch and linear edge is added by visit_if() */
3900    }
3901 }
3902
3903 enum aco_descriptor_type {
3904    ACO_DESC_IMAGE,
3905    ACO_DESC_FMASK,
3906    ACO_DESC_SAMPLER,
3907    ACO_DESC_BUFFER,
3908    ACO_DESC_PLANE_0,
3909    ACO_DESC_PLANE_1,
3910    ACO_DESC_PLANE_2,
3911 };
3912
3913 static bool
3914 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3915    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3916       return false;
3917    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3918    return dim == ac_image_cube ||
3919           dim == ac_image_1darray ||
3920           dim == ac_image_2darray ||
3921           dim == ac_image_2darraymsaa;
3922 }
3923
3924 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3925                       enum aco_descriptor_type desc_type,
3926                       const nir_tex_instr *tex_instr, bool image, bool write)
3927 {
3928 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3929    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3930    if (it != ctx->tex_desc.end())
3931       return it->second;
3932 */
3933    Temp index = Temp();
3934    bool index_set = false;
3935    unsigned constant_index = 0;
3936    unsigned descriptor_set;
3937    unsigned base_index;
3938    Builder bld(ctx->program, ctx->block);
3939
3940    if (!deref_instr) {
3941       assert(tex_instr && !image);
3942       descriptor_set = 0;
3943       base_index = tex_instr->sampler_index;
3944    } else {
3945       while(deref_instr->deref_type != nir_deref_type_var) {
3946          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3947          if (!array_size)
3948             array_size = 1;
3949
3950          assert(deref_instr->deref_type == nir_deref_type_array);
3951          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3952          if (const_value) {
3953             constant_index += array_size * const_value->u32;
3954          } else {
3955             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3956             if (indirect.type() == RegType::vgpr)
3957                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3958
3959             if (array_size != 1)
3960                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3961
3962             if (!index_set) {
3963                index = indirect;
3964                index_set = true;
3965             } else {
3966                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3967             }
3968          }
3969
3970          deref_instr = nir_src_as_deref(deref_instr->parent);
3971       }
3972       descriptor_set = deref_instr->var->data.descriptor_set;
3973       base_index = deref_instr->var->data.binding;
3974    }
3975
3976    Temp list = load_desc_ptr(ctx, descriptor_set);
3977    list = convert_pointer_to_64_bit(ctx, list);
3978
3979    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3980    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3981    unsigned offset = binding->offset;
3982    unsigned stride = binding->size;
3983    aco_opcode opcode;
3984    RegClass type;
3985
3986    assert(base_index < layout->binding_count);
3987
3988    switch (desc_type) {
3989    case ACO_DESC_IMAGE:
3990       type = s8;
3991       opcode = aco_opcode::s_load_dwordx8;
3992       break;
3993    case ACO_DESC_FMASK:
3994       type = s8;
3995       opcode = aco_opcode::s_load_dwordx8;
3996       offset += 32;
3997       break;
3998    case ACO_DESC_SAMPLER:
3999       type = s4;
4000       opcode = aco_opcode::s_load_dwordx4;
4001       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
4002          offset += radv_combined_image_descriptor_sampler_offset(binding);
4003       break;
4004    case ACO_DESC_BUFFER:
4005       type = s4;
4006       opcode = aco_opcode::s_load_dwordx4;
4007       break;
4008    case ACO_DESC_PLANE_0:
4009    case ACO_DESC_PLANE_1:
4010       type = s8;
4011       opcode = aco_opcode::s_load_dwordx8;
4012       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
4013       break;
4014    case ACO_DESC_PLANE_2:
4015       type = s4;
4016       opcode = aco_opcode::s_load_dwordx4;
4017       offset += 64;
4018       break;
4019    default:
4020       unreachable("invalid desc_type\n");
4021    }
4022
4023    offset += constant_index * stride;
4024
4025    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
4026       (!index_set || binding->immutable_samplers_equal)) {
4027       if (binding->immutable_samplers_equal)
4028          constant_index = 0;
4029
4030       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
4031       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
4032                         Operand(samplers[constant_index * 4 + 0]),
4033                         Operand(samplers[constant_index * 4 + 1]),
4034                         Operand(samplers[constant_index * 4 + 2]),
4035                         Operand(samplers[constant_index * 4 + 3]));
4036    }
4037
4038    Operand off;
4039    if (!index_set) {
4040       off = bld.copy(bld.def(s1), Operand(offset));
4041    } else {
4042       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
4043                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
4044    }
4045
4046    Temp res = bld.smem(opcode, bld.def(type), list, off);
4047
4048    if (desc_type == ACO_DESC_PLANE_2) {
4049       Temp components[8];
4050       for (unsigned i = 0; i < 8; i++)
4051          components[i] = bld.tmp(s1);
4052       bld.pseudo(aco_opcode::p_split_vector,
4053                  Definition(components[0]),
4054                  Definition(components[1]),
4055                  Definition(components[2]),
4056                  Definition(components[3]),
4057                  res);
4058
4059       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
4060       bld.pseudo(aco_opcode::p_split_vector,
4061                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
4062                  Definition(components[4]),
4063                  Definition(components[5]),
4064                  Definition(components[6]),
4065                  Definition(components[7]),
4066                  desc2);
4067
4068       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
4069                        components[0], components[1], components[2], components[3],
4070                        components[4], components[5], components[6], components[7]);
4071    }
4072
4073    return res;
4074 }
4075
4076 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
4077 {
4078    switch (dim) {
4079    case GLSL_SAMPLER_DIM_BUF:
4080       return 1;
4081    case GLSL_SAMPLER_DIM_1D:
4082       return array ? 2 : 1;
4083    case GLSL_SAMPLER_DIM_2D:
4084       return array ? 3 : 2;
4085    case GLSL_SAMPLER_DIM_MS:
4086       return array ? 4 : 3;
4087    case GLSL_SAMPLER_DIM_3D:
4088    case GLSL_SAMPLER_DIM_CUBE:
4089       return 3;
4090    case GLSL_SAMPLER_DIM_RECT:
4091    case GLSL_SAMPLER_DIM_SUBPASS:
4092       return 2;
4093    case GLSL_SAMPLER_DIM_SUBPASS_MS:
4094       return 3;
4095    default:
4096       break;
4097    }
4098    return 0;
4099 }
4100
4101
4102 /* Adjust the sample index according to FMASK.
4103  *
4104  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
4105  * which is the identity mapping. Each nibble says which physical sample
4106  * should be fetched to get that sample.
4107  *
4108  * For example, 0x11111100 means there are only 2 samples stored and
4109  * the second sample covers 3/4 of the pixel. When reading samples 0
4110  * and 1, return physical sample 0 (determined by the first two 0s
4111  * in FMASK), otherwise return physical sample 1.
4112  *
4113  * The sample index should be adjusted as follows:
4114  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
4115  */
4116 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
4117 {
4118    Builder bld(ctx->program, ctx->block);
4119    Temp fmask = bld.tmp(v1);
4120    unsigned dim = ctx->options->chip_class >= GFX10
4121                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
4122                   : 0;
4123
4124    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4125    load->operands[0] = Operand(coords);
4126    load->operands[1] = Operand(fmask_desc_ptr);
4127    load->definitions[0] = Definition(fmask);
4128    load->glc = false;
4129    load->dlc = false;
4130    load->dmask = 0x1;
4131    load->unrm = true;
4132    load->da = da;
4133    load->dim = dim;
4134    load->can_reorder = true; /* fmask images shouldn't be modified */
4135    ctx->block->instructions.emplace_back(std::move(load));
4136
4137    Operand sample_index4;
4138    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
4139       sample_index4 = Operand(sample_index.constantValue() << 2);
4140    } else if (sample_index.regClass() == s1) {
4141       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
4142    } else {
4143       assert(sample_index.regClass() == v1);
4144       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
4145    }
4146
4147    Temp final_sample;
4148    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
4149       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
4150    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
4151       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
4152    else
4153       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
4154
4155    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
4156     * resource descriptor is 0 (invalid),
4157     */
4158    Temp compare = bld.tmp(bld.lm);
4159    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
4160                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
4161
4162    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
4163
4164    /* Replace the MSAA sample index. */
4165    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4166 }
4167
4168 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4169 {
4170
4171    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4172    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4173    bool is_array = glsl_sampler_type_is_array(type);
4174    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4175    assert(!add_frag_pos && "Input attachments should be lowered.");
4176    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4177    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4178    int count = image_type_to_components_count(dim, is_array);
4179    std::vector<Operand> coords(count);
4180
4181    if (is_ms) {
4182       Operand sample_index;
4183       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4184       if (sample_cv)
4185          sample_index = Operand(sample_cv->u32);
4186       else
4187          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4188
4189       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4190          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4191          for (unsigned i = 0; i < vec->operands.size(); i++)
4192             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4193          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4194          vec->definitions[0] = Definition(fmask_load_address);
4195          ctx->block->instructions.emplace_back(std::move(vec));
4196
4197          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4198          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4199       }
4200       count--;
4201       coords[count] = sample_index;
4202    }
4203
4204    if (count == 1 && !gfx9_1d)
4205       return emit_extract_vector(ctx, src0, 0, v1);
4206
4207    if (gfx9_1d) {
4208       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4209       coords.resize(coords.size() + 1);
4210       coords[1] = Operand((uint32_t) 0);
4211       if (is_array)
4212          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4213    } else {
4214       for (int i = 0; i < count; i++)
4215          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4216    }
4217
4218    if (instr->intrinsic == nir_intrinsic_image_deref_load ||
4219        instr->intrinsic == nir_intrinsic_image_deref_store) {
4220       int lod_index = instr->intrinsic == nir_intrinsic_image_deref_load ? 3 : 4;
4221       bool level_zero = nir_src_is_const(instr->src[lod_index]) && nir_src_as_uint(instr->src[lod_index]) == 0;
4222
4223       if (!level_zero)
4224          coords.emplace_back(Operand(get_ssa_temp(ctx, instr->src[lod_index].ssa)));
4225    }
4226
4227    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4228    for (unsigned i = 0; i < coords.size(); i++)
4229       vec->operands[i] = coords[i];
4230    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4231    vec->definitions[0] = Definition(res);
4232    ctx->block->instructions.emplace_back(std::move(vec));
4233    return res;
4234 }
4235
4236
4237 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4238 {
4239    Builder bld(ctx->program, ctx->block);
4240    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4241    const struct glsl_type *type = glsl_without_array(var->type);
4242    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4243    bool is_array = glsl_sampler_type_is_array(type);
4244    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4245
4246    if (dim == GLSL_SAMPLER_DIM_BUF) {
4247       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4248       unsigned num_channels = util_last_bit(mask);
4249       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4250       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4251
4252       aco_opcode opcode;
4253       switch (num_channels) {
4254       case 1:
4255          opcode = aco_opcode::buffer_load_format_x;
4256          break;
4257       case 2:
4258          opcode = aco_opcode::buffer_load_format_xy;
4259          break;
4260       case 3:
4261          opcode = aco_opcode::buffer_load_format_xyz;
4262          break;
4263       case 4:
4264          opcode = aco_opcode::buffer_load_format_xyzw;
4265          break;
4266       default:
4267          unreachable(">4 channel buffer image load");
4268       }
4269       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4270       load->operands[0] = Operand(vindex);
4271       load->operands[1] = Operand(rsrc);
4272       load->operands[2] = Operand((uint32_t) 0);
4273       Temp tmp;
4274       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4275          tmp = dst;
4276       else
4277          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4278       load->definitions[0] = Definition(tmp);
4279       load->idxen = true;
4280       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4281       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4282       load->barrier = barrier_image;
4283       ctx->block->instructions.emplace_back(std::move(load));
4284
4285       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4286       return;
4287    }
4288
4289    Temp coords = get_image_coords(ctx, instr, type);
4290    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4291
4292    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4293    unsigned num_components = util_bitcount(dmask);
4294    Temp tmp;
4295    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4296       tmp = dst;
4297    else
4298       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4299
4300    bool level_zero = nir_src_is_const(instr->src[3]) && nir_src_as_uint(instr->src[3]) == 0;
4301    aco_opcode opcode = level_zero ? aco_opcode::image_load : aco_opcode::image_load_mip;
4302
4303    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 2, 1)};
4304    load->operands[0] = Operand(coords);
4305    load->operands[1] = Operand(resource);
4306    load->definitions[0] = Definition(tmp);
4307    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4308    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4309    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4310    load->dmask = dmask;
4311    load->unrm = true;
4312    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4313    load->barrier = barrier_image;
4314    ctx->block->instructions.emplace_back(std::move(load));
4315
4316    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4317    return;
4318 }
4319
4320 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4321 {
4322    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4323    const struct glsl_type *type = glsl_without_array(var->type);
4324    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4325    bool is_array = glsl_sampler_type_is_array(type);
4326    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4327
4328    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4329
4330    if (dim == GLSL_SAMPLER_DIM_BUF) {
4331       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4332       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4333       aco_opcode opcode;
4334       switch (data.size()) {
4335       case 1:
4336          opcode = aco_opcode::buffer_store_format_x;
4337          break;
4338       case 2:
4339          opcode = aco_opcode::buffer_store_format_xy;
4340          break;
4341       case 3:
4342          opcode = aco_opcode::buffer_store_format_xyz;
4343          break;
4344       case 4:
4345          opcode = aco_opcode::buffer_store_format_xyzw;
4346          break;
4347       default:
4348          unreachable(">4 channel buffer image store");
4349       }
4350       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4351       store->operands[0] = Operand(vindex);
4352       store->operands[1] = Operand(rsrc);
4353       store->operands[2] = Operand((uint32_t) 0);
4354       store->operands[3] = Operand(data);
4355       store->idxen = true;
4356       store->glc = glc;
4357       store->dlc = false;
4358       store->disable_wqm = true;
4359       store->barrier = barrier_image;
4360       ctx->program->needs_exact = true;
4361       ctx->block->instructions.emplace_back(std::move(store));
4362       return;
4363    }
4364
4365    assert(data.type() == RegType::vgpr);
4366    Temp coords = get_image_coords(ctx, instr, type);
4367    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4368
4369    bool level_zero = nir_src_is_const(instr->src[4]) && nir_src_as_uint(instr->src[4]) == 0;
4370    aco_opcode opcode = level_zero ? aco_opcode::image_store : aco_opcode::image_store_mip;
4371
4372    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(opcode, Format::MIMG, 4, 0)};
4373    store->operands[0] = Operand(coords);
4374    store->operands[1] = Operand(resource);
4375    store->operands[2] = Operand(s4);
4376    store->operands[3] = Operand(data);
4377    store->glc = glc;
4378    store->dlc = false;
4379    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4380    store->dmask = (1 << data.size()) - 1;
4381    store->unrm = true;
4382    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4383    store->disable_wqm = true;
4384    store->barrier = barrier_image;
4385    ctx->program->needs_exact = true;
4386    ctx->block->instructions.emplace_back(std::move(store));
4387    return;
4388 }
4389
4390 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4391 {
4392    /* return the previous value if dest is ever used */
4393    bool return_previous = false;
4394    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4395       return_previous = true;
4396       break;
4397    }
4398    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4399       return_previous = true;
4400       break;
4401    }
4402
4403    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4404    const struct glsl_type *type = glsl_without_array(var->type);
4405    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4406    bool is_array = glsl_sampler_type_is_array(type);
4407    Builder bld(ctx->program, ctx->block);
4408
4409    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4410    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4411
4412    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4413       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4414
4415    aco_opcode buf_op, image_op;
4416    switch (instr->intrinsic) {
4417       case nir_intrinsic_image_deref_atomic_add:
4418          buf_op = aco_opcode::buffer_atomic_add;
4419          image_op = aco_opcode::image_atomic_add;
4420          break;
4421       case nir_intrinsic_image_deref_atomic_umin:
4422          buf_op = aco_opcode::buffer_atomic_umin;
4423          image_op = aco_opcode::image_atomic_umin;
4424          break;
4425       case nir_intrinsic_image_deref_atomic_imin:
4426          buf_op = aco_opcode::buffer_atomic_smin;
4427          image_op = aco_opcode::image_atomic_smin;
4428          break;
4429       case nir_intrinsic_image_deref_atomic_umax:
4430          buf_op = aco_opcode::buffer_atomic_umax;
4431          image_op = aco_opcode::image_atomic_umax;
4432          break;
4433       case nir_intrinsic_image_deref_atomic_imax:
4434          buf_op = aco_opcode::buffer_atomic_smax;
4435          image_op = aco_opcode::image_atomic_smax;
4436          break;
4437       case nir_intrinsic_image_deref_atomic_and:
4438          buf_op = aco_opcode::buffer_atomic_and;
4439          image_op = aco_opcode::image_atomic_and;
4440          break;
4441       case nir_intrinsic_image_deref_atomic_or:
4442          buf_op = aco_opcode::buffer_atomic_or;
4443          image_op = aco_opcode::image_atomic_or;
4444          break;
4445       case nir_intrinsic_image_deref_atomic_xor:
4446          buf_op = aco_opcode::buffer_atomic_xor;
4447          image_op = aco_opcode::image_atomic_xor;
4448          break;
4449       case nir_intrinsic_image_deref_atomic_exchange:
4450          buf_op = aco_opcode::buffer_atomic_swap;
4451          image_op = aco_opcode::image_atomic_swap;
4452          break;
4453       case nir_intrinsic_image_deref_atomic_comp_swap:
4454          buf_op = aco_opcode::buffer_atomic_cmpswap;
4455          image_op = aco_opcode::image_atomic_cmpswap;
4456          break;
4457       default:
4458          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4459    }
4460
4461    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4462
4463    if (dim == GLSL_SAMPLER_DIM_BUF) {
4464       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4465       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4466       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4467       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4468       mubuf->operands[0] = Operand(vindex);
4469       mubuf->operands[1] = Operand(resource);
4470       mubuf->operands[2] = Operand((uint32_t)0);
4471       mubuf->operands[3] = Operand(data);
4472       if (return_previous)
4473          mubuf->definitions[0] = Definition(dst);
4474       mubuf->offset = 0;
4475       mubuf->idxen = true;
4476       mubuf->glc = return_previous;
4477       mubuf->dlc = false; /* Not needed for atomics */
4478       mubuf->disable_wqm = true;
4479       mubuf->barrier = barrier_image;
4480       ctx->program->needs_exact = true;
4481       ctx->block->instructions.emplace_back(std::move(mubuf));
4482       return;
4483    }
4484
4485    Temp coords = get_image_coords(ctx, instr, type);
4486    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4487    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4488    mimg->operands[0] = Operand(coords);
4489    mimg->operands[1] = Operand(resource);
4490    mimg->operands[2] = Operand(s4); /* no sampler */
4491    mimg->operands[3] = Operand(data);
4492    if (return_previous)
4493       mimg->definitions[0] = Definition(dst);
4494    mimg->glc = return_previous;
4495    mimg->dlc = false; /* Not needed for atomics */
4496    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4497    mimg->dmask = (1 << data.size()) - 1;
4498    mimg->unrm = true;
4499    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4500    mimg->disable_wqm = true;
4501    mimg->barrier = barrier_image;
4502    ctx->program->needs_exact = true;
4503    ctx->block->instructions.emplace_back(std::move(mimg));
4504    return;
4505 }
4506
4507 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4508 {
4509    if (in_elements && ctx->options->chip_class == GFX8) {
4510       /* we only have to divide by 1, 2, 4, 8, 12 or 16 */
4511       Builder bld(ctx->program, ctx->block);
4512
4513       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4514
4515       Temp size_div3 = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), bld.copy(bld.def(v1), Operand(0xaaaaaaabu)), size);
4516       size_div3 = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.as_uniform(size_div3), Operand(1u));
4517
4518       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4519       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4520
4521       Temp is12 = bld.sopc(aco_opcode::s_cmp_eq_i32, bld.def(s1, scc), stride, Operand(12u));
4522       size = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), size_div3, size, bld.scc(is12));
4523
4524       Temp shr_dst = dst.type() == RegType::vgpr ? bld.tmp(s1) : dst;
4525       bld.sop2(aco_opcode::s_lshr_b32, Definition(shr_dst), bld.def(s1, scc),
4526                size, bld.sop1(aco_opcode::s_ff1_i32_b32, bld.def(s1), stride));
4527       if (dst.type() == RegType::vgpr)
4528          bld.copy(Definition(dst), shr_dst);
4529
4530       /* TODO: we can probably calculate this faster with v_skip when stride != 12 */
4531    } else {
4532       emit_extract_vector(ctx, desc, 2, dst);
4533    }
4534 }
4535
4536 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4537 {
4538    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4539    const struct glsl_type *type = glsl_without_array(var->type);
4540    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4541    bool is_array = glsl_sampler_type_is_array(type);
4542    Builder bld(ctx->program, ctx->block);
4543
4544    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4545       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4546       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4547    }
4548
4549    /* LOD */
4550    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4551
4552    /* Resource */
4553    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4554
4555    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4556
4557    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4558    mimg->operands[0] = Operand(lod);
4559    mimg->operands[1] = Operand(resource);
4560    uint8_t& dmask = mimg->dmask;
4561    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4562    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4563    mimg->da = glsl_sampler_type_is_array(type);
4564    mimg->can_reorder = true;
4565    Definition& def = mimg->definitions[0];
4566    ctx->block->instructions.emplace_back(std::move(mimg));
4567
4568    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4569        glsl_sampler_type_is_array(type)) {
4570
4571       assert(instr->dest.ssa.num_components == 3);
4572       Temp tmp = {ctx->program->allocateId(), v3};
4573       def = Definition(tmp);
4574       emit_split_vector(ctx, tmp, 3);
4575
4576       /* divide 3rd value by 6 by multiplying with magic number */
4577       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4578       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4579
4580       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4581                  emit_extract_vector(ctx, tmp, 0, v1),
4582                  emit_extract_vector(ctx, tmp, 1, v1),
4583                  by_6);
4584
4585    } else if (ctx->options->chip_class == GFX9 &&
4586               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4587               glsl_sampler_type_is_array(type)) {
4588       assert(instr->dest.ssa.num_components == 2);
4589       def = Definition(dst);
4590       dmask = 0x5;
4591    } else {
4592       def = Definition(dst);
4593    }
4594
4595    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4596 }
4597
4598 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4599 {
4600    Builder bld(ctx->program, ctx->block);
4601    unsigned num_components = instr->num_components;
4602
4603    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4604    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4605    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4606
4607    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4608    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4609 }
4610
4611 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4612 {
4613    Builder bld(ctx->program, ctx->block);
4614    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4615    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4616    unsigned writemask = nir_intrinsic_write_mask(instr);
4617    Temp offset = get_ssa_temp(ctx, instr->src[2].ssa);
4618
4619    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4620    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4621
4622    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4623                ctx->options->chip_class >= GFX8;
4624    if (smem)
4625       offset = bld.as_uniform(offset);
4626    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4627
4628    while (writemask) {
4629       int start, count;
4630       u_bit_scan_consecutive_range(&writemask, &start, &count);
4631       if (count == 3 && (smem || ctx->options->chip_class == GFX6)) {
4632          /* GFX6 doesn't support storing vec3, split it. */
4633          writemask |= 1u << (start + 2);
4634          count = 2;
4635       }
4636       int num_bytes = count * elem_size_bytes;
4637
4638       if (num_bytes > 16) {
4639          assert(elem_size_bytes == 8);
4640          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4641          count = 2;
4642          num_bytes = 16;
4643       }
4644
4645       // TODO: check alignment of sub-dword stores
4646       // TODO: split 3 bytes. there is no store instruction for that
4647
4648       Temp write_data;
4649       if (count != instr->num_components) {
4650          emit_split_vector(ctx, data, instr->num_components);
4651          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4652          for (int i = 0; i < count; i++) {
4653             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4654             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4655          }
4656          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4657          vec->definitions[0] = Definition(write_data);
4658          ctx->block->instructions.emplace_back(std::move(vec));
4659       } else if (!smem && data.type() != RegType::vgpr) {
4660          assert(num_bytes % 4 == 0);
4661          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4662       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4663          assert(num_bytes % 4 == 0);
4664          write_data = bld.as_uniform(data);
4665       } else {
4666          write_data = data;
4667       }
4668
4669       aco_opcode vmem_op, smem_op;
4670       switch (num_bytes) {
4671          case 4:
4672             vmem_op = aco_opcode::buffer_store_dword;
4673             smem_op = aco_opcode::s_buffer_store_dword;
4674             break;
4675          case 8:
4676             vmem_op = aco_opcode::buffer_store_dwordx2;
4677             smem_op = aco_opcode::s_buffer_store_dwordx2;
4678             break;
4679          case 12:
4680             vmem_op = aco_opcode::buffer_store_dwordx3;
4681             smem_op = aco_opcode::last_opcode;
4682             assert(!smem && ctx->options->chip_class > GFX6);
4683             break;
4684          case 16:
4685             vmem_op = aco_opcode::buffer_store_dwordx4;
4686             smem_op = aco_opcode::s_buffer_store_dwordx4;
4687             break;
4688          default:
4689             unreachable("Store SSBO not implemented for this size.");
4690       }
4691       if (ctx->stage == fragment_fs)
4692          smem_op = aco_opcode::p_fs_buffer_store_smem;
4693
4694       if (smem) {
4695          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4696          store->operands[0] = Operand(rsrc);
4697          if (start) {
4698             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4699                                 offset, Operand(start * elem_size_bytes));
4700             store->operands[1] = Operand(off);
4701          } else {
4702             store->operands[1] = Operand(offset);
4703          }
4704          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4705             store->operands[1].setFixed(m0);
4706          store->operands[2] = Operand(write_data);
4707          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4708          store->dlc = false;
4709          store->disable_wqm = true;
4710          store->barrier = barrier_buffer;
4711          ctx->block->instructions.emplace_back(std::move(store));
4712          ctx->program->wb_smem_l1_on_end = true;
4713          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4714             ctx->block->kind |= block_kind_needs_lowering;
4715             ctx->program->needs_exact = true;
4716          }
4717       } else {
4718          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4719          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4720          store->operands[1] = Operand(rsrc);
4721          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4722          store->operands[3] = Operand(write_data);
4723          store->offset = start * elem_size_bytes;
4724          store->offen = (offset.type() == RegType::vgpr);
4725          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4726          store->dlc = false;
4727          store->disable_wqm = true;
4728          store->barrier = barrier_buffer;
4729          ctx->program->needs_exact = true;
4730          ctx->block->instructions.emplace_back(std::move(store));
4731       }
4732    }
4733 }
4734
4735 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4736 {
4737    /* return the previous value if dest is ever used */
4738    bool return_previous = false;
4739    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4740       return_previous = true;
4741       break;
4742    }
4743    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4744       return_previous = true;
4745       break;
4746    }
4747
4748    Builder bld(ctx->program, ctx->block);
4749    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4750
4751    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4752       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4753                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4754
4755    Temp offset = get_ssa_temp(ctx, instr->src[1].ssa);
4756    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4757    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4758
4759    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4760
4761    aco_opcode op32, op64;
4762    switch (instr->intrinsic) {
4763       case nir_intrinsic_ssbo_atomic_add:
4764          op32 = aco_opcode::buffer_atomic_add;
4765          op64 = aco_opcode::buffer_atomic_add_x2;
4766          break;
4767       case nir_intrinsic_ssbo_atomic_imin:
4768          op32 = aco_opcode::buffer_atomic_smin;
4769          op64 = aco_opcode::buffer_atomic_smin_x2;
4770          break;
4771       case nir_intrinsic_ssbo_atomic_umin:
4772          op32 = aco_opcode::buffer_atomic_umin;
4773          op64 = aco_opcode::buffer_atomic_umin_x2;
4774          break;
4775       case nir_intrinsic_ssbo_atomic_imax:
4776          op32 = aco_opcode::buffer_atomic_smax;
4777          op64 = aco_opcode::buffer_atomic_smax_x2;
4778          break;
4779       case nir_intrinsic_ssbo_atomic_umax:
4780          op32 = aco_opcode::buffer_atomic_umax;
4781          op64 = aco_opcode::buffer_atomic_umax_x2;
4782          break;
4783       case nir_intrinsic_ssbo_atomic_and:
4784          op32 = aco_opcode::buffer_atomic_and;
4785          op64 = aco_opcode::buffer_atomic_and_x2;
4786          break;
4787       case nir_intrinsic_ssbo_atomic_or:
4788          op32 = aco_opcode::buffer_atomic_or;
4789          op64 = aco_opcode::buffer_atomic_or_x2;
4790          break;
4791       case nir_intrinsic_ssbo_atomic_xor:
4792          op32 = aco_opcode::buffer_atomic_xor;
4793          op64 = aco_opcode::buffer_atomic_xor_x2;
4794          break;
4795       case nir_intrinsic_ssbo_atomic_exchange:
4796          op32 = aco_opcode::buffer_atomic_swap;
4797          op64 = aco_opcode::buffer_atomic_swap_x2;
4798          break;
4799       case nir_intrinsic_ssbo_atomic_comp_swap:
4800          op32 = aco_opcode::buffer_atomic_cmpswap;
4801          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4802          break;
4803       default:
4804          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4805    }
4806    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4807    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4808    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4809    mubuf->operands[1] = Operand(rsrc);
4810    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4811    mubuf->operands[3] = Operand(data);
4812    if (return_previous)
4813       mubuf->definitions[0] = Definition(dst);
4814    mubuf->offset = 0;
4815    mubuf->offen = (offset.type() == RegType::vgpr);
4816    mubuf->glc = return_previous;
4817    mubuf->dlc = false; /* Not needed for atomics */
4818    mubuf->disable_wqm = true;
4819    mubuf->barrier = barrier_buffer;
4820    ctx->program->needs_exact = true;
4821    ctx->block->instructions.emplace_back(std::move(mubuf));
4822 }
4823
4824 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4825
4826    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4827    Builder bld(ctx->program, ctx->block);
4828    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4829    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4830 }
4831
4832 Temp get_gfx6_global_rsrc(Builder& bld, Temp addr)
4833 {
4834    uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4835                         S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4836
4837    if (addr.type() == RegType::vgpr)
4838       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), Operand(0u), Operand(0u), Operand(-1u), Operand(rsrc_conf));
4839    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), addr, Operand(-1u), Operand(rsrc_conf));
4840 }
4841
4842 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4843 {
4844    Builder bld(ctx->program, ctx->block);
4845    unsigned num_components = instr->num_components;
4846    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4847
4848    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4849    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4850
4851    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4852    bool dlc = glc && ctx->options->chip_class >= GFX10;
4853    aco_opcode op;
4854    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4855       bool global = ctx->options->chip_class >= GFX9;
4856
4857       if (ctx->options->chip_class >= GFX7) {
4858          aco_opcode op;
4859          switch (num_bytes) {
4860          case 4:
4861             op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4862             break;
4863          case 8:
4864             op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4865             break;
4866          case 12:
4867             op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4868             break;
4869          case 16:
4870             op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4871             break;
4872          default:
4873             unreachable("load_global not implemented for this size.");
4874          }
4875
4876          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4877          flat->operands[0] = Operand(addr);
4878          flat->operands[1] = Operand(s1);
4879          flat->glc = glc;
4880          flat->dlc = dlc;
4881          flat->barrier = barrier_buffer;
4882
4883          if (dst.type() == RegType::sgpr) {
4884             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4885             flat->definitions[0] = Definition(vec);
4886             ctx->block->instructions.emplace_back(std::move(flat));
4887             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4888          } else {
4889             flat->definitions[0] = Definition(dst);
4890             ctx->block->instructions.emplace_back(std::move(flat));
4891          }
4892          emit_split_vector(ctx, dst, num_components);
4893       } else {
4894          assert(ctx->options->chip_class == GFX6);
4895
4896          /* GFX6 doesn't support loading vec3, expand to vec4. */
4897          num_bytes = num_bytes == 12 ? 16 : num_bytes;
4898
4899          aco_opcode op;
4900          switch (num_bytes) {
4901          case 4:
4902             op = aco_opcode::buffer_load_dword;
4903             break;
4904          case 8:
4905             op = aco_opcode::buffer_load_dwordx2;
4906             break;
4907          case 16:
4908             op = aco_opcode::buffer_load_dwordx4;
4909             break;
4910          default:
4911             unreachable("load_global not implemented for this size.");
4912          }
4913
4914          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
4915
4916          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
4917          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
4918          mubuf->operands[1] = Operand(rsrc);
4919          mubuf->operands[2] = Operand(0u);
4920          mubuf->glc = glc;
4921          mubuf->dlc = false;
4922          mubuf->offset = 0;
4923          mubuf->addr64 = addr.type() == RegType::vgpr;
4924          mubuf->disable_wqm = false;
4925          mubuf->barrier = barrier_buffer;
4926          aco_ptr<Instruction> instr = std::move(mubuf);
4927
4928          /* expand vector */
4929          if (dst.size() == 3) {
4930             Temp vec = bld.tmp(v4);
4931             instr->definitions[0] = Definition(vec);
4932             bld.insert(std::move(instr));
4933             emit_split_vector(ctx, vec, 4);
4934
4935             instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, 3, 1));
4936             instr->operands[0] = Operand(emit_extract_vector(ctx, vec, 0, v1));
4937             instr->operands[1] = Operand(emit_extract_vector(ctx, vec, 1, v1));
4938             instr->operands[2] = Operand(emit_extract_vector(ctx, vec, 2, v1));
4939          }
4940
4941          if (dst.type() == RegType::sgpr) {
4942             Temp vec = bld.tmp(RegType::vgpr, dst.size());
4943             instr->definitions[0] = Definition(vec);
4944             bld.insert(std::move(instr));
4945             expand_vector(ctx, vec, dst, num_components, (1 << num_components) - 1);
4946             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4947          } else {
4948             instr->definitions[0] = Definition(dst);
4949             bld.insert(std::move(instr));
4950             emit_split_vector(ctx, dst, num_components);
4951          }
4952       }
4953    } else {
4954       switch (num_bytes) {
4955          case 4:
4956             op = aco_opcode::s_load_dword;
4957             break;
4958          case 8:
4959             op = aco_opcode::s_load_dwordx2;
4960             break;
4961          case 12:
4962          case 16:
4963             op = aco_opcode::s_load_dwordx4;
4964             break;
4965          default:
4966             unreachable("load_global not implemented for this size.");
4967       }
4968       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4969       load->operands[0] = Operand(addr);
4970       load->operands[1] = Operand(0u);
4971       load->definitions[0] = Definition(dst);
4972       load->glc = glc;
4973       load->dlc = dlc;
4974       load->barrier = barrier_buffer;
4975       assert(ctx->options->chip_class >= GFX8 || !glc);
4976
4977       if (dst.size() == 3) {
4978          /* trim vector */
4979          Temp vec = bld.tmp(s4);
4980          load->definitions[0] = Definition(vec);
4981          ctx->block->instructions.emplace_back(std::move(load));
4982          emit_split_vector(ctx, vec, 4);
4983
4984          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4985                     emit_extract_vector(ctx, vec, 0, s1),
4986                     emit_extract_vector(ctx, vec, 1, s1),
4987                     emit_extract_vector(ctx, vec, 2, s1));
4988       } else {
4989          ctx->block->instructions.emplace_back(std::move(load));
4990       }
4991    }
4992 }
4993
4994 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4995 {
4996    Builder bld(ctx->program, ctx->block);
4997    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4998
4999    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5000    Temp addr = get_ssa_temp(ctx, instr->src[1].ssa);
5001
5002    if (ctx->options->chip_class >= GFX7)
5003       addr = as_vgpr(ctx, addr);
5004
5005    unsigned writemask = nir_intrinsic_write_mask(instr);
5006    while (writemask) {
5007       int start, count;
5008       u_bit_scan_consecutive_range(&writemask, &start, &count);
5009       if (count == 3 && ctx->options->chip_class == GFX6) {
5010          /* GFX6 doesn't support storing vec3, split it. */
5011          writemask |= 1u << (start + 2);
5012          count = 2;
5013       }
5014       unsigned num_bytes = count * elem_size_bytes;
5015
5016       Temp write_data = data;
5017       if (count != instr->num_components) {
5018          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5019          for (int i = 0; i < count; i++)
5020             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
5021          write_data = bld.tmp(RegType::vgpr, count);
5022          vec->definitions[0] = Definition(write_data);
5023          ctx->block->instructions.emplace_back(std::move(vec));
5024       }
5025
5026       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
5027       unsigned offset = start * elem_size_bytes;
5028
5029       if (ctx->options->chip_class >= GFX7) {
5030          if (offset > 0 && ctx->options->chip_class < GFX9) {
5031             Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
5032             Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
5033             Temp carry = bld.tmp(bld.lm);
5034             bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
5035
5036             bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
5037                      Operand(offset), addr0);
5038             bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
5039                      Operand(0u), addr1,
5040                      carry).def(1).setHint(vcc);
5041
5042             addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
5043
5044             offset = 0;
5045          }
5046
5047          bool global = ctx->options->chip_class >= GFX9;
5048          aco_opcode op;
5049          switch (num_bytes) {
5050          case 4:
5051             op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
5052             break;
5053          case 8:
5054             op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
5055             break;
5056          case 12:
5057             op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
5058             break;
5059          case 16:
5060             op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
5061             break;
5062          default:
5063             unreachable("store_global not implemented for this size.");
5064          }
5065
5066          aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
5067          flat->operands[0] = Operand(addr);
5068          flat->operands[1] = Operand(s1);
5069          flat->operands[2] = Operand(data);
5070          flat->glc = glc;
5071          flat->dlc = false;
5072          flat->offset = offset;
5073          flat->disable_wqm = true;
5074          flat->barrier = barrier_buffer;
5075          ctx->program->needs_exact = true;
5076          ctx->block->instructions.emplace_back(std::move(flat));
5077       } else {
5078          assert(ctx->options->chip_class == GFX6);
5079
5080          aco_opcode op;
5081          switch (num_bytes) {
5082          case 4:
5083             op = aco_opcode::buffer_store_dword;
5084             break;
5085          case 8:
5086             op = aco_opcode::buffer_store_dwordx2;
5087             break;
5088          case 16:
5089             op = aco_opcode::buffer_store_dwordx4;
5090             break;
5091          default:
5092             unreachable("store_global not implemented for this size.");
5093          }
5094
5095          Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5096
5097          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, 0)};
5098          mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5099          mubuf->operands[1] = Operand(rsrc);
5100          mubuf->operands[2] = Operand(0u);
5101          mubuf->operands[3] = Operand(write_data);
5102          mubuf->glc = glc;
5103          mubuf->dlc = false;
5104          mubuf->offset = offset;
5105          mubuf->addr64 = addr.type() == RegType::vgpr;
5106          mubuf->disable_wqm = true;
5107          mubuf->barrier = barrier_buffer;
5108          ctx->program->needs_exact = true;
5109          ctx->block->instructions.emplace_back(std::move(mubuf));
5110       }
5111    }
5112 }
5113
5114 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5115 {
5116    /* return the previous value if dest is ever used */
5117    bool return_previous = false;
5118    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5119       return_previous = true;
5120       break;
5121    }
5122    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5123       return_previous = true;
5124       break;
5125    }
5126
5127    Builder bld(ctx->program, ctx->block);
5128    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5129    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5130
5131    if (ctx->options->chip_class >= GFX7)
5132       addr = as_vgpr(ctx, addr);
5133
5134    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
5135       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
5136                         get_ssa_temp(ctx, instr->src[2].ssa), data);
5137
5138    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5139
5140    aco_opcode op32, op64;
5141
5142    if (ctx->options->chip_class >= GFX7) {
5143       bool global = ctx->options->chip_class >= GFX9;
5144       switch (instr->intrinsic) {
5145          case nir_intrinsic_global_atomic_add:
5146             op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
5147             op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
5148             break;
5149          case nir_intrinsic_global_atomic_imin:
5150             op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
5151             op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
5152             break;
5153          case nir_intrinsic_global_atomic_umin:
5154             op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
5155             op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
5156             break;
5157          case nir_intrinsic_global_atomic_imax:
5158             op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
5159             op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
5160             break;
5161          case nir_intrinsic_global_atomic_umax:
5162             op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
5163             op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
5164             break;
5165          case nir_intrinsic_global_atomic_and:
5166             op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
5167             op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
5168             break;
5169          case nir_intrinsic_global_atomic_or:
5170             op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
5171             op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
5172             break;
5173          case nir_intrinsic_global_atomic_xor:
5174             op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
5175             op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
5176             break;
5177          case nir_intrinsic_global_atomic_exchange:
5178             op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
5179             op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
5180             break;
5181          case nir_intrinsic_global_atomic_comp_swap:
5182             op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
5183             op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
5184             break;
5185          default:
5186             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5187       }
5188
5189       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5190       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
5191       flat->operands[0] = Operand(addr);
5192       flat->operands[1] = Operand(s1);
5193       flat->operands[2] = Operand(data);
5194       if (return_previous)
5195          flat->definitions[0] = Definition(dst);
5196       flat->glc = return_previous;
5197       flat->dlc = false; /* Not needed for atomics */
5198       flat->offset = 0;
5199       flat->disable_wqm = true;
5200       flat->barrier = barrier_buffer;
5201       ctx->program->needs_exact = true;
5202       ctx->block->instructions.emplace_back(std::move(flat));
5203    } else {
5204       assert(ctx->options->chip_class == GFX6);
5205
5206       switch (instr->intrinsic) {
5207          case nir_intrinsic_global_atomic_add:
5208             op32 = aco_opcode::buffer_atomic_add;
5209             op64 = aco_opcode::buffer_atomic_add_x2;
5210             break;
5211          case nir_intrinsic_global_atomic_imin:
5212             op32 = aco_opcode::buffer_atomic_smin;
5213             op64 = aco_opcode::buffer_atomic_smin_x2;
5214             break;
5215          case nir_intrinsic_global_atomic_umin:
5216             op32 = aco_opcode::buffer_atomic_umin;
5217             op64 = aco_opcode::buffer_atomic_umin_x2;
5218             break;
5219          case nir_intrinsic_global_atomic_imax:
5220             op32 = aco_opcode::buffer_atomic_smax;
5221             op64 = aco_opcode::buffer_atomic_smax_x2;
5222             break;
5223          case nir_intrinsic_global_atomic_umax:
5224             op32 = aco_opcode::buffer_atomic_umax;
5225             op64 = aco_opcode::buffer_atomic_umax_x2;
5226             break;
5227          case nir_intrinsic_global_atomic_and:
5228             op32 = aco_opcode::buffer_atomic_and;
5229             op64 = aco_opcode::buffer_atomic_and_x2;
5230             break;
5231          case nir_intrinsic_global_atomic_or:
5232             op32 = aco_opcode::buffer_atomic_or;
5233             op64 = aco_opcode::buffer_atomic_or_x2;
5234             break;
5235          case nir_intrinsic_global_atomic_xor:
5236             op32 = aco_opcode::buffer_atomic_xor;
5237             op64 = aco_opcode::buffer_atomic_xor_x2;
5238             break;
5239          case nir_intrinsic_global_atomic_exchange:
5240             op32 = aco_opcode::buffer_atomic_swap;
5241             op64 = aco_opcode::buffer_atomic_swap_x2;
5242             break;
5243          case nir_intrinsic_global_atomic_comp_swap:
5244             op32 = aco_opcode::buffer_atomic_cmpswap;
5245             op64 = aco_opcode::buffer_atomic_cmpswap_x2;
5246             break;
5247          default:
5248             unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
5249       }
5250
5251       Temp rsrc = get_gfx6_global_rsrc(bld, addr);
5252
5253       aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
5254
5255       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
5256       mubuf->operands[0] = addr.type() == RegType::vgpr ? Operand(addr) : Operand(v1);
5257       mubuf->operands[1] = Operand(rsrc);
5258       mubuf->operands[2] = Operand(0u);
5259       mubuf->operands[3] = Operand(data);
5260       if (return_previous)
5261          mubuf->definitions[0] = Definition(dst);
5262       mubuf->glc = return_previous;
5263       mubuf->dlc = false;
5264       mubuf->offset = 0;
5265       mubuf->addr64 = addr.type() == RegType::vgpr;
5266       mubuf->disable_wqm = true;
5267       mubuf->barrier = barrier_buffer;
5268       ctx->program->needs_exact = true;
5269       ctx->block->instructions.emplace_back(std::move(mubuf));
5270    }
5271 }
5272
5273 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
5274    Builder bld(ctx->program, ctx->block);
5275    switch(instr->intrinsic) {
5276       case nir_intrinsic_group_memory_barrier:
5277       case nir_intrinsic_memory_barrier:
5278          bld.barrier(aco_opcode::p_memory_barrier_common);
5279          break;
5280       case nir_intrinsic_memory_barrier_buffer:
5281          bld.barrier(aco_opcode::p_memory_barrier_buffer);
5282          break;
5283       case nir_intrinsic_memory_barrier_image:
5284          bld.barrier(aco_opcode::p_memory_barrier_image);
5285          break;
5286       case nir_intrinsic_memory_barrier_shared:
5287          bld.barrier(aco_opcode::p_memory_barrier_shared);
5288          break;
5289       default:
5290          unreachable("Unimplemented memory barrier intrinsic");
5291          break;
5292    }
5293 }
5294
5295 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5296 {
5297    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
5298    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5299    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
5300    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5301    Builder bld(ctx->program, ctx->block);
5302
5303    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
5304    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5305    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
5306 }
5307
5308 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
5309 {
5310    unsigned writemask = nir_intrinsic_write_mask(instr);
5311    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
5312    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5313    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5314    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
5315
5316    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
5317    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
5318 }
5319
5320 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
5321 {
5322    unsigned offset = nir_intrinsic_base(instr);
5323    Operand m = load_lds_size_m0(ctx);
5324    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5325    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5326
5327    unsigned num_operands = 3;
5328    aco_opcode op32, op64, op32_rtn, op64_rtn;
5329    switch(instr->intrinsic) {
5330       case nir_intrinsic_shared_atomic_add:
5331          op32 = aco_opcode::ds_add_u32;
5332          op64 = aco_opcode::ds_add_u64;
5333          op32_rtn = aco_opcode::ds_add_rtn_u32;
5334          op64_rtn = aco_opcode::ds_add_rtn_u64;
5335          break;
5336       case nir_intrinsic_shared_atomic_imin:
5337          op32 = aco_opcode::ds_min_i32;
5338          op64 = aco_opcode::ds_min_i64;
5339          op32_rtn = aco_opcode::ds_min_rtn_i32;
5340          op64_rtn = aco_opcode::ds_min_rtn_i64;
5341          break;
5342       case nir_intrinsic_shared_atomic_umin:
5343          op32 = aco_opcode::ds_min_u32;
5344          op64 = aco_opcode::ds_min_u64;
5345          op32_rtn = aco_opcode::ds_min_rtn_u32;
5346          op64_rtn = aco_opcode::ds_min_rtn_u64;
5347          break;
5348       case nir_intrinsic_shared_atomic_imax:
5349          op32 = aco_opcode::ds_max_i32;
5350          op64 = aco_opcode::ds_max_i64;
5351          op32_rtn = aco_opcode::ds_max_rtn_i32;
5352          op64_rtn = aco_opcode::ds_max_rtn_i64;
5353          break;
5354       case nir_intrinsic_shared_atomic_umax:
5355          op32 = aco_opcode::ds_max_u32;
5356          op64 = aco_opcode::ds_max_u64;
5357          op32_rtn = aco_opcode::ds_max_rtn_u32;
5358          op64_rtn = aco_opcode::ds_max_rtn_u64;
5359          break;
5360       case nir_intrinsic_shared_atomic_and:
5361          op32 = aco_opcode::ds_and_b32;
5362          op64 = aco_opcode::ds_and_b64;
5363          op32_rtn = aco_opcode::ds_and_rtn_b32;
5364          op64_rtn = aco_opcode::ds_and_rtn_b64;
5365          break;
5366       case nir_intrinsic_shared_atomic_or:
5367          op32 = aco_opcode::ds_or_b32;
5368          op64 = aco_opcode::ds_or_b64;
5369          op32_rtn = aco_opcode::ds_or_rtn_b32;
5370          op64_rtn = aco_opcode::ds_or_rtn_b64;
5371          break;
5372       case nir_intrinsic_shared_atomic_xor:
5373          op32 = aco_opcode::ds_xor_b32;
5374          op64 = aco_opcode::ds_xor_b64;
5375          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5376          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5377          break;
5378       case nir_intrinsic_shared_atomic_exchange:
5379          op32 = aco_opcode::ds_write_b32;
5380          op64 = aco_opcode::ds_write_b64;
5381          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5382          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5383          break;
5384       case nir_intrinsic_shared_atomic_comp_swap:
5385          op32 = aco_opcode::ds_cmpst_b32;
5386          op64 = aco_opcode::ds_cmpst_b64;
5387          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5388          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5389          num_operands = 4;
5390          break;
5391       default:
5392          unreachable("Unhandled shared atomic intrinsic");
5393    }
5394
5395    /* return the previous value if dest is ever used */
5396    bool return_previous = false;
5397    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5398       return_previous = true;
5399       break;
5400    }
5401    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5402       return_previous = true;
5403       break;
5404    }
5405
5406    aco_opcode op;
5407    if (data.size() == 1) {
5408       assert(instr->dest.ssa.bit_size == 32);
5409       op = return_previous ? op32_rtn : op32;
5410    } else {
5411       assert(instr->dest.ssa.bit_size == 64);
5412       op = return_previous ? op64_rtn : op64;
5413    }
5414
5415    if (offset > 65535) {
5416       Builder bld(ctx->program, ctx->block);
5417       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5418       offset = 0;
5419    }
5420
5421    aco_ptr<DS_instruction> ds;
5422    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5423    ds->operands[0] = Operand(address);
5424    ds->operands[1] = Operand(data);
5425    if (num_operands == 4)
5426       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5427    ds->operands[num_operands - 1] = m;
5428    ds->offset0 = offset;
5429    if (return_previous)
5430       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5431    ctx->block->instructions.emplace_back(std::move(ds));
5432 }
5433
5434 Temp get_scratch_resource(isel_context *ctx)
5435 {
5436    Builder bld(ctx->program, ctx->block);
5437    Temp scratch_addr = ctx->program->private_segment_buffer;
5438    if (ctx->stage != compute_cs)
5439       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5440
5441    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5442                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5443
5444    if (ctx->program->chip_class >= GFX10) {
5445       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5446                    S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) |
5447                    S_008F0C_RESOURCE_LEVEL(1);
5448    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5449       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5450                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5451    }
5452
5453    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5454    if (ctx->program->chip_class <= GFX8)
5455       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5456
5457    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5458 }
5459
5460 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5461    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5462    Builder bld(ctx->program, ctx->block);
5463    Temp rsrc = get_scratch_resource(ctx);
5464    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5465    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5466
5467    aco_opcode op;
5468    switch (dst.size()) {
5469       case 1:
5470          op = aco_opcode::buffer_load_dword;
5471          break;
5472       case 2:
5473          op = aco_opcode::buffer_load_dwordx2;
5474          break;
5475       case 3:
5476          op = aco_opcode::buffer_load_dwordx3;
5477          break;
5478       case 4:
5479          op = aco_opcode::buffer_load_dwordx4;
5480          break;
5481       case 6:
5482       case 8: {
5483          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5484          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5485                                 bld.def(v4), offset, rsrc,
5486                                 ctx->program->scratch_offset, 0, true);
5487          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5488                                                   aco_opcode::buffer_load_dwordx4,
5489                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5490                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5491          emit_split_vector(ctx, lower, 2);
5492          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5493          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5494          if (dst.size() == 8) {
5495             emit_split_vector(ctx, upper, 2);
5496             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5497             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5498          } else {
5499             elems[2] = upper;
5500          }
5501
5502          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5503                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5504          for (unsigned i = 0; i < dst.size() / 2; i++)
5505             vec->operands[i] = Operand(elems[i]);
5506          vec->definitions[0] = Definition(dst);
5507          bld.insert(std::move(vec));
5508          ctx->allocated_vec.emplace(dst.id(), elems);
5509          return;
5510       }
5511       default:
5512          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5513    }
5514
5515    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5516    emit_split_vector(ctx, dst, instr->num_components);
5517 }
5518
5519 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5520    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5521    Builder bld(ctx->program, ctx->block);
5522    Temp rsrc = get_scratch_resource(ctx);
5523    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5524    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5525
5526    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5527    unsigned writemask = nir_intrinsic_write_mask(instr);
5528
5529    while (writemask) {
5530       int start, count;
5531       u_bit_scan_consecutive_range(&writemask, &start, &count);
5532       int num_bytes = count * elem_size_bytes;
5533
5534       if (num_bytes > 16) {
5535          assert(elem_size_bytes == 8);
5536          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5537          count = 2;
5538          num_bytes = 16;
5539       }
5540
5541       // TODO: check alignment of sub-dword stores
5542       // TODO: split 3 bytes. there is no store instruction for that
5543
5544       Temp write_data;
5545       if (count != instr->num_components) {
5546          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5547          for (int i = 0; i < count; i++) {
5548             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5549             vec->operands[i] = Operand(elem);
5550          }
5551          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5552          vec->definitions[0] = Definition(write_data);
5553          ctx->block->instructions.emplace_back(std::move(vec));
5554       } else {
5555          write_data = data;
5556       }
5557
5558       aco_opcode op;
5559       switch (num_bytes) {
5560          case 4:
5561             op = aco_opcode::buffer_store_dword;
5562             break;
5563          case 8:
5564             op = aco_opcode::buffer_store_dwordx2;
5565             break;
5566          case 12:
5567             op = aco_opcode::buffer_store_dwordx3;
5568             break;
5569          case 16:
5570             op = aco_opcode::buffer_store_dwordx4;
5571             break;
5572          default:
5573             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5574       }
5575
5576       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5577    }
5578 }
5579
5580 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5581    uint8_t log2_ps_iter_samples;
5582    if (ctx->program->info->ps.force_persample) {
5583       log2_ps_iter_samples =
5584          util_logbase2(ctx->options->key.fs.num_samples);
5585    } else {
5586       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5587    }
5588
5589    /* The bit pattern matches that used by fixed function fragment
5590     * processing. */
5591    static const unsigned ps_iter_masks[] = {
5592       0xffff, /* not used */
5593       0x5555,
5594       0x1111,
5595       0x0101,
5596       0x0001,
5597    };
5598    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5599
5600    Builder bld(ctx->program, ctx->block);
5601
5602    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5603                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5604    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5605    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5606    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5607    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5608 }
5609
5610 void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *instr) {
5611    Builder bld(ctx->program, ctx->block);
5612
5613    unsigned stream = nir_intrinsic_stream_id(instr);
5614    Temp next_vertex = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5615    next_vertex = bld.v_mul_imm(bld.def(v1), next_vertex, 4u);
5616    nir_const_value *next_vertex_cv = nir_src_as_const_value(instr->src[0]);
5617
5618    /* get GSVS ring */
5619    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_GSVS_GS * 16u));
5620
5621    unsigned num_components =
5622       ctx->program->info->gs.num_stream_output_components[stream];
5623    assert(num_components);
5624
5625    unsigned stride = 4u * num_components * ctx->shader->info.gs.vertices_out;
5626    unsigned stream_offset = 0;
5627    for (unsigned i = 0; i < stream; i++) {
5628       unsigned prev_stride = 4u * ctx->program->info->gs.num_stream_output_components[i] * ctx->shader->info.gs.vertices_out;
5629       stream_offset += prev_stride * ctx->program->wave_size;
5630    }
5631
5632    /* Limit on the stride field for <= GFX7. */
5633    assert(stride < (1 << 14));
5634
5635    Temp gsvs_dwords[4];
5636    for (unsigned i = 0; i < 4; i++)
5637       gsvs_dwords[i] = bld.tmp(s1);
5638    bld.pseudo(aco_opcode::p_split_vector,
5639               Definition(gsvs_dwords[0]),
5640               Definition(gsvs_dwords[1]),
5641               Definition(gsvs_dwords[2]),
5642               Definition(gsvs_dwords[3]),
5643               gsvs_ring);
5644
5645    if (stream_offset) {
5646       Temp stream_offset_tmp = bld.copy(bld.def(s1), Operand(stream_offset));
5647
5648       Temp carry = bld.tmp(s1);
5649       gsvs_dwords[0] = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), gsvs_dwords[0], stream_offset_tmp);
5650       gsvs_dwords[1] = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(0u), bld.scc(carry));
5651    }
5652
5653    gsvs_dwords[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), gsvs_dwords[1], Operand(S_008F04_STRIDE(stride)));
5654    gsvs_dwords[2] = bld.copy(bld.def(s1), Operand((uint32_t)ctx->program->wave_size));
5655
5656    gsvs_ring = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
5657                           gsvs_dwords[0], gsvs_dwords[1], gsvs_dwords[2], gsvs_dwords[3]);
5658
5659    unsigned offset = 0;
5660    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; i++) {
5661       if (ctx->program->info->gs.output_streams[i] != stream)
5662          continue;
5663
5664       for (unsigned j = 0; j < 4; j++) {
5665          if (!(ctx->program->info->gs.output_usage_mask[i] & (1 << j)))
5666             continue;
5667
5668          if (ctx->outputs.mask[i] & (1 << j)) {
5669             Operand vaddr_offset = next_vertex_cv ? Operand(v1) : Operand(next_vertex);
5670             unsigned const_offset = (offset + (next_vertex_cv ? next_vertex_cv->u32 : 0u)) * 4u;
5671             if (const_offset >= 4096u) {
5672                if (vaddr_offset.isUndefined())
5673                   vaddr_offset = bld.copy(bld.def(v1), Operand(const_offset / 4096u * 4096u));
5674                else
5675                   vaddr_offset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), vaddr_offset);
5676                const_offset %= 4096u;
5677             }
5678
5679             aco_ptr<MTBUF_instruction> mtbuf{create_instruction<MTBUF_instruction>(aco_opcode::tbuffer_store_format_x, Format::MTBUF, 4, 0)};
5680             mtbuf->operands[0] = vaddr_offset;
5681             mtbuf->operands[1] = Operand(gsvs_ring);
5682             mtbuf->operands[2] = Operand(get_arg(ctx, ctx->args->gs2vs_offset));
5683             mtbuf->operands[3] = Operand(ctx->outputs.outputs[i][j]);
5684             mtbuf->offen = !vaddr_offset.isUndefined();
5685             mtbuf->dfmt = V_008F0C_BUF_DATA_FORMAT_32;
5686             mtbuf->nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
5687             mtbuf->offset = const_offset;
5688             mtbuf->glc = true;
5689             mtbuf->slc = true;
5690             mtbuf->barrier = barrier_gs_data;
5691             mtbuf->can_reorder = true;
5692             bld.insert(std::move(mtbuf));
5693          }
5694
5695          offset += ctx->shader->info.gs.vertices_out;
5696       }
5697
5698       /* outputs for the next vertex are undefined and keeping them around can
5699        * create invalid IR with control flow */
5700       ctx->outputs.mask[i] = 0;
5701    }
5702
5703    bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(false, true, stream));
5704 }
5705
5706 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5707 {
5708    Builder bld(ctx->program, ctx->block);
5709
5710    if (cluster_size == 1) {
5711       return src;
5712    } if (op == nir_op_iand && cluster_size == 4) {
5713       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5714       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5715       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5716                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5717    } else if (op == nir_op_ior && cluster_size == 4) {
5718       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5719       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5720                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5721    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5722       //subgroupAnd(val) -> (exec & ~val) == 0
5723       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5724       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
5725       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc), cond);
5726    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5727       //subgroupOr(val) -> (val & exec) != 0
5728       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5729       return bool_to_vector_condition(ctx, tmp);
5730    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5731       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5732       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5733       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5734       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5735       return bool_to_vector_condition(ctx, tmp);
5736    } else {
5737       //subgroupClustered{And,Or,Xor}(val, n) ->
5738       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5739       //cluster_offset = ~(n - 1) & lane_id
5740       //cluster_mask = ((1 << n) - 1)
5741       //subgroupClusteredAnd():
5742       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5743       //subgroupClusteredOr():
5744       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5745       //subgroupClusteredXor():
5746       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5747       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5748       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5749
5750       Temp tmp;
5751       if (op == nir_op_iand)
5752          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5753       else
5754          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5755
5756       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5757
5758       if (ctx->program->chip_class <= GFX7)
5759          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5760       else if (ctx->program->wave_size == 64)
5761          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5762       else
5763          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5764       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5765       if (cluster_mask != 0xffffffff)
5766          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5767
5768       Definition cmp_def = Definition();
5769       if (op == nir_op_iand) {
5770          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5771       } else if (op == nir_op_ior) {
5772          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5773       } else if (op == nir_op_ixor) {
5774          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5775                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5776          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5777       }
5778       cmp_def.setHint(vcc);
5779       return cmp_def.getTemp();
5780    }
5781 }
5782
5783 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5784 {
5785    Builder bld(ctx->program, ctx->block);
5786
5787    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5788    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5789    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5790    Temp tmp;
5791    if (op == nir_op_iand)
5792       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5793    else
5794       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5795
5796    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5797    Temp lo = lohi.def(0).getTemp();
5798    Temp hi = lohi.def(1).getTemp();
5799    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5800
5801    Definition cmp_def = Definition();
5802    if (op == nir_op_iand)
5803       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5804    else if (op == nir_op_ior)
5805       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5806    else if (op == nir_op_ixor)
5807       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5808                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5809    cmp_def.setHint(vcc);
5810    return cmp_def.getTemp();
5811 }
5812
5813 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5814 {
5815    Builder bld(ctx->program, ctx->block);
5816
5817    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5818    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5819    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5820    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5821    if (op == nir_op_iand)
5822       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5823    else if (op == nir_op_ior)
5824       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5825    else if (op == nir_op_ixor)
5826       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5827
5828    assert(false);
5829    return Temp();
5830 }
5831
5832 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5833 {
5834    Builder bld(ctx->program, ctx->block);
5835    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5836    if (src.regClass().type() == RegType::vgpr) {
5837       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5838    } else if (src.regClass() == s1) {
5839       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5840    } else if (src.regClass() == s2) {
5841       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5842    } else {
5843       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5844       nir_print_instr(&instr->instr, stderr);
5845       fprintf(stderr, "\n");
5846    }
5847 }
5848
5849 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5850 {
5851    Builder bld(ctx->program, ctx->block);
5852    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5853    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5854    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5855
5856    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5857    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5858    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5859    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5860
5861    /* Build DD X/Y */
5862    if (ctx->program->chip_class >= GFX8) {
5863       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5864       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5865       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5866       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5867       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5868       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5869    } else {
5870       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5871       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5872       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5873       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5874       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5875       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5876       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5877       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5878       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5879       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5880    }
5881
5882    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5883    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5884    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5885    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5886    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5887    Temp wqm1 = bld.tmp(v1);
5888    emit_wqm(ctx, tmp1, wqm1, true);
5889    Temp wqm2 = bld.tmp(v1);
5890    emit_wqm(ctx, tmp2, wqm2, true);
5891    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5892    return;
5893 }
5894
5895 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5896 {
5897    Builder bld(ctx->program, ctx->block);
5898    switch(instr->intrinsic) {
5899    case nir_intrinsic_load_barycentric_sample:
5900    case nir_intrinsic_load_barycentric_pixel:
5901    case nir_intrinsic_load_barycentric_centroid: {
5902       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5903       Temp bary = Temp(0, s2);
5904       switch (mode) {
5905       case INTERP_MODE_SMOOTH:
5906       case INTERP_MODE_NONE:
5907          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5908             bary = get_arg(ctx, ctx->args->ac.persp_center);
5909          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5910             bary = ctx->persp_centroid;
5911          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5912             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5913          break;
5914       case INTERP_MODE_NOPERSPECTIVE:
5915          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5916             bary = get_arg(ctx, ctx->args->ac.linear_center);
5917          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5918             bary = ctx->linear_centroid;
5919          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5920             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5921          break;
5922       default:
5923          break;
5924       }
5925       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5926       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5927       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5928       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5929                  Operand(p1), Operand(p2));
5930       emit_split_vector(ctx, dst, 2);
5931       break;
5932    }
5933    case nir_intrinsic_load_barycentric_at_sample: {
5934       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5935       switch (ctx->options->key.fs.num_samples) {
5936          case 2: sample_pos_offset += 1 << 3; break;
5937          case 4: sample_pos_offset += 3 << 3; break;
5938          case 8: sample_pos_offset += 7 << 3; break;
5939          default: break;
5940       }
5941       Temp sample_pos;
5942       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5943       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5944       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5945       if (addr.type() == RegType::sgpr) {
5946          Operand offset;
5947          if (const_addr) {
5948             sample_pos_offset += const_addr->u32 << 3;
5949             offset = Operand(sample_pos_offset);
5950          } else if (ctx->options->chip_class >= GFX9) {
5951             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5952          } else {
5953             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5954             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5955          }
5956
5957          Operand off = bld.copy(bld.def(s1), Operand(offset));
5958          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, off);
5959
5960       } else if (ctx->options->chip_class >= GFX9) {
5961          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5962          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5963       } else if (ctx->options->chip_class >= GFX7) {
5964          /* addr += private_segment_buffer + sample_pos_offset */
5965          Temp tmp0 = bld.tmp(s1);
5966          Temp tmp1 = bld.tmp(s1);
5967          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5968          Definition scc_tmp = bld.def(s1, scc);
5969          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5970          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5971          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5972          Temp pck0 = bld.tmp(v1);
5973          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5974          tmp1 = as_vgpr(ctx, tmp1);
5975          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5976          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5977
5978          /* sample_pos = flat_load_dwordx2 addr */
5979          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5980       } else {
5981          assert(ctx->options->chip_class == GFX6);
5982
5983          uint32_t rsrc_conf = S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5984                               S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5985          Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), private_segment_buffer, Operand(0u), Operand(rsrc_conf));
5986
5987          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5988          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), addr, Operand(0u));
5989
5990          sample_pos = bld.tmp(v2);
5991
5992          aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dwordx2, Format::MUBUF, 3, 1)};
5993          load->definitions[0] = Definition(sample_pos);
5994          load->operands[0] = Operand(addr);
5995          load->operands[1] = Operand(rsrc);
5996          load->operands[2] = Operand(0u);
5997          load->offset = sample_pos_offset;
5998          load->offen = 0;
5999          load->addr64 = true;
6000          load->glc = false;
6001          load->dlc = false;
6002          load->disable_wqm = false;
6003          load->barrier = barrier_none;
6004          load->can_reorder = true;
6005          ctx->block->instructions.emplace_back(std::move(load));
6006       }
6007
6008       /* sample_pos -= 0.5 */
6009       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
6010       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
6011       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
6012       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
6013       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
6014
6015       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6016       break;
6017    }
6018    case nir_intrinsic_load_barycentric_at_offset: {
6019       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
6020       RegClass rc = RegClass(offset.type(), 1);
6021       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
6022       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
6023       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
6024       break;
6025    }
6026    case nir_intrinsic_load_front_face: {
6027       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6028                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
6029       break;
6030    }
6031    case nir_intrinsic_load_view_index:
6032    case nir_intrinsic_load_layer_id: {
6033       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & (sw_vs | sw_gs))) {
6034          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6035          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
6036          break;
6037       }
6038
6039       unsigned idx = nir_intrinsic_base(instr);
6040       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6041                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
6042       break;
6043    }
6044    case nir_intrinsic_load_frag_coord: {
6045       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
6046       break;
6047    }
6048    case nir_intrinsic_load_sample_pos: {
6049       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
6050       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
6051       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6052                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
6053                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
6054       break;
6055    }
6056    case nir_intrinsic_load_interpolated_input:
6057       visit_load_interpolated_input(ctx, instr);
6058       break;
6059    case nir_intrinsic_store_output:
6060       visit_store_output(ctx, instr);
6061       break;
6062    case nir_intrinsic_load_input:
6063       visit_load_input(ctx, instr);
6064       break;
6065    case nir_intrinsic_load_per_vertex_input:
6066       visit_load_per_vertex_input(ctx, instr);
6067       break;
6068    case nir_intrinsic_load_ubo:
6069       visit_load_ubo(ctx, instr);
6070       break;
6071    case nir_intrinsic_load_push_constant:
6072       visit_load_push_constant(ctx, instr);
6073       break;
6074    case nir_intrinsic_load_constant:
6075       visit_load_constant(ctx, instr);
6076       break;
6077    case nir_intrinsic_vulkan_resource_index:
6078       visit_load_resource(ctx, instr);
6079       break;
6080    case nir_intrinsic_discard:
6081       visit_discard(ctx, instr);
6082       break;
6083    case nir_intrinsic_discard_if:
6084       visit_discard_if(ctx, instr);
6085       break;
6086    case nir_intrinsic_load_shared:
6087       visit_load_shared(ctx, instr);
6088       break;
6089    case nir_intrinsic_store_shared:
6090       visit_store_shared(ctx, instr);
6091       break;
6092    case nir_intrinsic_shared_atomic_add:
6093    case nir_intrinsic_shared_atomic_imin:
6094    case nir_intrinsic_shared_atomic_umin:
6095    case nir_intrinsic_shared_atomic_imax:
6096    case nir_intrinsic_shared_atomic_umax:
6097    case nir_intrinsic_shared_atomic_and:
6098    case nir_intrinsic_shared_atomic_or:
6099    case nir_intrinsic_shared_atomic_xor:
6100    case nir_intrinsic_shared_atomic_exchange:
6101    case nir_intrinsic_shared_atomic_comp_swap:
6102       visit_shared_atomic(ctx, instr);
6103       break;
6104    case nir_intrinsic_image_deref_load:
6105       visit_image_load(ctx, instr);
6106       break;
6107    case nir_intrinsic_image_deref_store:
6108       visit_image_store(ctx, instr);
6109       break;
6110    case nir_intrinsic_image_deref_atomic_add:
6111    case nir_intrinsic_image_deref_atomic_umin:
6112    case nir_intrinsic_image_deref_atomic_imin:
6113    case nir_intrinsic_image_deref_atomic_umax:
6114    case nir_intrinsic_image_deref_atomic_imax:
6115    case nir_intrinsic_image_deref_atomic_and:
6116    case nir_intrinsic_image_deref_atomic_or:
6117    case nir_intrinsic_image_deref_atomic_xor:
6118    case nir_intrinsic_image_deref_atomic_exchange:
6119    case nir_intrinsic_image_deref_atomic_comp_swap:
6120       visit_image_atomic(ctx, instr);
6121       break;
6122    case nir_intrinsic_image_deref_size:
6123       visit_image_size(ctx, instr);
6124       break;
6125    case nir_intrinsic_load_ssbo:
6126       visit_load_ssbo(ctx, instr);
6127       break;
6128    case nir_intrinsic_store_ssbo:
6129       visit_store_ssbo(ctx, instr);
6130       break;
6131    case nir_intrinsic_load_global:
6132       visit_load_global(ctx, instr);
6133       break;
6134    case nir_intrinsic_store_global:
6135       visit_store_global(ctx, instr);
6136       break;
6137    case nir_intrinsic_global_atomic_add:
6138    case nir_intrinsic_global_atomic_imin:
6139    case nir_intrinsic_global_atomic_umin:
6140    case nir_intrinsic_global_atomic_imax:
6141    case nir_intrinsic_global_atomic_umax:
6142    case nir_intrinsic_global_atomic_and:
6143    case nir_intrinsic_global_atomic_or:
6144    case nir_intrinsic_global_atomic_xor:
6145    case nir_intrinsic_global_atomic_exchange:
6146    case nir_intrinsic_global_atomic_comp_swap:
6147       visit_global_atomic(ctx, instr);
6148       break;
6149    case nir_intrinsic_ssbo_atomic_add:
6150    case nir_intrinsic_ssbo_atomic_imin:
6151    case nir_intrinsic_ssbo_atomic_umin:
6152    case nir_intrinsic_ssbo_atomic_imax:
6153    case nir_intrinsic_ssbo_atomic_umax:
6154    case nir_intrinsic_ssbo_atomic_and:
6155    case nir_intrinsic_ssbo_atomic_or:
6156    case nir_intrinsic_ssbo_atomic_xor:
6157    case nir_intrinsic_ssbo_atomic_exchange:
6158    case nir_intrinsic_ssbo_atomic_comp_swap:
6159       visit_atomic_ssbo(ctx, instr);
6160       break;
6161    case nir_intrinsic_load_scratch:
6162       visit_load_scratch(ctx, instr);
6163       break;
6164    case nir_intrinsic_store_scratch:
6165       visit_store_scratch(ctx, instr);
6166       break;
6167    case nir_intrinsic_get_buffer_size:
6168       visit_get_buffer_size(ctx, instr);
6169       break;
6170    case nir_intrinsic_control_barrier: {
6171       unsigned* bsize = ctx->program->info->cs.block_size;
6172       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
6173       if (workgroup_size > ctx->program->wave_size)
6174          bld.sopp(aco_opcode::s_barrier);
6175       break;
6176    }
6177    case nir_intrinsic_group_memory_barrier:
6178    case nir_intrinsic_memory_barrier:
6179    case nir_intrinsic_memory_barrier_buffer:
6180    case nir_intrinsic_memory_barrier_image:
6181    case nir_intrinsic_memory_barrier_shared:
6182       emit_memory_barrier(ctx, instr);
6183       break;
6184    case nir_intrinsic_memory_barrier_tcs_patch:
6185       break;
6186    case nir_intrinsic_load_num_work_groups: {
6187       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6188       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
6189       emit_split_vector(ctx, dst, 3);
6190       break;
6191    }
6192    case nir_intrinsic_load_local_invocation_id: {
6193       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6194       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
6195       emit_split_vector(ctx, dst, 3);
6196       break;
6197    }
6198    case nir_intrinsic_load_work_group_id: {
6199       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6200       struct ac_arg *args = ctx->args->ac.workgroup_ids;
6201       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
6202                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
6203                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
6204                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
6205       emit_split_vector(ctx, dst, 3);
6206       break;
6207    }
6208    case nir_intrinsic_load_local_invocation_index: {
6209       Temp id = emit_mbcnt(ctx, bld.def(v1));
6210
6211       /* The tg_size bits [6:11] contain the subgroup id,
6212        * we need this multiplied by the wave size, and then OR the thread id to it.
6213        */
6214       if (ctx->program->wave_size == 64) {
6215          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
6216          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
6217                                 get_arg(ctx, ctx->args->ac.tg_size));
6218          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
6219       } else {
6220          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
6221          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
6222                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6223          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
6224       }
6225       break;
6226    }
6227    case nir_intrinsic_load_subgroup_id: {
6228       if (ctx->stage == compute_cs) {
6229          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
6230                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
6231       } else {
6232          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
6233       }
6234       break;
6235    }
6236    case nir_intrinsic_load_subgroup_invocation: {
6237       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
6238       break;
6239    }
6240    case nir_intrinsic_load_num_subgroups: {
6241       if (ctx->stage == compute_cs)
6242          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
6243                   get_arg(ctx, ctx->args->ac.tg_size));
6244       else
6245          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
6246       break;
6247    }
6248    case nir_intrinsic_ballot: {
6249       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6250       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6251       Definition tmp = bld.def(dst.regClass());
6252       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
6253       if (instr->src[0].ssa->bit_size == 1) {
6254          assert(src.regClass() == bld.lm);
6255          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
6256       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
6257          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
6258       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
6259          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
6260       } else {
6261          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6262          nir_print_instr(&instr->instr, stderr);
6263          fprintf(stderr, "\n");
6264       }
6265       if (dst.size() != bld.lm.size()) {
6266          /* Wave32 with ballot size set to 64 */
6267          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
6268       }
6269       emit_wqm(ctx, tmp.getTemp(), dst);
6270       break;
6271    }
6272    case nir_intrinsic_shuffle:
6273    case nir_intrinsic_read_invocation: {
6274       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6275       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
6276          emit_uniform_subgroup(ctx, instr, src);
6277       } else {
6278          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
6279          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
6280             tid = bld.as_uniform(tid);
6281          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6282          if (src.regClass() == v1) {
6283             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
6284          } else if (src.regClass() == v2) {
6285             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6286             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6287             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
6288             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
6289             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6290             emit_split_vector(ctx, dst, 2);
6291          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
6292             assert(src.regClass() == bld.lm);
6293             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
6294             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6295          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
6296             assert(src.regClass() == bld.lm);
6297             Temp tmp;
6298             if (ctx->program->chip_class <= GFX7)
6299                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
6300             else if (ctx->program->wave_size == 64)
6301                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
6302             else
6303                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
6304             tmp = emit_extract_vector(ctx, tmp, 0, v1);
6305             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
6306             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
6307          } else {
6308             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6309             nir_print_instr(&instr->instr, stderr);
6310             fprintf(stderr, "\n");
6311          }
6312       }
6313       break;
6314    }
6315    case nir_intrinsic_load_sample_id: {
6316       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6317                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
6318       break;
6319    }
6320    case nir_intrinsic_load_sample_mask_in: {
6321       visit_load_sample_mask_in(ctx, instr);
6322       break;
6323    }
6324    case nir_intrinsic_read_first_invocation: {
6325       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6326       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6327       if (src.regClass() == v1) {
6328          emit_wqm(ctx,
6329                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
6330                   dst);
6331       } else if (src.regClass() == v2) {
6332          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6333          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6334          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
6335          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
6336          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6337          emit_split_vector(ctx, dst, 2);
6338       } else if (instr->dest.ssa.bit_size == 1) {
6339          assert(src.regClass() == bld.lm);
6340          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
6341                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
6342          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6343       } else if (src.regClass() == s1) {
6344          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
6345       } else if (src.regClass() == s2) {
6346          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
6347       } else {
6348          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6349          nir_print_instr(&instr->instr, stderr);
6350          fprintf(stderr, "\n");
6351       }
6352       break;
6353    }
6354    case nir_intrinsic_vote_all: {
6355       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6356       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6357       assert(src.regClass() == bld.lm);
6358       assert(dst.regClass() == bld.lm);
6359
6360       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
6361       Temp cond = bool_to_vector_condition(ctx, emit_wqm(ctx, tmp));
6362       bld.sop1(Builder::s_not, Definition(dst), bld.def(s1, scc), cond);
6363       break;
6364    }
6365    case nir_intrinsic_vote_any: {
6366       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6367       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6368       assert(src.regClass() == bld.lm);
6369       assert(dst.regClass() == bld.lm);
6370
6371       Temp tmp = bool_to_scalar_condition(ctx, src);
6372       bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
6373       break;
6374    }
6375    case nir_intrinsic_reduce:
6376    case nir_intrinsic_inclusive_scan:
6377    case nir_intrinsic_exclusive_scan: {
6378       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6379       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6380       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
6381       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
6382          nir_intrinsic_cluster_size(instr) : 0;
6383       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
6384
6385       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
6386          emit_uniform_subgroup(ctx, instr, src);
6387       } else if (instr->dest.ssa.bit_size == 1) {
6388          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
6389             op = nir_op_iand;
6390          else if (op == nir_op_iadd)
6391             op = nir_op_ixor;
6392          else if (op == nir_op_umax || op == nir_op_imax)
6393             op = nir_op_ior;
6394          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
6395
6396          switch (instr->intrinsic) {
6397          case nir_intrinsic_reduce:
6398             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
6399             break;
6400          case nir_intrinsic_exclusive_scan:
6401             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
6402             break;
6403          case nir_intrinsic_inclusive_scan:
6404             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
6405             break;
6406          default:
6407             assert(false);
6408          }
6409       } else if (cluster_size == 1) {
6410          bld.copy(Definition(dst), src);
6411       } else {
6412          src = as_vgpr(ctx, src);
6413
6414          ReduceOp reduce_op;
6415          switch (op) {
6416          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
6417             CASE(iadd)
6418             CASE(imul)
6419             CASE(fadd)
6420             CASE(fmul)
6421             CASE(imin)
6422             CASE(umin)
6423             CASE(fmin)
6424             CASE(imax)
6425             CASE(umax)
6426             CASE(fmax)
6427             CASE(iand)
6428             CASE(ior)
6429             CASE(ixor)
6430             default:
6431                unreachable("unknown reduction op");
6432          #undef CASE
6433          }
6434
6435          aco_opcode aco_op;
6436          switch (instr->intrinsic) {
6437             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
6438             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
6439             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
6440             default:
6441                unreachable("unknown reduce intrinsic");
6442          }
6443
6444          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
6445          reduce->operands[0] = Operand(src);
6446          // filled in by aco_reduce_assign.cpp, used internally as part of the
6447          // reduce sequence
6448          assert(dst.size() == 1 || dst.size() == 2);
6449          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
6450          reduce->operands[2] = Operand(v1.as_linear());
6451
6452          Temp tmp_dst = bld.tmp(dst.regClass());
6453          reduce->definitions[0] = Definition(tmp_dst);
6454          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
6455          reduce->definitions[2] = Definition();
6456          reduce->definitions[3] = Definition(scc, s1);
6457          reduce->definitions[4] = Definition();
6458          reduce->reduce_op = reduce_op;
6459          reduce->cluster_size = cluster_size;
6460          ctx->block->instructions.emplace_back(std::move(reduce));
6461
6462          emit_wqm(ctx, tmp_dst, dst);
6463       }
6464       break;
6465    }
6466    case nir_intrinsic_quad_broadcast: {
6467       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6468       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6469          emit_uniform_subgroup(ctx, instr, src);
6470       } else {
6471          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6472          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
6473          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
6474
6475          if (instr->dest.ssa.bit_size == 1) {
6476             assert(src.regClass() == bld.lm);
6477             assert(dst.regClass() == bld.lm);
6478             uint32_t half_mask = 0x11111111u << lane;
6479             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
6480             Temp tmp = bld.tmp(bld.lm);
6481             bld.sop1(Builder::s_wqm, Definition(tmp),
6482                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
6483                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6484             emit_wqm(ctx, tmp, dst);
6485          } else if (instr->dest.ssa.bit_size == 32) {
6486             if (ctx->program->chip_class >= GFX8)
6487                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6488             else
6489                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6490          } else if (instr->dest.ssa.bit_size == 64) {
6491             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6492             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6493             if (ctx->program->chip_class >= GFX8) {
6494                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6495                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6496             } else {
6497                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6498                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6499             }
6500             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6501             emit_split_vector(ctx, dst, 2);
6502          } else {
6503             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6504             nir_print_instr(&instr->instr, stderr);
6505             fprintf(stderr, "\n");
6506          }
6507       }
6508       break;
6509    }
6510    case nir_intrinsic_quad_swap_horizontal:
6511    case nir_intrinsic_quad_swap_vertical:
6512    case nir_intrinsic_quad_swap_diagonal:
6513    case nir_intrinsic_quad_swizzle_amd: {
6514       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6515       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6516          emit_uniform_subgroup(ctx, instr, src);
6517          break;
6518       }
6519       uint16_t dpp_ctrl = 0;
6520       switch (instr->intrinsic) {
6521       case nir_intrinsic_quad_swap_horizontal:
6522          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6523          break;
6524       case nir_intrinsic_quad_swap_vertical:
6525          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6526          break;
6527       case nir_intrinsic_quad_swap_diagonal:
6528          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6529          break;
6530       case nir_intrinsic_quad_swizzle_amd:
6531          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6532          break;
6533       default:
6534          break;
6535       }
6536       if (ctx->program->chip_class < GFX8)
6537          dpp_ctrl |= (1 << 15);
6538
6539       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6540       if (instr->dest.ssa.bit_size == 1) {
6541          assert(src.regClass() == bld.lm);
6542          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6543          if (ctx->program->chip_class >= GFX8)
6544             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6545          else
6546             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6547          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6548          emit_wqm(ctx, tmp, dst);
6549       } else if (instr->dest.ssa.bit_size == 32) {
6550          Temp tmp;
6551          if (ctx->program->chip_class >= GFX8)
6552             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6553          else
6554             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6555          emit_wqm(ctx, tmp, dst);
6556       } else if (instr->dest.ssa.bit_size == 64) {
6557          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6558          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6559          if (ctx->program->chip_class >= GFX8) {
6560             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6561             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6562          } else {
6563             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6564             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6565          }
6566          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6567          emit_split_vector(ctx, dst, 2);
6568       } else {
6569          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6570          nir_print_instr(&instr->instr, stderr);
6571          fprintf(stderr, "\n");
6572       }
6573       break;
6574    }
6575    case nir_intrinsic_masked_swizzle_amd: {
6576       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6577       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6578          emit_uniform_subgroup(ctx, instr, src);
6579          break;
6580       }
6581       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6582       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6583       if (dst.regClass() == v1) {
6584          emit_wqm(ctx,
6585                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6586                   dst);
6587       } else if (dst.regClass() == v2) {
6588          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6589          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6590          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6591          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6592          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6593          emit_split_vector(ctx, dst, 2);
6594       } else {
6595          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6596          nir_print_instr(&instr->instr, stderr);
6597          fprintf(stderr, "\n");
6598       }
6599       break;
6600    }
6601    case nir_intrinsic_write_invocation_amd: {
6602       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6603       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6604       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6605       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6606       if (dst.regClass() == v1) {
6607          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6608          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6609       } else if (dst.regClass() == v2) {
6610          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6611          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6612          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6613          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6614          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6615          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6616          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6617          emit_split_vector(ctx, dst, 2);
6618       } else {
6619          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6620          nir_print_instr(&instr->instr, stderr);
6621          fprintf(stderr, "\n");
6622       }
6623       break;
6624    }
6625    case nir_intrinsic_mbcnt_amd: {
6626       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6627       RegClass rc = RegClass(src.type(), 1);
6628       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6629       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6630       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6631       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6632       emit_wqm(ctx, wqm_tmp, dst);
6633       break;
6634    }
6635    case nir_intrinsic_load_helper_invocation: {
6636       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6637       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6638       ctx->block->kind |= block_kind_needs_lowering;
6639       ctx->program->needs_exact = true;
6640       break;
6641    }
6642    case nir_intrinsic_is_helper_invocation: {
6643       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6644       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6645       ctx->block->kind |= block_kind_needs_lowering;
6646       ctx->program->needs_exact = true;
6647       break;
6648    }
6649    case nir_intrinsic_demote:
6650       bld.pseudo(aco_opcode::p_demote_to_helper, Operand(-1u));
6651
6652       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6653          ctx->cf_info.exec_potentially_empty = true;
6654       ctx->block->kind |= block_kind_uses_demote;
6655       ctx->program->needs_exact = true;
6656       break;
6657    case nir_intrinsic_demote_if: {
6658       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6659       assert(src.regClass() == bld.lm);
6660       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6661       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6662
6663       if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
6664          ctx->cf_info.exec_potentially_empty = true;
6665       ctx->block->kind |= block_kind_uses_demote;
6666       ctx->program->needs_exact = true;
6667       break;
6668    }
6669    case nir_intrinsic_first_invocation: {
6670       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6671                get_ssa_temp(ctx, &instr->dest.ssa));
6672       break;
6673    }
6674    case nir_intrinsic_shader_clock:
6675       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6676       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6677       break;
6678    case nir_intrinsic_load_vertex_id_zero_base: {
6679       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6680       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6681       break;
6682    }
6683    case nir_intrinsic_load_first_vertex: {
6684       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6685       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6686       break;
6687    }
6688    case nir_intrinsic_load_base_instance: {
6689       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6690       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6691       break;
6692    }
6693    case nir_intrinsic_load_instance_id: {
6694       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6695       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6696       break;
6697    }
6698    case nir_intrinsic_load_draw_id: {
6699       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6700       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6701       break;
6702    }
6703    case nir_intrinsic_load_invocation_id: {
6704       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6705       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6706       if (ctx->options->chip_class >= GFX10)
6707          bld.vop2_e64(aco_opcode::v_and_b32, Definition(dst), Operand(127u), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6708       else
6709          bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_invocation_id));
6710       break;
6711    }
6712    case nir_intrinsic_load_primitive_id: {
6713       assert(ctx->shader->info.stage == MESA_SHADER_GEOMETRY);
6714       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6715       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.gs_prim_id));
6716       break;
6717    }
6718    case nir_intrinsic_emit_vertex_with_counter: {
6719       visit_emit_vertex_with_counter(ctx, instr);
6720       break;
6721    }
6722    case nir_intrinsic_end_primitive_with_counter: {
6723       unsigned stream = nir_intrinsic_stream_id(instr);
6724       bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx->gs_wave_id), -1, sendmsg_gs(true, false, stream));
6725       break;
6726    }
6727    case nir_intrinsic_set_vertex_count: {
6728       /* unused, the HW keeps track of this for us */
6729       break;
6730    }
6731    default:
6732       fprintf(stderr, "Unimplemented intrinsic instr: ");
6733       nir_print_instr(&instr->instr, stderr);
6734       fprintf(stderr, "\n");
6735       abort();
6736
6737       break;
6738    }
6739 }
6740
6741
6742 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6743                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6744                     enum glsl_base_type *stype)
6745 {
6746    nir_deref_instr *texture_deref_instr = NULL;
6747    nir_deref_instr *sampler_deref_instr = NULL;
6748    int plane = -1;
6749
6750    for (unsigned i = 0; i < instr->num_srcs; i++) {
6751       switch (instr->src[i].src_type) {
6752       case nir_tex_src_texture_deref:
6753          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6754          break;
6755       case nir_tex_src_sampler_deref:
6756          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6757          break;
6758       case nir_tex_src_plane:
6759          plane = nir_src_as_int(instr->src[i].src);
6760          break;
6761       default:
6762          break;
6763       }
6764    }
6765
6766    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6767
6768    if (!sampler_deref_instr)
6769       sampler_deref_instr = texture_deref_instr;
6770
6771    if (plane >= 0) {
6772       assert(instr->op != nir_texop_txf_ms &&
6773              instr->op != nir_texop_samples_identical);
6774       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6775       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6776    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6777       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6778    } else if (instr->op == nir_texop_fragment_mask_fetch) {
6779       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6780    } else {
6781       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6782    }
6783    if (samp_ptr) {
6784       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6785
6786       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6787          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6788          Builder bld(ctx->program, ctx->block);
6789
6790          /* to avoid unnecessary moves, we split and recombine sampler and image */
6791          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6792                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6793          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6794          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6795                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6796                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6797          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6798                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6799
6800          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6801          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6802                                img[0], img[1], img[2], img[3],
6803                                img[4], img[5], img[6], img[7]);
6804          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6805                                 samp[0], samp[1], samp[2], samp[3]);
6806       }
6807    }
6808    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6809                      instr->op == nir_texop_samples_identical))
6810       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6811 }
6812
6813 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6814                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6815 {
6816    Builder bld(ctx->program, ctx->block);
6817
6818    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6819    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6820    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6821
6822    Operand neg_one(0xbf800000u);
6823    Operand one(0x3f800000u);
6824    Operand two(0x40000000u);
6825    Operand four(0x40800000u);
6826
6827    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6828    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6829    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6830
6831    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6832    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(bld.lm), two, id);
6833    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6834    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6835
6836    // select sc
6837    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6838    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6839                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6840                        one, is_ma_y);
6841    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6842
6843    // select tc
6844    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6845    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6846    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6847
6848    // select ma
6849    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6850                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6851                   deriv_z, is_ma_z);
6852    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6853    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6854 }
6855
6856 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6857 {
6858    Builder bld(ctx->program, ctx->block);
6859    Temp coord_args[4], ma, tc, sc, id;
6860    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6861       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6862
6863    if (is_array) {
6864       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6865
6866       // see comment in ac_prepare_cube_coords()
6867       if (ctx->options->chip_class <= GFX8)
6868          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6869    }
6870
6871    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6872
6873    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6874    vop3a->operands[0] = Operand(ma);
6875    vop3a->abs[0] = true;
6876    Temp invma = bld.tmp(v1);
6877    vop3a->definitions[0] = Definition(invma);
6878    ctx->block->instructions.emplace_back(std::move(vop3a));
6879
6880    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6881    if (!is_deriv)
6882       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6883
6884    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6885    if (!is_deriv)
6886       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6887
6888    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6889
6890    if (is_deriv) {
6891       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6892       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6893
6894       for (unsigned i = 0; i < 2; i++) {
6895          // see comment in ac_prepare_cube_coords()
6896          Temp deriv_ma;
6897          Temp deriv_sc, deriv_tc;
6898          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6899                            &deriv_ma, &deriv_sc, &deriv_tc);
6900
6901          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6902
6903          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6904                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6905                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6906          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6907                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6908                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6909          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6910       }
6911
6912       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6913       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6914    }
6915
6916    if (is_array)
6917       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6918    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6919
6920 }
6921
6922 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6923 {
6924    Temp coord_vec[3];
6925    for (unsigned i = 0; i < coords.size(); i++)
6926       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6927
6928    Builder bld(ctx->program, ctx->block);
6929    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6930
6931    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6932    for (unsigned i = 0; i < coords.size(); i++)
6933       vec->operands[i] = Operand(coord_vec[i]);
6934    Temp res = bld.tmp(RegType::vgpr, coords.size());
6935    vec->definitions[0] = Definition(res);
6936    ctx->block->instructions.emplace_back(std::move(vec));
6937    return res;
6938 }
6939
6940 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6941 {
6942    if (vec->parent_instr->type != nir_instr_type_alu)
6943       return;
6944    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6945    if (vec_instr->op != nir_op_vec(vec->num_components))
6946       return;
6947
6948    for (unsigned i = 0; i < vec->num_components; i++) {
6949       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6950               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6951    }
6952 }
6953
6954 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6955 {
6956    Builder bld(ctx->program, ctx->block);
6957    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6958         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6959    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6960         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6961    nir_const_value *sample_index_cv = NULL;
6962    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6963    enum glsl_base_type stype;
6964    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6965
6966    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6967                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6968    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6969                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6970
6971    for (unsigned i = 0; i < instr->num_srcs; i++) {
6972       switch (instr->src[i].src_type) {
6973       case nir_tex_src_coord:
6974          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6975          break;
6976       case nir_tex_src_bias:
6977          if (instr->op == nir_texop_txb) {
6978             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6979             has_bias = true;
6980          }
6981          break;
6982       case nir_tex_src_lod: {
6983          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6984
6985          if (val && val->f32 <= 0.0) {
6986             level_zero = true;
6987          } else {
6988             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6989             has_lod = true;
6990          }
6991          break;
6992       }
6993       case nir_tex_src_comparator:
6994          if (instr->is_shadow) {
6995             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6996             has_compare = true;
6997          }
6998          break;
6999       case nir_tex_src_offset:
7000          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
7001          get_const_vec(instr->src[i].src.ssa, const_offset);
7002          has_offset = true;
7003          break;
7004       case nir_tex_src_ddx:
7005          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
7006          has_ddx = true;
7007          break;
7008       case nir_tex_src_ddy:
7009          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
7010          has_ddy = true;
7011          break;
7012       case nir_tex_src_ms_index:
7013          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
7014          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
7015          has_sample_index = true;
7016          break;
7017       case nir_tex_src_texture_offset:
7018       case nir_tex_src_sampler_offset:
7019       default:
7020          break;
7021       }
7022    }
7023 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
7024    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
7025       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
7026
7027    if (instr->op == nir_texop_texture_samples) {
7028       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
7029
7030       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
7031       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
7032       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
7033       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
7034
7035       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
7036                samples, Operand(1u), bld.scc(is_msaa));
7037       return;
7038    }
7039
7040    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
7041       aco_ptr<Instruction> tmp_instr;
7042       Temp acc, pack = Temp();
7043
7044       uint32_t pack_const = 0;
7045       for (unsigned i = 0; i < offset.size(); i++) {
7046          if (!const_offset[i])
7047             continue;
7048          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
7049       }
7050
7051       if (offset.type() == RegType::sgpr) {
7052          for (unsigned i = 0; i < offset.size(); i++) {
7053             if (const_offset[i])
7054                continue;
7055
7056             acc = emit_extract_vector(ctx, offset, i, s1);
7057             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
7058
7059             if (i) {
7060                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
7061             }
7062
7063             if (pack == Temp()) {
7064                pack = acc;
7065             } else {
7066                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
7067             }
7068          }
7069
7070          if (pack_const && pack != Temp())
7071             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
7072       } else {
7073          for (unsigned i = 0; i < offset.size(); i++) {
7074             if (const_offset[i])
7075                continue;
7076
7077             acc = emit_extract_vector(ctx, offset, i, v1);
7078             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
7079
7080             if (i) {
7081                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
7082             }
7083
7084             if (pack == Temp()) {
7085                pack = acc;
7086             } else {
7087                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
7088             }
7089          }
7090
7091          if (pack_const && pack != Temp())
7092             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
7093       }
7094       if (pack_const && pack == Temp())
7095          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
7096       else if (pack == Temp())
7097          has_offset = false;
7098       else
7099          offset = pack;
7100    }
7101
7102    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
7103       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
7104
7105    /* pack derivatives */
7106    if (has_ddx || has_ddy) {
7107       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
7108          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
7109                              ddx, Operand(0u), ddy, Operand(0u));
7110       } else {
7111          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
7112       }
7113       has_derivs = true;
7114    }
7115
7116    if (instr->coord_components > 1 &&
7117        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7118        instr->is_array &&
7119        instr->op != nir_texop_txf)
7120       coords = apply_round_slice(ctx, coords, 1);
7121
7122    if (instr->coord_components > 2 &&
7123       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
7124        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7125        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
7126        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7127        instr->is_array &&
7128        instr->op != nir_texop_txf &&
7129        instr->op != nir_texop_txf_ms &&
7130        instr->op != nir_texop_fragment_fetch &&
7131        instr->op != nir_texop_fragment_mask_fetch)
7132       coords = apply_round_slice(ctx, coords, 2);
7133
7134    if (ctx->options->chip_class == GFX9 &&
7135        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7136        instr->op != nir_texop_lod && instr->coord_components) {
7137       assert(coords.size() > 0 && coords.size() < 3);
7138
7139       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
7140       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
7141       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
7142       if (coords.size() > 1)
7143          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
7144       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
7145       vec->definitions[0] = Definition(coords);
7146       ctx->block->instructions.emplace_back(std::move(vec));
7147    }
7148
7149    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
7150
7151    if (instr->op == nir_texop_samples_identical)
7152       resource = fmask_ptr;
7153
7154    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
7155              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
7156             instr->op != nir_texop_txs &&
7157             instr->op != nir_texop_fragment_fetch &&
7158             instr->op != nir_texop_fragment_mask_fetch) {
7159       assert(has_sample_index);
7160       Operand op(sample_index);
7161       if (sample_index_cv)
7162          op = Operand(sample_index_cv->u32);
7163       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
7164    }
7165
7166    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
7167       Temp split_coords[coords.size()];
7168       emit_split_vector(ctx, coords, coords.size());
7169       for (unsigned i = 0; i < coords.size(); i++)
7170          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
7171
7172       unsigned i = 0;
7173       for (; i < std::min(offset.size(), instr->coord_components); i++) {
7174          Temp off = emit_extract_vector(ctx, offset, i, v1);
7175          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
7176       }
7177
7178       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
7179       for (unsigned i = 0; i < coords.size(); i++)
7180          vec->operands[i] = Operand(split_coords[i]);
7181       coords = bld.tmp(coords.regClass());
7182       vec->definitions[0] = Definition(coords);
7183       ctx->block->instructions.emplace_back(std::move(vec));
7184
7185       has_offset = false;
7186    }
7187
7188    /* Build tex instruction */
7189    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
7190    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
7191                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
7192                   : 0;
7193    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7194    Temp tmp_dst = dst;
7195
7196    /* gather4 selects the component by dmask and always returns vec4 */
7197    if (instr->op == nir_texop_tg4) {
7198       assert(instr->dest.ssa.num_components == 4);
7199       if (instr->is_shadow)
7200          dmask = 1;
7201       else
7202          dmask = 1 << instr->component;
7203       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
7204          tmp_dst = bld.tmp(v4);
7205    } else if (instr->op == nir_texop_samples_identical) {
7206       tmp_dst = bld.tmp(v1);
7207    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
7208       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
7209    }
7210
7211    aco_ptr<MIMG_instruction> tex;
7212    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
7213       if (!has_lod)
7214          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7215
7216       bool div_by_6 = instr->op == nir_texop_txs &&
7217                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
7218                       instr->is_array &&
7219                       (dmask & (1 << 2));
7220       if (tmp_dst.id() == dst.id() && div_by_6)
7221          tmp_dst = bld.tmp(tmp_dst.regClass());
7222
7223       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7224       tex->operands[0] = Operand(as_vgpr(ctx,lod));
7225       tex->operands[1] = Operand(resource);
7226       if (ctx->options->chip_class == GFX9 &&
7227           instr->op == nir_texop_txs &&
7228           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
7229           instr->is_array) {
7230          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
7231       } else if (instr->op == nir_texop_query_levels) {
7232          tex->dmask = 1 << 3;
7233       } else {
7234          tex->dmask = dmask;
7235       }
7236       tex->da = da;
7237       tex->definitions[0] = Definition(tmp_dst);
7238       tex->dim = dim;
7239       tex->can_reorder = true;
7240       ctx->block->instructions.emplace_back(std::move(tex));
7241
7242       if (div_by_6) {
7243          /* divide 3rd value by 6 by multiplying with magic number */
7244          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7245          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
7246          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
7247          assert(instr->dest.ssa.num_components == 3);
7248          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
7249          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7250                               emit_extract_vector(ctx, tmp_dst, 0, v1),
7251                               emit_extract_vector(ctx, tmp_dst, 1, v1),
7252                               by_6);
7253
7254       }
7255
7256       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7257       return;
7258    }
7259
7260    Temp tg4_compare_cube_wa64 = Temp();
7261
7262    if (tg4_integer_workarounds) {
7263       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
7264       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
7265       tex->operands[1] = Operand(resource);
7266       tex->dim = dim;
7267       tex->dmask = 0x3;
7268       tex->da = da;
7269       Temp size = bld.tmp(v2);
7270       tex->definitions[0] = Definition(size);
7271       tex->can_reorder = true;
7272       ctx->block->instructions.emplace_back(std::move(tex));
7273       emit_split_vector(ctx, size, size.size());
7274
7275       Temp half_texel[2];
7276       for (unsigned i = 0; i < 2; i++) {
7277          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
7278          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
7279          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
7280          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
7281       }
7282
7283       Temp orig_coords[2] = {
7284          emit_extract_vector(ctx, coords, 0, v1),
7285          emit_extract_vector(ctx, coords, 1, v1)};
7286       Temp new_coords[2] = {
7287          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
7288          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
7289       };
7290
7291       if (tg4_integer_cube_workaround) {
7292          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
7293          Temp desc[resource.size()];
7294          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
7295                                                                            Format::PSEUDO, 1, resource.size())};
7296          split->operands[0] = Operand(resource);
7297          for (unsigned i = 0; i < resource.size(); i++) {
7298             desc[i] = bld.tmp(s1);
7299             split->definitions[i] = Definition(desc[i]);
7300          }
7301          ctx->block->instructions.emplace_back(std::move(split));
7302
7303          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
7304          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
7305                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
7306
7307          Temp nfmt;
7308          if (stype == GLSL_TYPE_UINT) {
7309             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7310                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
7311                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
7312                             bld.scc(compare_cube_wa));
7313          } else {
7314             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
7315                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
7316                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
7317                             bld.scc(compare_cube_wa));
7318          }
7319          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
7320          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
7321
7322          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
7323
7324          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
7325                             Operand((uint32_t)C_008F14_NUM_FORMAT));
7326          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
7327
7328          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
7329                                                                          Format::PSEUDO, resource.size(), 1)};
7330          for (unsigned i = 0; i < resource.size(); i++)
7331             vec->operands[i] = Operand(desc[i]);
7332          resource = bld.tmp(resource.regClass());
7333          vec->definitions[0] = Definition(resource);
7334          ctx->block->instructions.emplace_back(std::move(vec));
7335
7336          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7337                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
7338          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7339                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
7340       }
7341
7342       if (coords.size() == 3) {
7343          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
7344                              new_coords[0], new_coords[1],
7345                              emit_extract_vector(ctx, coords, 2, v1));
7346       } else {
7347          assert(coords.size() == 2);
7348          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
7349                              new_coords[0], new_coords[1]);
7350       }
7351    }
7352
7353    std::vector<Operand> args;
7354    if (has_offset)
7355       args.emplace_back(Operand(offset));
7356    if (has_bias)
7357       args.emplace_back(Operand(bias));
7358    if (has_compare)
7359       args.emplace_back(Operand(compare));
7360    if (has_derivs)
7361       args.emplace_back(Operand(derivs));
7362    args.emplace_back(Operand(coords));
7363    if (has_sample_index)
7364       args.emplace_back(Operand(sample_index));
7365    if (has_lod)
7366       args.emplace_back(lod);
7367
7368    Temp arg;
7369    if (args.size() > 1) {
7370       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
7371       unsigned size = 0;
7372       for (unsigned i = 0; i < args.size(); i++) {
7373          size += args[i].size();
7374          vec->operands[i] = args[i];
7375       }
7376       RegClass rc = RegClass(RegType::vgpr, size);
7377       Temp tmp = bld.tmp(rc);
7378       vec->definitions[0] = Definition(tmp);
7379       ctx->block->instructions.emplace_back(std::move(vec));
7380       arg = tmp;
7381    } else {
7382       assert(args[0].isTemp());
7383       arg = as_vgpr(ctx, args[0].getTemp());
7384    }
7385
7386    /* we don't need the bias, sample index, compare value or offset to be
7387     * computed in WQM but if the p_create_vector copies the coordinates, then it
7388     * needs to be in WQM */
7389    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
7390        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
7391        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
7392       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
7393
7394    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
7395       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
7396
7397       assert(coords.size() == 1);
7398       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
7399       aco_opcode op;
7400       switch (last_bit) {
7401       case 1:
7402          op = aco_opcode::buffer_load_format_x; break;
7403       case 2:
7404          op = aco_opcode::buffer_load_format_xy; break;
7405       case 3:
7406          op = aco_opcode::buffer_load_format_xyz; break;
7407       case 4:
7408          op = aco_opcode::buffer_load_format_xyzw; break;
7409       default:
7410          unreachable("Tex instruction loads more than 4 components.");
7411       }
7412
7413       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
7414       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
7415          tmp_dst = dst;
7416       else
7417          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
7418
7419       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
7420       mubuf->operands[0] = Operand(coords);
7421       mubuf->operands[1] = Operand(resource);
7422       mubuf->operands[2] = Operand((uint32_t) 0);
7423       mubuf->definitions[0] = Definition(tmp_dst);
7424       mubuf->idxen = true;
7425       mubuf->can_reorder = true;
7426       ctx->block->instructions.emplace_back(std::move(mubuf));
7427
7428       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
7429       return;
7430    }
7431
7432
7433    if (instr->op == nir_texop_txf ||
7434        instr->op == nir_texop_txf_ms ||
7435        instr->op == nir_texop_samples_identical ||
7436        instr->op == nir_texop_fragment_fetch ||
7437        instr->op == nir_texop_fragment_mask_fetch) {
7438       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS || instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
7439       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
7440       tex->operands[0] = Operand(arg);
7441       tex->operands[1] = Operand(resource);
7442       tex->dim = dim;
7443       tex->dmask = dmask;
7444       tex->unrm = true;
7445       tex->da = da;
7446       tex->definitions[0] = Definition(tmp_dst);
7447       tex->can_reorder = true;
7448       ctx->block->instructions.emplace_back(std::move(tex));
7449
7450       if (instr->op == nir_texop_samples_identical) {
7451          assert(dmask == 1 && dst.regClass() == v1);
7452          assert(dst.id() != tmp_dst.id());
7453
7454          Temp tmp = bld.tmp(bld.lm);
7455          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
7456          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
7457
7458       } else {
7459          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
7460       }
7461       return;
7462    }
7463
7464    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
7465    aco_opcode opcode = aco_opcode::image_sample;
7466    if (has_offset) { /* image_sample_*_o */
7467       if (has_compare) {
7468          opcode = aco_opcode::image_sample_c_o;
7469          if (has_derivs)
7470             opcode = aco_opcode::image_sample_c_d_o;
7471          if (has_bias)
7472             opcode = aco_opcode::image_sample_c_b_o;
7473          if (level_zero)
7474             opcode = aco_opcode::image_sample_c_lz_o;
7475          if (has_lod)
7476             opcode = aco_opcode::image_sample_c_l_o;
7477       } else {
7478          opcode = aco_opcode::image_sample_o;
7479          if (has_derivs)
7480             opcode = aco_opcode::image_sample_d_o;
7481          if (has_bias)
7482             opcode = aco_opcode::image_sample_b_o;
7483          if (level_zero)
7484             opcode = aco_opcode::image_sample_lz_o;
7485          if (has_lod)
7486             opcode = aco_opcode::image_sample_l_o;
7487       }
7488    } else { /* no offset */
7489       if (has_compare) {
7490          opcode = aco_opcode::image_sample_c;
7491          if (has_derivs)
7492             opcode = aco_opcode::image_sample_c_d;
7493          if (has_bias)
7494             opcode = aco_opcode::image_sample_c_b;
7495          if (level_zero)
7496             opcode = aco_opcode::image_sample_c_lz;
7497          if (has_lod)
7498             opcode = aco_opcode::image_sample_c_l;
7499       } else {
7500          opcode = aco_opcode::image_sample;
7501          if (has_derivs)
7502             opcode = aco_opcode::image_sample_d;
7503          if (has_bias)
7504             opcode = aco_opcode::image_sample_b;
7505          if (level_zero)
7506             opcode = aco_opcode::image_sample_lz;
7507          if (has_lod)
7508             opcode = aco_opcode::image_sample_l;
7509       }
7510    }
7511
7512    if (instr->op == nir_texop_tg4) {
7513       if (has_offset) {
7514          opcode = aco_opcode::image_gather4_lz_o;
7515          if (has_compare)
7516             opcode = aco_opcode::image_gather4_c_lz_o;
7517       } else {
7518          opcode = aco_opcode::image_gather4_lz;
7519          if (has_compare)
7520             opcode = aco_opcode::image_gather4_c_lz;
7521       }
7522    } else if (instr->op == nir_texop_lod) {
7523       opcode = aco_opcode::image_get_lod;
7524    }
7525
7526    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7527    tex->operands[0] = Operand(arg);
7528    tex->operands[1] = Operand(resource);
7529    tex->operands[2] = Operand(sampler);
7530    tex->dim = dim;
7531    tex->dmask = dmask;
7532    tex->da = da;
7533    tex->definitions[0] = Definition(tmp_dst);
7534    tex->can_reorder = true;
7535    ctx->block->instructions.emplace_back(std::move(tex));
7536
7537    if (tg4_integer_cube_workaround) {
7538       assert(tmp_dst.id() != dst.id());
7539       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7540
7541       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7542       Temp val[4];
7543       for (unsigned i = 0; i < dst.size(); i++) {
7544          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7545          Temp cvt_val;
7546          if (stype == GLSL_TYPE_UINT)
7547             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7548          else
7549             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7550          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7551       }
7552       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7553       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7554                            val[0], val[1], val[2], val[3]);
7555    }
7556    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7557    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7558
7559 }
7560
7561
7562 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7563 {
7564    Temp tmp = get_ssa_temp(ctx, ssa);
7565    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7566       return Operand(tmp.regClass());
7567    else
7568       return Operand(tmp);
7569 }
7570
7571 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7572 {
7573    aco_ptr<Pseudo_instruction> phi;
7574    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7575    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7576
7577    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7578    logical |= ctx->block->kind & block_kind_merge;
7579    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7580
7581    /* we want a sorted list of sources, since the predecessor list is also sorted */
7582    std::map<unsigned, nir_ssa_def*> phi_src;
7583    nir_foreach_phi_src(src, instr)
7584       phi_src[src->pred->index] = src->src.ssa;
7585
7586    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7587    unsigned num_operands = 0;
7588    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7589    unsigned num_defined = 0;
7590    unsigned cur_pred_idx = 0;
7591    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7592       if (cur_pred_idx < preds.size()) {
7593          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7594          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7595          unsigned skipped = 0;
7596          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7597             skipped++;
7598          if (cur_pred_idx + skipped < preds.size()) {
7599             for (unsigned i = 0; i < skipped; i++)
7600                operands[num_operands++] = Operand(dst.regClass());
7601             cur_pred_idx += skipped;
7602          } else {
7603             continue;
7604          }
7605       }
7606       cur_pred_idx++;
7607       Operand op = get_phi_operand(ctx, src.second);
7608       operands[num_operands++] = op;
7609       num_defined += !op.isUndefined();
7610    }
7611    /* handle block_kind_continue_or_break at loop exit blocks */
7612    while (cur_pred_idx++ < preds.size())
7613       operands[num_operands++] = Operand(dst.regClass());
7614
7615    if (num_defined == 0) {
7616       Builder bld(ctx->program, ctx->block);
7617       if (dst.regClass() == s1) {
7618          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7619       } else if (dst.regClass() == v1) {
7620          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7621       } else {
7622          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7623          for (unsigned i = 0; i < dst.size(); i++)
7624             vec->operands[i] = Operand(0u);
7625          vec->definitions[0] = Definition(dst);
7626          ctx->block->instructions.emplace_back(std::move(vec));
7627       }
7628       return;
7629    }
7630
7631    /* we can use a linear phi in some cases if one src is undef */
7632    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7633       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7634
7635       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7636       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7637       assert(invert->kind & block_kind_invert);
7638
7639       unsigned then_block = invert->linear_preds[0];
7640
7641       Block* insert_block = NULL;
7642       for (unsigned i = 0; i < num_operands; i++) {
7643          Operand op = operands[i];
7644          if (op.isUndefined())
7645             continue;
7646          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7647          phi->operands[0] = op;
7648          break;
7649       }
7650       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7651       phi->operands[1] = Operand(dst.regClass());
7652       phi->definitions[0] = Definition(dst);
7653       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7654       return;
7655    }
7656
7657    /* try to scalarize vector phis */
7658    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7659       // TODO: scalarize linear phis on divergent ifs
7660       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7661       std::array<Temp, NIR_MAX_VEC_COMPONENTS> new_vec;
7662       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7663          Operand src = operands[i];
7664          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7665             can_scalarize = false;
7666       }
7667       if (can_scalarize) {
7668          unsigned num_components = instr->dest.ssa.num_components;
7669          assert(dst.size() % num_components == 0);
7670          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7671
7672          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7673          for (unsigned k = 0; k < num_components; k++) {
7674             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7675             for (unsigned i = 0; i < num_operands; i++) {
7676                Operand src = operands[i];
7677                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7678             }
7679             Temp phi_dst = {ctx->program->allocateId(), rc};
7680             phi->definitions[0] = Definition(phi_dst);
7681             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7682             new_vec[k] = phi_dst;
7683             vec->operands[k] = Operand(phi_dst);
7684          }
7685          vec->definitions[0] = Definition(dst);
7686          ctx->block->instructions.emplace_back(std::move(vec));
7687          ctx->allocated_vec.emplace(dst.id(), new_vec);
7688          return;
7689       }
7690    }
7691
7692    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7693    for (unsigned i = 0; i < num_operands; i++)
7694       phi->operands[i] = operands[i];
7695    phi->definitions[0] = Definition(dst);
7696    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7697 }
7698
7699
7700 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7701 {
7702    Temp dst = get_ssa_temp(ctx, &instr->def);
7703
7704    assert(dst.type() == RegType::sgpr);
7705
7706    if (dst.size() == 1) {
7707       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7708    } else {
7709       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7710       for (unsigned i = 0; i < dst.size(); i++)
7711          vec->operands[i] = Operand(0u);
7712       vec->definitions[0] = Definition(dst);
7713       ctx->block->instructions.emplace_back(std::move(vec));
7714    }
7715 }
7716
7717 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7718 {
7719    Builder bld(ctx->program, ctx->block);
7720    Block *logical_target;
7721    append_logical_end(ctx->block);
7722    unsigned idx = ctx->block->index;
7723
7724    switch (instr->type) {
7725    case nir_jump_break:
7726       logical_target = ctx->cf_info.parent_loop.exit;
7727       add_logical_edge(idx, logical_target);
7728       ctx->block->kind |= block_kind_break;
7729
7730       if (!ctx->cf_info.parent_if.is_divergent &&
7731           !ctx->cf_info.parent_loop.has_divergent_continue) {
7732          /* uniform break - directly jump out of the loop */
7733          ctx->block->kind |= block_kind_uniform;
7734          ctx->cf_info.has_branch = true;
7735          bld.branch(aco_opcode::p_branch);
7736          add_linear_edge(idx, logical_target);
7737          return;
7738       }
7739       ctx->cf_info.parent_loop.has_divergent_branch = true;
7740       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7741       break;
7742    case nir_jump_continue:
7743       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7744       add_logical_edge(idx, logical_target);
7745       ctx->block->kind |= block_kind_continue;
7746
7747       if (ctx->cf_info.parent_if.is_divergent) {
7748          /* for potential uniform breaks after this continue,
7749             we must ensure that they are handled correctly */
7750          ctx->cf_info.parent_loop.has_divergent_continue = true;
7751          ctx->cf_info.parent_loop.has_divergent_branch = true;
7752          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7753       } else {
7754          /* uniform continue - directly jump to the loop header */
7755          ctx->block->kind |= block_kind_uniform;
7756          ctx->cf_info.has_branch = true;
7757          bld.branch(aco_opcode::p_branch);
7758          add_linear_edge(idx, logical_target);
7759          return;
7760       }
7761       break;
7762    default:
7763       fprintf(stderr, "Unknown NIR jump instr: ");
7764       nir_print_instr(&instr->instr, stderr);
7765       fprintf(stderr, "\n");
7766       abort();
7767    }
7768
7769    /* remove critical edges from linear CFG */
7770    bld.branch(aco_opcode::p_branch);
7771    Block* break_block = ctx->program->create_and_insert_block();
7772    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7773    break_block->kind |= block_kind_uniform;
7774    add_linear_edge(idx, break_block);
7775    /* the loop_header pointer might be invalidated by this point */
7776    if (instr->type == nir_jump_continue)
7777       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7778    add_linear_edge(break_block->index, logical_target);
7779    bld.reset(break_block);
7780    bld.branch(aco_opcode::p_branch);
7781
7782    Block* continue_block = ctx->program->create_and_insert_block();
7783    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7784    add_linear_edge(idx, continue_block);
7785    append_logical_start(continue_block);
7786    ctx->block = continue_block;
7787    return;
7788 }
7789
7790 void visit_block(isel_context *ctx, nir_block *block)
7791 {
7792    nir_foreach_instr(instr, block) {
7793       switch (instr->type) {
7794       case nir_instr_type_alu:
7795          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7796          break;
7797       case nir_instr_type_load_const:
7798          visit_load_const(ctx, nir_instr_as_load_const(instr));
7799          break;
7800       case nir_instr_type_intrinsic:
7801          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7802          break;
7803       case nir_instr_type_tex:
7804          visit_tex(ctx, nir_instr_as_tex(instr));
7805          break;
7806       case nir_instr_type_phi:
7807          visit_phi(ctx, nir_instr_as_phi(instr));
7808          break;
7809       case nir_instr_type_ssa_undef:
7810          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7811          break;
7812       case nir_instr_type_deref:
7813          break;
7814       case nir_instr_type_jump:
7815          visit_jump(ctx, nir_instr_as_jump(instr));
7816          break;
7817       default:
7818          fprintf(stderr, "Unknown NIR instr type: ");
7819          nir_print_instr(instr, stderr);
7820          fprintf(stderr, "\n");
7821          //abort();
7822       }
7823    }
7824
7825    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7826       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7827 }
7828
7829
7830
7831 static void visit_loop(isel_context *ctx, nir_loop *loop)
7832 {
7833    append_logical_end(ctx->block);
7834    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7835    Builder bld(ctx->program, ctx->block);
7836    bld.branch(aco_opcode::p_branch);
7837    unsigned loop_preheader_idx = ctx->block->index;
7838
7839    Block loop_exit = Block();
7840    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7841    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7842
7843    Block* loop_header = ctx->program->create_and_insert_block();
7844    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7845    loop_header->kind |= block_kind_loop_header;
7846    add_edge(loop_preheader_idx, loop_header);
7847    ctx->block = loop_header;
7848
7849    /* emit loop body */
7850    unsigned loop_header_idx = loop_header->index;
7851    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7852    append_logical_start(ctx->block);
7853    visit_cf_list(ctx, &loop->body);
7854
7855    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7856    if (!ctx->cf_info.has_branch) {
7857       append_logical_end(ctx->block);
7858       if (ctx->cf_info.exec_potentially_empty) {
7859          /* Discards can result in code running with an empty exec mask.
7860           * This would result in divergent breaks not ever being taken. As a
7861           * workaround, break the loop when the loop mask is empty instead of
7862           * always continuing. */
7863          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7864          unsigned block_idx = ctx->block->index;
7865
7866          /* create helper blocks to avoid critical edges */
7867          Block *break_block = ctx->program->create_and_insert_block();
7868          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7869          break_block->kind = block_kind_uniform;
7870          bld.reset(break_block);
7871          bld.branch(aco_opcode::p_branch);
7872          add_linear_edge(block_idx, break_block);
7873          add_linear_edge(break_block->index, &loop_exit);
7874
7875          Block *continue_block = ctx->program->create_and_insert_block();
7876          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7877          continue_block->kind = block_kind_uniform;
7878          bld.reset(continue_block);
7879          bld.branch(aco_opcode::p_branch);
7880          add_linear_edge(block_idx, continue_block);
7881          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7882
7883          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7884          ctx->block = &ctx->program->blocks[block_idx];
7885       } else {
7886          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7887          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7888             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7889          else
7890             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7891       }
7892
7893       bld.reset(ctx->block);
7894       bld.branch(aco_opcode::p_branch);
7895    }
7896
7897    /* fixup phis in loop header from unreachable blocks */
7898    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7899       bool linear = ctx->cf_info.has_branch;
7900       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7901       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7902          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7903              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7904             /* the last operand should be the one that needs to be removed */
7905             instr->operands.pop_back();
7906          } else if (!is_phi(instr)) {
7907             break;
7908          }
7909       }
7910    }
7911
7912    ctx->cf_info.has_branch = false;
7913
7914    // TODO: if the loop has not a single exit, we must add one °°
7915    /* emit loop successor block */
7916    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7917    append_logical_start(ctx->block);
7918
7919    #if 0
7920    // TODO: check if it is beneficial to not branch on continues
7921    /* trim linear phis in loop header */
7922    for (auto&& instr : loop_entry->instructions) {
7923       if (instr->opcode == aco_opcode::p_linear_phi) {
7924          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7925          new_phi->definitions[0] = instr->definitions[0];
7926          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7927             new_phi->operands[i] = instr->operands[i];
7928          /* check that the remaining operands are all the same */
7929          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7930             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7931          instr.swap(new_phi);
7932       } else if (instr->opcode == aco_opcode::p_phi) {
7933          continue;
7934       } else {
7935          break;
7936       }
7937    }
7938    #endif
7939 }
7940
7941 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7942 {
7943    ic->cond = cond;
7944
7945    append_logical_end(ctx->block);
7946    ctx->block->kind |= block_kind_branch;
7947
7948    /* branch to linear then block */
7949    assert(cond.regClass() == ctx->program->lane_mask);
7950    aco_ptr<Pseudo_branch_instruction> branch;
7951    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7952    branch->operands[0] = Operand(cond);
7953    ctx->block->instructions.push_back(std::move(branch));
7954
7955    ic->BB_if_idx = ctx->block->index;
7956    ic->BB_invert = Block();
7957    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7958    /* Invert blocks are intentionally not marked as top level because they
7959     * are not part of the logical cfg. */
7960    ic->BB_invert.kind |= block_kind_invert;
7961    ic->BB_endif = Block();
7962    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7963    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7964
7965    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7966    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7967    ctx->cf_info.parent_if.is_divergent = true;
7968    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7969
7970    /** emit logical then block */
7971    Block* BB_then_logical = ctx->program->create_and_insert_block();
7972    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7973    add_edge(ic->BB_if_idx, BB_then_logical);
7974    ctx->block = BB_then_logical;
7975    append_logical_start(BB_then_logical);
7976 }
7977
7978 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7979 {
7980    Block *BB_then_logical = ctx->block;
7981    append_logical_end(BB_then_logical);
7982     /* branch from logical then block to invert block */
7983    aco_ptr<Pseudo_branch_instruction> branch;
7984    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7985    BB_then_logical->instructions.emplace_back(std::move(branch));
7986    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7987    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7988       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7989    BB_then_logical->kind |= block_kind_uniform;
7990    assert(!ctx->cf_info.has_branch);
7991    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7992    ctx->cf_info.parent_loop.has_divergent_branch = false;
7993
7994    /** emit linear then block */
7995    Block* BB_then_linear = ctx->program->create_and_insert_block();
7996    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7997    BB_then_linear->kind |= block_kind_uniform;
7998    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7999    /* branch from linear then block to invert block */
8000    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8001    BB_then_linear->instructions.emplace_back(std::move(branch));
8002    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
8003
8004    /** emit invert merge block */
8005    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
8006    ic->invert_idx = ctx->block->index;
8007
8008    /* branch to linear else block (skip else) */
8009    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
8010    branch->operands[0] = Operand(ic->cond);
8011    ctx->block->instructions.push_back(std::move(branch));
8012
8013    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
8014    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
8015
8016    /** emit logical else block */
8017    Block* BB_else_logical = ctx->program->create_and_insert_block();
8018    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8019    add_logical_edge(ic->BB_if_idx, BB_else_logical);
8020    add_linear_edge(ic->invert_idx, BB_else_logical);
8021    ctx->block = BB_else_logical;
8022    append_logical_start(BB_else_logical);
8023 }
8024
8025 static void end_divergent_if(isel_context *ctx, if_context *ic)
8026 {
8027    Block *BB_else_logical = ctx->block;
8028    append_logical_end(BB_else_logical);
8029
8030    /* branch from logical else block to endif block */
8031    aco_ptr<Pseudo_branch_instruction> branch;
8032    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8033    BB_else_logical->instructions.emplace_back(std::move(branch));
8034    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
8035    if (!ctx->cf_info.parent_loop.has_divergent_branch)
8036       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
8037    BB_else_logical->kind |= block_kind_uniform;
8038
8039    assert(!ctx->cf_info.has_branch);
8040    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
8041
8042
8043    /** emit linear else block */
8044    Block* BB_else_linear = ctx->program->create_and_insert_block();
8045    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8046    BB_else_linear->kind |= block_kind_uniform;
8047    add_linear_edge(ic->invert_idx, BB_else_linear);
8048
8049    /* branch from linear else block to endif block */
8050    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8051    BB_else_linear->instructions.emplace_back(std::move(branch));
8052    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
8053
8054
8055    /** emit endif merge block */
8056    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
8057    append_logical_start(ctx->block);
8058
8059
8060    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
8061    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
8062    /* uniform control flow never has an empty exec-mask */
8063    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
8064       ctx->cf_info.exec_potentially_empty = false;
8065 }
8066
8067 static void visit_if(isel_context *ctx, nir_if *if_stmt)
8068 {
8069    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
8070    Builder bld(ctx->program, ctx->block);
8071    aco_ptr<Pseudo_branch_instruction> branch;
8072
8073    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
8074       /**
8075        * Uniform conditionals are represented in the following way*) :
8076        *
8077        * The linear and logical CFG:
8078        *                        BB_IF
8079        *                        /    \
8080        *       BB_THEN (logical)      BB_ELSE (logical)
8081        *                        \    /
8082        *                        BB_ENDIF
8083        *
8084        * *) Exceptions may be due to break and continue statements within loops
8085        *    If a break/continue happens within uniform control flow, it branches
8086        *    to the loop exit/entry block. Otherwise, it branches to the next
8087        *    merge block.
8088        **/
8089       append_logical_end(ctx->block);
8090       ctx->block->kind |= block_kind_uniform;
8091
8092       /* emit branch */
8093       assert(cond.regClass() == bld.lm);
8094       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
8095       cond = bool_to_scalar_condition(ctx, cond);
8096
8097       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
8098       branch->operands[0] = Operand(cond);
8099       branch->operands[0].setFixed(scc);
8100       ctx->block->instructions.emplace_back(std::move(branch));
8101
8102       unsigned BB_if_idx = ctx->block->index;
8103       Block BB_endif = Block();
8104       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
8105       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
8106
8107       /** emit then block */
8108       Block* BB_then = ctx->program->create_and_insert_block();
8109       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8110       add_edge(BB_if_idx, BB_then);
8111       append_logical_start(BB_then);
8112       ctx->block = BB_then;
8113       visit_cf_list(ctx, &if_stmt->then_list);
8114       BB_then = ctx->block;
8115       bool then_branch = ctx->cf_info.has_branch;
8116       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
8117
8118       if (!then_branch) {
8119          append_logical_end(BB_then);
8120          /* branch from then block to endif block */
8121          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8122          BB_then->instructions.emplace_back(std::move(branch));
8123          add_linear_edge(BB_then->index, &BB_endif);
8124          if (!then_branch_divergent)
8125             add_logical_edge(BB_then->index, &BB_endif);
8126          BB_then->kind |= block_kind_uniform;
8127       }
8128
8129       ctx->cf_info.has_branch = false;
8130       ctx->cf_info.parent_loop.has_divergent_branch = false;
8131
8132       /** emit else block */
8133       Block* BB_else = ctx->program->create_and_insert_block();
8134       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
8135       add_edge(BB_if_idx, BB_else);
8136       append_logical_start(BB_else);
8137       ctx->block = BB_else;
8138       visit_cf_list(ctx, &if_stmt->else_list);
8139       BB_else = ctx->block;
8140
8141       if (!ctx->cf_info.has_branch) {
8142          append_logical_end(BB_else);
8143          /* branch from then block to endif block */
8144          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
8145          BB_else->instructions.emplace_back(std::move(branch));
8146          add_linear_edge(BB_else->index, &BB_endif);
8147          if (!ctx->cf_info.parent_loop.has_divergent_branch)
8148             add_logical_edge(BB_else->index, &BB_endif);
8149          BB_else->kind |= block_kind_uniform;
8150       }
8151
8152       ctx->cf_info.has_branch &= then_branch;
8153       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
8154
8155       /** emit endif merge block */
8156       if (!ctx->cf_info.has_branch) {
8157          ctx->block = ctx->program->insert_block(std::move(BB_endif));
8158          append_logical_start(ctx->block);
8159       }
8160    } else { /* non-uniform condition */
8161       /**
8162        * To maintain a logical and linear CFG without critical edges,
8163        * non-uniform conditionals are represented in the following way*) :
8164        *
8165        * The linear CFG:
8166        *                        BB_IF
8167        *                        /    \
8168        *       BB_THEN (logical)      BB_THEN (linear)
8169        *                        \    /
8170        *                        BB_INVERT (linear)
8171        *                        /    \
8172        *       BB_ELSE (logical)      BB_ELSE (linear)
8173        *                        \    /
8174        *                        BB_ENDIF
8175        *
8176        * The logical CFG:
8177        *                        BB_IF
8178        *                        /    \
8179        *       BB_THEN (logical)      BB_ELSE (logical)
8180        *                        \    /
8181        *                        BB_ENDIF
8182        *
8183        * *) Exceptions may be due to break and continue statements within loops
8184        **/
8185
8186       if_context ic;
8187
8188       begin_divergent_if_then(ctx, &ic, cond);
8189       visit_cf_list(ctx, &if_stmt->then_list);
8190
8191       begin_divergent_if_else(ctx, &ic);
8192       visit_cf_list(ctx, &if_stmt->else_list);
8193
8194       end_divergent_if(ctx, &ic);
8195    }
8196 }
8197
8198 static void visit_cf_list(isel_context *ctx,
8199                           struct exec_list *list)
8200 {
8201    foreach_list_typed(nir_cf_node, node, node, list) {
8202       switch (node->type) {
8203       case nir_cf_node_block:
8204          visit_block(ctx, nir_cf_node_as_block(node));
8205          break;
8206       case nir_cf_node_if:
8207          visit_if(ctx, nir_cf_node_as_if(node));
8208          break;
8209       case nir_cf_node_loop:
8210          visit_loop(ctx, nir_cf_node_as_loop(node));
8211          break;
8212       default:
8213          unreachable("unimplemented cf list type");
8214       }
8215    }
8216 }
8217
8218 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
8219 {
8220    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
8221    uint64_t mask = ctx->outputs.mask[slot];
8222    if (!is_pos && !mask)
8223       return;
8224    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
8225       return;
8226    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8227    exp->enabled_mask = mask;
8228    for (unsigned i = 0; i < 4; ++i) {
8229       if (mask & (1 << i))
8230          exp->operands[i] = Operand(ctx->outputs.outputs[slot][i]);
8231       else
8232          exp->operands[i] = Operand(v1);
8233    }
8234    /* Navi10-14 skip POS0 exports if EXEC=0 and DONE=0, causing a hang.
8235     * Setting valid_mask=1 prevents it and has no other effect.
8236     */
8237    exp->valid_mask = ctx->options->chip_class >= GFX10 && is_pos && *next_pos == 0;
8238    exp->done = false;
8239    exp->compressed = false;
8240    if (is_pos)
8241       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8242    else
8243       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
8244    ctx->block->instructions.emplace_back(std::move(exp));
8245 }
8246
8247 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
8248 {
8249    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
8250    exp->enabled_mask = 0;
8251    for (unsigned i = 0; i < 4; ++i)
8252       exp->operands[i] = Operand(v1);
8253    if (ctx->outputs.mask[VARYING_SLOT_PSIZ]) {
8254       exp->operands[0] = Operand(ctx->outputs.outputs[VARYING_SLOT_PSIZ][0]);
8255       exp->enabled_mask |= 0x1;
8256    }
8257    if (ctx->outputs.mask[VARYING_SLOT_LAYER]) {
8258       exp->operands[2] = Operand(ctx->outputs.outputs[VARYING_SLOT_LAYER][0]);
8259       exp->enabled_mask |= 0x4;
8260    }
8261    if (ctx->outputs.mask[VARYING_SLOT_VIEWPORT]) {
8262       if (ctx->options->chip_class < GFX9) {
8263          exp->operands[3] = Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]);
8264          exp->enabled_mask |= 0x8;
8265       } else {
8266          Builder bld(ctx->program, ctx->block);
8267
8268          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
8269                              Operand(ctx->outputs.outputs[VARYING_SLOT_VIEWPORT][0]));
8270          if (exp->operands[2].isTemp())
8271             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
8272
8273          exp->operands[2] = Operand(out);
8274          exp->enabled_mask |= 0x4;
8275       }
8276    }
8277    exp->valid_mask = ctx->options->chip_class >= GFX10 && *next_pos == 0;
8278    exp->done = false;
8279    exp->compressed = false;
8280    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
8281    ctx->block->instructions.emplace_back(std::move(exp));
8282 }
8283
8284 static void create_vs_exports(isel_context *ctx)
8285 {
8286    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
8287
8288    if (outinfo->export_prim_id) {
8289       ctx->outputs.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
8290       ctx->outputs.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
8291    }
8292
8293    if (ctx->options->key.has_multiview_view_index) {
8294       ctx->outputs.mask[VARYING_SLOT_LAYER] |= 0x1;
8295       ctx->outputs.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
8296    }
8297
8298    /* the order these position exports are created is important */
8299    int next_pos = 0;
8300    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
8301    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
8302       export_vs_psiz_layer_viewport(ctx, &next_pos);
8303    }
8304    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8305       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
8306    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8307       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
8308
8309    if (ctx->export_clip_dists) {
8310       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
8311          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
8312       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
8313          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
8314    }
8315
8316    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8317       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
8318           i != VARYING_SLOT_PRIMITIVE_ID)
8319          continue;
8320
8321       export_vs_varying(ctx, i, false, NULL);
8322    }
8323 }
8324
8325 static void export_fs_mrt_z(isel_context *ctx)
8326 {
8327    Builder bld(ctx->program, ctx->block);
8328    unsigned enabled_channels = 0;
8329    bool compr = false;
8330    Operand values[4];
8331
8332    for (unsigned i = 0; i < 4; ++i) {
8333       values[i] = Operand(v1);
8334    }
8335
8336    /* Both stencil and sample mask only need 16-bits. */
8337    if (!ctx->program->info->ps.writes_z &&
8338        (ctx->program->info->ps.writes_stencil ||
8339         ctx->program->info->ps.writes_sample_mask)) {
8340       compr = true; /* COMPR flag */
8341
8342       if (ctx->program->info->ps.writes_stencil) {
8343          /* Stencil should be in X[23:16]. */
8344          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8345          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
8346          enabled_channels |= 0x3;
8347       }
8348
8349       if (ctx->program->info->ps.writes_sample_mask) {
8350          /* SampleMask should be in Y[15:0]. */
8351          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8352          enabled_channels |= 0xc;
8353      }
8354    } else {
8355       if (ctx->program->info->ps.writes_z) {
8356          values[0] = Operand(ctx->outputs.outputs[FRAG_RESULT_DEPTH][0]);
8357          enabled_channels |= 0x1;
8358       }
8359
8360       if (ctx->program->info->ps.writes_stencil) {
8361          values[1] = Operand(ctx->outputs.outputs[FRAG_RESULT_STENCIL][0]);
8362          enabled_channels |= 0x2;
8363       }
8364
8365       if (ctx->program->info->ps.writes_sample_mask) {
8366          values[2] = Operand(ctx->outputs.outputs[FRAG_RESULT_SAMPLE_MASK][0]);
8367          enabled_channels |= 0x4;
8368       }
8369    }
8370
8371    /* GFX6 (except OLAND and HAINAN) has a bug that it only looks at the X
8372     * writemask component.
8373     */
8374    if (ctx->options->chip_class == GFX6 &&
8375        ctx->options->family != CHIP_OLAND &&
8376        ctx->options->family != CHIP_HAINAN) {
8377             enabled_channels |= 0x1;
8378    }
8379
8380    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8381            enabled_channels, V_008DFC_SQ_EXP_MRTZ, compr);
8382 }
8383
8384 static void export_fs_mrt_color(isel_context *ctx, int slot)
8385 {
8386    Builder bld(ctx->program, ctx->block);
8387    unsigned write_mask = ctx->outputs.mask[slot];
8388    Operand values[4];
8389
8390    for (unsigned i = 0; i < 4; ++i) {
8391       if (write_mask & (1 << i)) {
8392          values[i] = Operand(ctx->outputs.outputs[slot][i]);
8393       } else {
8394          values[i] = Operand(v1);
8395       }
8396    }
8397
8398    unsigned target, col_format;
8399    unsigned enabled_channels = 0;
8400    aco_opcode compr_op = (aco_opcode)0;
8401
8402    slot -= FRAG_RESULT_DATA0;
8403    target = V_008DFC_SQ_EXP_MRT + slot;
8404    col_format = (ctx->options->key.fs.col_format >> (4 * slot)) & 0xf;
8405
8406    bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1;
8407    bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1;
8408
8409    switch (col_format)
8410    {
8411    case V_028714_SPI_SHADER_ZERO:
8412       enabled_channels = 0; /* writemask */
8413       target = V_008DFC_SQ_EXP_NULL;
8414       break;
8415
8416    case V_028714_SPI_SHADER_32_R:
8417       enabled_channels = 1;
8418       break;
8419
8420    case V_028714_SPI_SHADER_32_GR:
8421       enabled_channels = 0x3;
8422       break;
8423
8424    case V_028714_SPI_SHADER_32_AR:
8425       if (ctx->options->chip_class >= GFX10) {
8426          /* Special case: on GFX10, the outputs are different for 32_AR */
8427          enabled_channels = 0x3;
8428          values[1] = values[3];
8429          values[3] = Operand(v1);
8430       } else {
8431          enabled_channels = 0x9;
8432       }
8433       break;
8434
8435    case V_028714_SPI_SHADER_FP16_ABGR:
8436       enabled_channels = 0x5;
8437       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
8438       break;
8439
8440    case V_028714_SPI_SHADER_UNORM16_ABGR:
8441       enabled_channels = 0x5;
8442       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
8443       break;
8444
8445    case V_028714_SPI_SHADER_SNORM16_ABGR:
8446       enabled_channels = 0x5;
8447       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
8448       break;
8449
8450    case V_028714_SPI_SHADER_UINT16_ABGR: {
8451       enabled_channels = 0x5;
8452       compr_op = aco_opcode::v_cvt_pk_u16_u32;
8453       if (is_int8 || is_int10) {
8454          /* clamp */
8455          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
8456          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8457
8458          for (unsigned i = 0; i < 4; i++) {
8459             if ((write_mask >> i) & 1) {
8460                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
8461                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
8462                                     values[i]);
8463             }
8464          }
8465       }
8466       break;
8467    }
8468
8469    case V_028714_SPI_SHADER_SINT16_ABGR:
8470       enabled_channels = 0x5;
8471       compr_op = aco_opcode::v_cvt_pk_i16_i32;
8472       if (is_int8 || is_int10) {
8473          /* clamp */
8474          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
8475          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
8476          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
8477          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
8478
8479          for (unsigned i = 0; i < 4; i++) {
8480             if ((write_mask >> i) & 1) {
8481                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
8482                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
8483                                     values[i]);
8484                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
8485                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
8486                                     values[i]);
8487             }
8488          }
8489       }
8490       break;
8491
8492    case V_028714_SPI_SHADER_32_ABGR:
8493       enabled_channels = 0xF;
8494       break;
8495
8496    default:
8497       break;
8498    }
8499
8500    if (target == V_008DFC_SQ_EXP_NULL)
8501       return;
8502
8503    if ((bool) compr_op) {
8504       for (int i = 0; i < 2; i++) {
8505          /* check if at least one of the values to be compressed is enabled */
8506          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
8507          if (enabled) {
8508             enabled_channels |= enabled << (i*2);
8509             values[i] = bld.vop3(compr_op, bld.def(v1),
8510                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
8511                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
8512          } else {
8513             values[i] = Operand(v1);
8514          }
8515       }
8516       values[2] = Operand(v1);
8517       values[3] = Operand(v1);
8518    } else {
8519       for (int i = 0; i < 4; i++)
8520          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
8521    }
8522
8523    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
8524            enabled_channels, target, (bool) compr_op);
8525 }
8526
8527 static void create_fs_exports(isel_context *ctx)
8528 {
8529    /* Export depth, stencil and sample mask. */
8530    if (ctx->outputs.mask[FRAG_RESULT_DEPTH] ||
8531        ctx->outputs.mask[FRAG_RESULT_STENCIL] ||
8532        ctx->outputs.mask[FRAG_RESULT_SAMPLE_MASK]) {
8533       export_fs_mrt_z(ctx);
8534    }
8535
8536    /* Export all color render targets. */
8537    for (unsigned i = FRAG_RESULT_DATA0; i < FRAG_RESULT_DATA7 + 1; ++i) {
8538       if (ctx->outputs.mask[i])
8539          export_fs_mrt_color(ctx, i);
8540    }
8541 }
8542
8543 static void emit_stream_output(isel_context *ctx,
8544                                Temp const *so_buffers,
8545                                Temp const *so_write_offset,
8546                                const struct radv_stream_output *output)
8547 {
8548    unsigned num_comps = util_bitcount(output->component_mask);
8549    unsigned writemask = (1 << num_comps) - 1;
8550    unsigned loc = output->location;
8551    unsigned buf = output->buffer;
8552
8553    assert(num_comps && num_comps <= 4);
8554    if (!num_comps || num_comps > 4)
8555       return;
8556
8557    unsigned start = ffs(output->component_mask) - 1;
8558
8559    Temp out[4];
8560    bool all_undef = true;
8561    assert(ctx->stage == vertex_vs || ctx->stage == gs_copy_vs);
8562    for (unsigned i = 0; i < num_comps; i++) {
8563       out[i] = ctx->outputs.outputs[loc][start + i];
8564       all_undef = all_undef && !out[i].id();
8565    }
8566    if (all_undef)
8567       return;
8568
8569    while (writemask) {
8570       int start, count;
8571       u_bit_scan_consecutive_range(&writemask, &start, &count);
8572       if (count == 3 && ctx->options->chip_class == GFX6) {
8573          /* GFX6 doesn't support storing vec3, split it. */
8574          writemask |= 1u << (start + 2);
8575          count = 2;
8576       }
8577
8578       unsigned offset = output->offset + start * 4;
8579
8580       Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, count)};
8581       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
8582       for (int i = 0; i < count; ++i)
8583          vec->operands[i] = (ctx->outputs.mask[loc] & 1 << (start + i)) ? Operand(out[start + i]) : Operand(0u);
8584       vec->definitions[0] = Definition(write_data);
8585       ctx->block->instructions.emplace_back(std::move(vec));
8586
8587       aco_opcode opcode;
8588       switch (count) {
8589       case 1:
8590          opcode = aco_opcode::buffer_store_dword;
8591          break;
8592       case 2:
8593          opcode = aco_opcode::buffer_store_dwordx2;
8594          break;
8595       case 3:
8596          opcode = aco_opcode::buffer_store_dwordx3;
8597          break;
8598       case 4:
8599          opcode = aco_opcode::buffer_store_dwordx4;
8600          break;
8601       default:
8602          unreachable("Unsupported dword count.");
8603       }
8604
8605       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
8606       store->operands[0] = Operand(so_write_offset[buf]);
8607       store->operands[1] = Operand(so_buffers[buf]);
8608       store->operands[2] = Operand((uint32_t) 0);
8609       store->operands[3] = Operand(write_data);
8610       if (offset > 4095) {
8611          /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
8612          Builder bld(ctx->program, ctx->block);
8613          store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
8614       } else {
8615          store->offset = offset;
8616       }
8617       store->offen = true;
8618       store->glc = true;
8619       store->dlc = false;
8620       store->slc = true;
8621       store->can_reorder = true;
8622       ctx->block->instructions.emplace_back(std::move(store));
8623    }
8624 }
8625
8626 static void emit_streamout(isel_context *ctx, unsigned stream)
8627 {
8628    Builder bld(ctx->program, ctx->block);
8629
8630    Temp so_buffers[4];
8631    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
8632    for (unsigned i = 0; i < 4; i++) {
8633       unsigned stride = ctx->program->info->so.strides[i];
8634       if (!stride)
8635          continue;
8636
8637       Operand off = bld.copy(bld.def(s1), Operand(i * 16u));
8638       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, off);
8639    }
8640
8641    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8642                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
8643
8644    Temp tid = emit_mbcnt(ctx, bld.def(v1));
8645
8646    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(bld.lm), so_vtx_count, tid);
8647
8648    if_context ic;
8649    begin_divergent_if_then(ctx, &ic, can_emit);
8650
8651    bld.reset(ctx->block);
8652
8653    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
8654
8655    Temp so_write_offset[4];
8656
8657    for (unsigned i = 0; i < 4; i++) {
8658       unsigned stride = ctx->program->info->so.strides[i];
8659       if (!stride)
8660          continue;
8661
8662       if (stride == 1) {
8663          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
8664                                 get_arg(ctx, ctx->args->streamout_write_idx),
8665                                 get_arg(ctx, ctx->args->streamout_offset[i]));
8666          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
8667
8668          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
8669       } else {
8670          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
8671          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
8672                                  get_arg(ctx, ctx->args->streamout_offset[i]));
8673          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
8674       }
8675    }
8676
8677    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
8678       struct radv_stream_output *output =
8679          &ctx->program->info->so.outputs[i];
8680       if (stream != output->stream)
8681          continue;
8682
8683       emit_stream_output(ctx, so_buffers, so_write_offset, output);
8684    }
8685
8686    begin_divergent_if_else(ctx, &ic);
8687    end_divergent_if(ctx, &ic);
8688 }
8689
8690 } /* end namespace */
8691
8692 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
8693 {
8694    /* Split all arguments except for the first (ring_offsets) and the last
8695     * (exec) so that the dead channels don't stay live throughout the program.
8696     */
8697    for (int i = 1; i < startpgm->definitions.size() - 1; i++) {
8698       if (startpgm->definitions[i].regClass().size() > 1) {
8699          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
8700                            startpgm->definitions[i].regClass().size());
8701       }
8702    }
8703 }
8704
8705 void handle_bc_optimize(isel_context *ctx)
8706 {
8707    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
8708    Builder bld(ctx->program, ctx->block);
8709    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
8710    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
8711    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
8712    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
8713    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
8714    if (uses_center && uses_centroid) {
8715       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
8716                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
8717
8718       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
8719          Temp new_coord[2];
8720          for (unsigned i = 0; i < 2; i++) {
8721             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
8722             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
8723             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8724                                     persp_centroid, persp_center, sel);
8725          }
8726          ctx->persp_centroid = bld.tmp(v2);
8727          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
8728                     Operand(new_coord[0]), Operand(new_coord[1]));
8729          emit_split_vector(ctx, ctx->persp_centroid, 2);
8730       }
8731
8732       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
8733          Temp new_coord[2];
8734          for (unsigned i = 0; i < 2; i++) {
8735             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
8736             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
8737             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
8738                                     linear_centroid, linear_center, sel);
8739          }
8740          ctx->linear_centroid = bld.tmp(v2);
8741          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
8742                     Operand(new_coord[0]), Operand(new_coord[1]));
8743          emit_split_vector(ctx, ctx->linear_centroid, 2);
8744       }
8745    }
8746 }
8747
8748 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
8749 {
8750    Program *program = ctx->program;
8751
8752    unsigned float_controls = shader->info.float_controls_execution_mode;
8753
8754    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
8755       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
8756    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
8757       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
8758                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
8759
8760    program->next_fp_mode.must_flush_denorms32 =
8761       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
8762    program->next_fp_mode.must_flush_denorms16_64 =
8763       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8764                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8765
8766    program->next_fp_mode.care_about_round32 =
8767       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8768
8769    program->next_fp_mode.care_about_round16_64 =
8770       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8771                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8772
8773    /* default to preserving fp16 and fp64 denorms, since it's free */
8774    if (program->next_fp_mode.must_flush_denorms16_64)
8775       program->next_fp_mode.denorm16_64 = 0;
8776    else
8777       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8778
8779    /* preserving fp32 denorms is expensive, so only do it if asked */
8780    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8781       program->next_fp_mode.denorm32 = fp_denorm_keep;
8782    else
8783       program->next_fp_mode.denorm32 = 0;
8784
8785    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8786       program->next_fp_mode.round32 = fp_round_tz;
8787    else
8788       program->next_fp_mode.round32 = fp_round_ne;
8789
8790    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8791       program->next_fp_mode.round16_64 = fp_round_tz;
8792    else
8793       program->next_fp_mode.round16_64 = fp_round_ne;
8794
8795    ctx->block->fp_mode = program->next_fp_mode;
8796 }
8797
8798 void cleanup_cfg(Program *program)
8799 {
8800    /* create linear_succs/logical_succs */
8801    for (Block& BB : program->blocks) {
8802       for (unsigned idx : BB.linear_preds)
8803          program->blocks[idx].linear_succs.emplace_back(BB.index);
8804       for (unsigned idx : BB.logical_preds)
8805          program->blocks[idx].logical_succs.emplace_back(BB.index);
8806    }
8807 }
8808
8809 void select_program(Program *program,
8810                     unsigned shader_count,
8811                     struct nir_shader *const *shaders,
8812                     ac_shader_config* config,
8813                     struct radv_shader_args *args)
8814 {
8815    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args, false);
8816
8817    for (unsigned i = 0; i < shader_count; i++) {
8818       nir_shader *nir = shaders[i];
8819       init_context(&ctx, nir);
8820
8821       setup_fp_mode(&ctx, nir);
8822
8823       if (!i) {
8824          /* needs to be after init_context() for FS */
8825          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8826          append_logical_start(ctx.block);
8827          split_arguments(&ctx, startpgm);
8828       }
8829
8830       if_context ic;
8831       if (shader_count >= 2) {
8832          Builder bld(ctx.program, ctx.block);
8833          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | (i * 8u)));
8834          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8835          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8836
8837          begin_divergent_if_then(&ctx, &ic, cond);
8838       }
8839
8840       if (i) {
8841          Builder bld(ctx.program, ctx.block);
8842          assert(ctx.stage == vertex_geometry_gs);
8843          bld.barrier(aco_opcode::p_memory_barrier_shared);
8844          bld.sopp(aco_opcode::s_barrier);
8845
8846          ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u));
8847       } else if (ctx.stage == geometry_gs)
8848          ctx.gs_wave_id = get_arg(&ctx, args->gs_wave_id);
8849
8850       if (ctx.stage == fragment_fs)
8851          handle_bc_optimize(&ctx);
8852
8853       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8854       visit_cf_list(&ctx, &func->body);
8855
8856       if (ctx.program->info->so.num_outputs && ctx.stage == vertex_vs)
8857          emit_streamout(&ctx, 0);
8858
8859       if (ctx.stage == vertex_vs) {
8860          create_vs_exports(&ctx);
8861       } else if (nir->info.stage == MESA_SHADER_GEOMETRY) {
8862          Builder bld(ctx.program, ctx.block);
8863          bld.barrier(aco_opcode::p_memory_barrier_gs_data);
8864          bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0));
8865       }
8866
8867       if (ctx.stage == fragment_fs)
8868          create_fs_exports(&ctx);
8869
8870       if (shader_count >= 2) {
8871          begin_divergent_if_else(&ctx, &ic);
8872          end_divergent_if(&ctx, &ic);
8873       }
8874
8875       ralloc_free(ctx.divergent_vals);
8876    }
8877
8878    program->config->float_mode = program->blocks[0].fp_mode.val;
8879
8880    append_logical_end(ctx.block);
8881    ctx.block->kind |= block_kind_uniform | block_kind_export_end;
8882    Builder bld(ctx.program, ctx.block);
8883    if (ctx.program->wb_smem_l1_on_end)
8884       bld.smem(aco_opcode::s_dcache_wb, false);
8885    bld.sopp(aco_opcode::s_endpgm);
8886
8887    cleanup_cfg(program);
8888 }
8889
8890 void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader,
8891                            ac_shader_config* config,
8892                            struct radv_shader_args *args)
8893 {
8894    isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true);
8895
8896    program->next_fp_mode.preserve_signed_zero_inf_nan32 = false;
8897    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false;
8898    program->next_fp_mode.must_flush_denorms32 = false;
8899    program->next_fp_mode.must_flush_denorms16_64 = false;
8900    program->next_fp_mode.care_about_round32 = false;
8901    program->next_fp_mode.care_about_round16_64 = false;
8902    program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8903    program->next_fp_mode.denorm32 = 0;
8904    program->next_fp_mode.round32 = fp_round_ne;
8905    program->next_fp_mode.round16_64 = fp_round_ne;
8906    ctx.block->fp_mode = program->next_fp_mode;
8907
8908    add_startpgm(&ctx);
8909    append_logical_start(ctx.block);
8910
8911    Builder bld(ctx.program, ctx.block);
8912
8913    Temp gsvs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), program->private_segment_buffer, Operand(RING_GSVS_VS * 16u));
8914
8915    Operand stream_id(0u);
8916    if (args->shader_info->so.num_outputs)
8917       stream_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
8918                            get_arg(&ctx, ctx.args->streamout_config), Operand(0x20018u));
8919
8920    Temp vtx_offset = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), get_arg(&ctx, ctx.args->ac.vertex_id));
8921
8922    std::stack<Block> endif_blocks;
8923
8924    for (unsigned stream = 0; stream < 4; stream++) {
8925       if (stream_id.isConstant() && stream != stream_id.constantValue())
8926          continue;
8927
8928       unsigned num_components = args->shader_info->gs.num_stream_output_components[stream];
8929       if (stream > 0 && (!num_components || !args->shader_info->so.num_outputs))
8930          continue;
8931
8932       memset(ctx.outputs.mask, 0, sizeof(ctx.outputs.mask));
8933
8934       unsigned BB_if_idx = ctx.block->index;
8935       Block BB_endif = Block();
8936       if (!stream_id.isConstant()) {
8937          /* begin IF */
8938          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), stream_id, Operand(stream));
8939          append_logical_end(ctx.block);
8940          ctx.block->kind |= block_kind_uniform;
8941          bld.branch(aco_opcode::p_cbranch_z, cond);
8942
8943          BB_endif.kind |= ctx.block->kind & block_kind_top_level;
8944
8945          ctx.block = ctx.program->create_and_insert_block();
8946          add_edge(BB_if_idx, ctx.block);
8947          bld.reset(ctx.block);
8948          append_logical_start(ctx.block);
8949       }
8950
8951       unsigned offset = 0;
8952       for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
8953          if (args->shader_info->gs.output_streams[i] != stream)
8954             continue;
8955
8956          unsigned output_usage_mask = args->shader_info->gs.output_usage_mask[i];
8957          unsigned length = util_last_bit(output_usage_mask);
8958          for (unsigned j = 0; j < length; ++j) {
8959             if (!(output_usage_mask & (1 << j)))
8960                continue;
8961
8962             unsigned const_offset = offset * args->shader_info->gs.vertices_out * 16 * 4;
8963             Temp voffset = vtx_offset;
8964             if (const_offset >= 4096u) {
8965                voffset = bld.vadd32(bld.def(v1), Operand(const_offset / 4096u * 4096u), voffset);
8966                const_offset %= 4096u;
8967             }
8968
8969             aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(aco_opcode::buffer_load_dword, Format::MUBUF, 3, 1)};
8970             mubuf->definitions[0] = bld.def(v1);
8971             mubuf->operands[0] = Operand(voffset);
8972             mubuf->operands[1] = Operand(gsvs_ring);
8973             mubuf->operands[2] = Operand(0u);
8974             mubuf->offen = true;
8975             mubuf->offset = const_offset;
8976             mubuf->glc = true;
8977             mubuf->slc = true;
8978             mubuf->dlc = args->options->chip_class >= GFX10;
8979             mubuf->barrier = barrier_none;
8980             mubuf->can_reorder = true;
8981
8982             ctx.outputs.mask[i] |= 1 << j;
8983             ctx.outputs.outputs[i][j] = mubuf->definitions[0].getTemp();
8984
8985             bld.insert(std::move(mubuf));
8986
8987             offset++;
8988          }
8989       }
8990
8991       if (args->shader_info->so.num_outputs) {
8992          emit_streamout(&ctx, stream);
8993          bld.reset(ctx.block);
8994       }
8995
8996       if (stream == 0) {
8997          create_vs_exports(&ctx);
8998          ctx.block->kind |= block_kind_export_end;
8999       }
9000
9001       if (!stream_id.isConstant()) {
9002          append_logical_end(ctx.block);
9003
9004          /* branch from then block to endif block */
9005          bld.branch(aco_opcode::p_branch);
9006          add_edge(ctx.block->index, &BB_endif);
9007          ctx.block->kind |= block_kind_uniform;
9008
9009          /* emit else block */
9010          ctx.block = ctx.program->create_and_insert_block();
9011          add_edge(BB_if_idx, ctx.block);
9012          bld.reset(ctx.block);
9013          append_logical_start(ctx.block);
9014
9015          endif_blocks.push(std::move(BB_endif));
9016       }
9017    }
9018
9019    while (!endif_blocks.empty()) {
9020       Block BB_endif = std::move(endif_blocks.top());
9021       endif_blocks.pop();
9022
9023       Block *BB_else = ctx.block;
9024
9025       append_logical_end(BB_else);
9026       /* branch from else block to endif block */
9027       bld.branch(aco_opcode::p_branch);
9028       add_edge(BB_else->index, &BB_endif);
9029       BB_else->kind |= block_kind_uniform;
9030
9031       /** emit endif merge block */
9032       ctx.block = program->insert_block(std::move(BB_endif));
9033       bld.reset(ctx.block);
9034       append_logical_start(ctx.block);
9035    }
9036
9037    program->config->float_mode = program->blocks[0].fp_mode.val;
9038
9039    append_logical_end(ctx.block);
9040    ctx.block->kind |= block_kind_uniform;
9041    bld.sopp(aco_opcode::s_endpgm);
9042
9043    cleanup_cfg(program);
9044 }
9045 }