src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 127 {
 128    Builder bld(ctx->program, ctx->block);
 129
 130    if (!dst.id())
 131       dst = bld.tmp(src.regClass());
 132
 133    if (ctx->stage != fragment_fs) {
 134       if (!dst.id())
 135          return src;
 136
 137       if (src.type() == RegType::vgpr || src.size() > 1)
 138          bld.copy(Definition(dst), src);
 139       else
 140          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 141       return dst;
 142    }
 143
 144    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 145    ctx->program->needs_wqm |= program_needs_wqm;
 146    return dst;
 147 }
 148
 149 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 150 {
 151    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 152
 153    /* Currently not implemented on GFX6-7 */
 154    assert(ctx->options->chip_class >= GFX8);
 155
 156    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 157       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 158    }
 159
 160    /* GFX10, wave64 mode:
 161     * The bpermute instruction is limited to half-wave operation, which means that it can't
 162     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 163     * emulate it here.
 164     */
 165    if (!ctx->has_gfx10_wave64_bpermute) {
 166       ctx->has_gfx10_wave64_bpermute = true;
 167       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 168       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 169    }
 170
 171    Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u));
 172    lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1), lane_id);
 173    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 174    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 175    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 176
 177    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 178                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 179 }
 180
 181 Temp as_vgpr(isel_context *ctx, Temp val)
 182 {
 183    if (val.type() == RegType::sgpr) {
 184       Builder bld(ctx->program, ctx->block);
 185       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 186    }
 187    assert(val.type() == RegType::vgpr);
 188    return val;
 189 }
 190
 191 //assumes a != 0xffffffff
 192 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 193 {
 194    assert(b != 0);
 195    Builder bld(ctx->program, ctx->block);
 196
 197    if (util_is_power_of_two_or_zero(b)) {
 198       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 199       return;
 200    }
 201
 202    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 203
 204    assert(info.multiplier <= 0xffffffff);
 205
 206    bool pre_shift = info.pre_shift != 0;
 207    bool increment = info.increment != 0;
 208    bool multiply = true;
 209    bool post_shift = info.post_shift != 0;
 210
 211    if (!pre_shift && !increment && !multiply && !post_shift) {
 212       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 213       return;
 214    }
 215
 216    Temp pre_shift_dst = a;
 217    if (pre_shift) {
 218       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 219       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 220    }
 221
 222    Temp increment_dst = pre_shift_dst;
 223    if (increment) {
 224       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 225       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 226    }
 227
 228    Temp multiply_dst = increment_dst;
 229    if (multiply) {
 230       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 231       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 232                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 233    }
 234
 235    if (post_shift) {
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 237    }
 238 }
 239
 240 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 241 {
 242    Builder bld(ctx->program, ctx->block);
 243    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 244 }
 245
 246
 247 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 248 {
 249    /* no need to extract the whole vector */
 250    if (src.regClass() == dst_rc) {
 251       assert(idx == 0);
 252       return src;
 253    }
 254    assert(src.size() > idx);
 255    Builder bld(ctx->program, ctx->block);
 256    auto it = ctx->allocated_vec.find(src.id());
 257    /* the size check needs to be early because elements other than 0 may be garbage */
 258    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 259       if (it->second[idx].regClass() == dst_rc) {
 260          return it->second[idx];
 261       } else {
 262          assert(dst_rc.size() == it->second[idx].regClass().size());
 263          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 264          return bld.copy(bld.def(dst_rc), it->second[idx]);
 265       }
 266    }
 267
 268    if (src.size() == dst_rc.size()) {
 269       assert(idx == 0);
 270       return bld.copy(bld.def(dst_rc), src);
 271    } else {
 272       Temp dst = bld.tmp(dst_rc);
 273       emit_extract_vector(ctx, src, idx, dst);
 274       return dst;
 275    }
 276 }
 277
 278 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 279 {
 280    if (num_components == 1)
 281       return;
 282    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 283       return;
 284    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 285    split->operands[0] = Operand(vec_src);
 286    std::array<Temp,4> elems;
 287    for (unsigned i = 0; i < num_components; i++) {
 288       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 289       split->definitions[i] = Definition(elems[i]);
 290    }
 291    ctx->block->instructions.emplace_back(std::move(split));
 292    ctx->allocated_vec.emplace(vec_src.id(), elems);
 293 }
 294
 295 /* This vector expansion uses a mask to determine which elements in the new vector
 296  * come from the original vector. The other elements are undefined. */
 297 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 298 {
 299    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 300
 301    if (vec_src == dst)
 302       return;
 303
 304    Builder bld(ctx->program, ctx->block);
 305    if (num_components == 1) {
 306       if (dst.type() == RegType::sgpr)
 307          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 308       else
 309          bld.copy(Definition(dst), vec_src);
 310       return;
 311    }
 312
 313    unsigned component_size = dst.size() / num_components;
 314    std::array<Temp,4> elems;
 315
 316    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 317    vec->definitions[0] = Definition(dst);
 318    unsigned k = 0;
 319    for (unsigned i = 0; i < num_components; i++) {
 320       if (mask & (1 << i)) {
 321          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 322          if (dst.type() == RegType::sgpr)
 323             src = bld.as_uniform(src);
 324          vec->operands[i] = Operand(src);
 325       } else {
 326          vec->operands[i] = Operand(0u);
 327       }
 328       elems[i] = vec->operands[i].getTemp();
 329    }
 330    ctx->block->instructions.emplace_back(std::move(vec));
 331    ctx->allocated_vec.emplace(dst.id(), elems);
 332 }
 333
 334 Temp as_divergent_bool(isel_context *ctx, Temp val, bool vcc_hint)
 335 {
 336    if (val.regClass() == s2) {
 337       return val;
 338    } else {
 339       assert(val.regClass() == s1);
 340       Builder bld(ctx->program, ctx->block);
 341       Definition& def = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2),
 342                                  Operand((uint32_t) -1), Operand(0u), bld.scc(val)).def(0);
 343       if (vcc_hint)
 344          def.setHint(vcc);
 345       return def.getTemp();
 346    }
 347 }
 348
 349 Temp as_uniform_bool(isel_context *ctx, Temp val)
 350 {
 351    if (val.regClass() == s1) {
 352       return val;
 353    } else {
 354       assert(val.regClass() == s2);
 355       Builder bld(ctx->program, ctx->block);
 356       /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 357       return bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), Operand(0u), emit_wqm(ctx, val));
 358    }
 359 }
 360
 361 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 362 {
 363    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 364       return get_ssa_temp(ctx, src.src.ssa);
 365
 366    if (src.src.ssa->num_components == size) {
 367       bool identity_swizzle = true;
 368       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 369          if (src.swizzle[i] != i)
 370             identity_swizzle = false;
 371       }
 372       if (identity_swizzle)
 373          return get_ssa_temp(ctx, src.src.ssa);
 374    }
 375
 376    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 377    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 378    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 379    assert(vec.size() % elem_size == 0);
 380
 381    RegClass elem_rc = RegClass(vec.type(), elem_size);
 382    if (size == 1) {
 383       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 384    } else {
 385       assert(size <= 4);
 386       std::array<Temp,4> elems;
 387       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 388       for (unsigned i = 0; i < size; ++i) {
 389          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 390          vec_instr->operands[i] = Operand{elems[i]};
 391       }
 392       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 393       vec_instr->definitions[0] = Definition(dst);
 394       ctx->block->instructions.emplace_back(std::move(vec_instr));
 395       ctx->allocated_vec.emplace(dst.id(), elems);
 396       return dst;
 397    }
 398 }
 399
 400 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 401 {
 402    if (ptr.size() == 2)
 403       return ptr;
 404    Builder bld(ctx->program, ctx->block);
 405    if (ptr.type() == RegType::vgpr)
 406       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 407    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 408                      ptr, Operand((unsigned)ctx->options->address32_hi));
 409 }
 410
 411 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 412 {
 413    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 414    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 415    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 416    sop2->definitions[0] = Definition(dst);
 417    if (writes_scc)
 418       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 419    ctx->block->instructions.emplace_back(std::move(sop2));
 420 }
 421
 422 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 423 {
 424    Builder bld(ctx->program, ctx->block);
 425    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 426    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 427    if (src1.type() == RegType::sgpr) {
 428       if (commutative && src0.type() == RegType::vgpr) {
 429          Temp t = src0;
 430          src0 = src1;
 431          src1 = t;
 432       } else if (src0.type() == RegType::vgpr &&
 433                  op != aco_opcode::v_madmk_f32 &&
 434                  op != aco_opcode::v_madak_f32 &&
 435                  op != aco_opcode::v_madmk_f16 &&
 436                  op != aco_opcode::v_madak_f16) {
 437          /* If the instruction is not commutative, we emit a VOP3A instruction */
 438          bld.vop2_e64(op, Definition(dst), src0, src1);
 439          return;
 440       } else {
 441          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 442       }
 443    }
 444    bld.vop2(op, Definition(dst), src0, src1);
 445 }
 446
 447 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 448 {
 449    Temp src0 = get_alu_src(ctx, instr->src[0]);
 450    Temp src1 = get_alu_src(ctx, instr->src[1]);
 451    Temp src2 = get_alu_src(ctx, instr->src[2]);
 452
 453    /* ensure that the instruction has at most 1 sgpr operand
 454     * The optimizer will inline constants for us */
 455    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 456       src0 = as_vgpr(ctx, src0);
 457    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 458       src1 = as_vgpr(ctx, src1);
 459    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 460       src2 = as_vgpr(ctx, src2);
 461
 462    Builder bld(ctx->program, ctx->block);
 463    bld.vop3(op, Definition(dst), src0, src1, src2);
 464 }
 465
 466 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 467 {
 468    Builder bld(ctx->program, ctx->block);
 469    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 470 }
 471
 472 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 473 {
 474    Temp src0 = get_alu_src(ctx, instr->src[0]);
 475    Temp src1 = get_alu_src(ctx, instr->src[1]);
 476    aco_ptr<Instruction> vopc;
 477    if (src1.type() == RegType::sgpr) {
 478       if (src0.type() == RegType::vgpr) {
 479          /* to swap the operands, we might also have to change the opcode */
 480          switch (op) {
 481             case aco_opcode::v_cmp_lt_f32:
 482                op = aco_opcode::v_cmp_gt_f32;
 483                break;
 484             case aco_opcode::v_cmp_ge_f32:
 485                op = aco_opcode::v_cmp_le_f32;
 486                break;
 487             case aco_opcode::v_cmp_lt_i32:
 488                op = aco_opcode::v_cmp_gt_i32;
 489                break;
 490             case aco_opcode::v_cmp_ge_i32:
 491                op = aco_opcode::v_cmp_le_i32;
 492                break;
 493             case aco_opcode::v_cmp_lt_u32:
 494                op = aco_opcode::v_cmp_gt_u32;
 495                break;
 496             case aco_opcode::v_cmp_ge_u32:
 497                op = aco_opcode::v_cmp_le_u32;
 498                break;
 499             case aco_opcode::v_cmp_lt_f64:
 500                op = aco_opcode::v_cmp_gt_f64;
 501                break;
 502             case aco_opcode::v_cmp_ge_f64:
 503                op = aco_opcode::v_cmp_le_f64;
 504                break;
 505             case aco_opcode::v_cmp_lt_i64:
 506                op = aco_opcode::v_cmp_gt_i64;
 507                break;
 508             case aco_opcode::v_cmp_ge_i64:
 509                op = aco_opcode::v_cmp_le_i64;
 510                break;
 511             case aco_opcode::v_cmp_lt_u64:
 512                op = aco_opcode::v_cmp_gt_u64;
 513                break;
 514             case aco_opcode::v_cmp_ge_u64:
 515                op = aco_opcode::v_cmp_le_u64;
 516                break;
 517             default: /* eq and ne are commutative */
 518                break;
 519          }
 520          Temp t = src0;
 521          src0 = src1;
 522          src1 = t;
 523       } else {
 524          src1 = as_vgpr(ctx, src1);
 525       }
 526    }
 527    Builder bld(ctx->program, ctx->block);
 528    bld.vopc(op, Definition(dst), src0, src1).def(0).setHint(vcc);
 529 }
 530
 531 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 532 {
 533    if (dst.regClass() == s2) {
 534       emit_vopc_instruction(ctx, instr, op, dst);
 535       if (!ctx->divergent_vals[instr->dest.dest.ssa.index])
 536          emit_split_vector(ctx, dst, 2);
 537    } else if (dst.regClass() == s1) {
 538       Temp src0 = get_alu_src(ctx, instr->src[0]);
 539       Temp src1 = get_alu_src(ctx, instr->src[1]);
 540       assert(src0.type() == RegType::sgpr && src1.type() == RegType::sgpr);
 541
 542       Builder bld(ctx->program, ctx->block);
 543       bld.sopc(op, bld.scc(Definition(dst)), src0, src1);
 544
 545    } else {
 546       assert(false);
 547    }
 548 }
 549
 550 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 551 {
 552    Builder bld(ctx->program, ctx->block);
 553    Temp src0 = get_alu_src(ctx, instr->src[0]);
 554    Temp src1 = get_alu_src(ctx, instr->src[1]);
 555    if (dst.regClass() == s2) {
 556       bld.sop2(op64, Definition(dst), bld.def(s1, scc),
 557                as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
 558    } else {
 559       assert(dst.regClass() == s1);
 560       bld.sop2(op32, bld.def(s1), bld.scc(Definition(dst)),
 561                as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
 562    }
 563 }
 564
 565
 566 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 567 {
 568    Builder bld(ctx->program, ctx->block);
 569    Temp cond = get_alu_src(ctx, instr->src[0]);
 570    Temp then = get_alu_src(ctx, instr->src[1]);
 571    Temp els = get_alu_src(ctx, instr->src[2]);
 572
 573    if (dst.type() == RegType::vgpr) {
 574       cond = as_divergent_bool(ctx, cond, true);
 575
 576       aco_ptr<Instruction> bcsel;
 577       if (dst.size() == 1) {
 578          then = as_vgpr(ctx, then);
 579          els = as_vgpr(ctx, els);
 580
 581          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 582       } else if (dst.size() == 2) {
 583          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 584          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 585          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 586          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 587
 588          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 589          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 590
 591          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 592       } else {
 593          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 594          nir_print_instr(&instr->instr, stderr);
 595          fprintf(stderr, "\n");
 596       }
 597       return;
 598    }
 599
 600    if (instr->dest.dest.ssa.bit_size != 1) { /* uniform condition and values in sgpr */
 601       if (dst.regClass() == s1 || dst.regClass() == s2) {
 602          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 603          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 604          bld.sop2(op, Definition(dst), then, els, bld.scc(as_uniform_bool(ctx, cond)));
 605       } else {
 606          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 607          nir_print_instr(&instr->instr, stderr);
 608          fprintf(stderr, "\n");
 609       }
 610       return;
 611    }
 612
 613    /* boolean bcsel */
 614    assert(instr->dest.dest.ssa.bit_size == 1);
 615
 616    if (dst.regClass() == s1)
 617       cond = as_uniform_bool(ctx, cond);
 618
 619    if (cond.regClass() == s1) { /* uniform selection */
 620       aco_opcode op;
 621       if (dst.regClass() == s2) {
 622          op = aco_opcode::s_cselect_b64;
 623          then = as_divergent_bool(ctx, then, false);
 624          els = as_divergent_bool(ctx, els, false);
 625       } else {
 626          assert(dst.regClass() == s1);
 627          op = aco_opcode::s_cselect_b32;
 628          then = as_uniform_bool(ctx, then);
 629          els = as_uniform_bool(ctx, els);
 630       }
 631       bld.sop2(op, Definition(dst), then, els, bld.scc(cond));
 632       return;
 633    }
 634
 635    /* divergent boolean bcsel
 636     * this implements bcsel on bools: dst = s0 ? s1 : s2
 637     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 638    assert (dst.regClass() == s2);
 639    then = as_divergent_bool(ctx, then, false);
 640    els = as_divergent_bool(ctx, els, false);
 641
 642    if (cond.id() != then.id())
 643       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 644
 645    if (cond.id() == els.id())
 646       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 647    else
 648       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 649                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 650 }
 651
 652 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 653 {
 654    if (!instr->dest.dest.is_ssa) {
 655       fprintf(stderr, "nir alu dst not in ssa: ");
 656       nir_print_instr(&instr->instr, stderr);
 657       fprintf(stderr, "\n");
 658       abort();
 659    }
 660    Builder bld(ctx->program, ctx->block);
 661    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 662    switch(instr->op) {
 663    case nir_op_vec2:
 664    case nir_op_vec3:
 665    case nir_op_vec4: {
 666       std::array<Temp,4> elems;
 667       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 668       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 669          elems[i] = get_alu_src(ctx, instr->src[i]);
 670          vec->operands[i] = Operand{elems[i]};
 671       }
 672       vec->definitions[0] = Definition(dst);
 673       ctx->block->instructions.emplace_back(std::move(vec));
 674       ctx->allocated_vec.emplace(dst.id(), elems);
 675       break;
 676    }
 677    case nir_op_mov: {
 678       Temp src = get_alu_src(ctx, instr->src[0]);
 679       aco_ptr<Instruction> mov;
 680       if (dst.type() == RegType::sgpr) {
 681          if (src.type() == RegType::vgpr)
 682             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 683          else if (src.regClass() == s1)
 684             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 685          else if (src.regClass() == s2)
 686             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 687          else
 688             unreachable("wrong src register class for nir_op_imov");
 689       } else if (dst.regClass() == v1) {
 690          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 691       } else if (dst.regClass() == v2) {
 692          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 693       } else {
 694          nir_print_instr(&instr->instr, stderr);
 695          unreachable("Should have been lowered to scalar.");
 696       }
 697       break;
 698    }
 699    case nir_op_inot: {
 700       Temp src = get_alu_src(ctx, instr->src[0]);
 701       /* uniform booleans */
 702       if (instr->dest.dest.ssa.bit_size == 1 && dst.regClass() == s1) {
 703          if (src.regClass() == s1) {
 704             /* in this case, src is either 1 or 0 */
 705             bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.scc(Definition(dst)), Operand(1u), src);
 706          } else {
 707             /* src is either exec_mask or 0 */
 708             assert(src.regClass() == s2);
 709             bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(Definition(dst)), Operand(0u), src);
 710          }
 711       } else if (dst.regClass() == v1) {
 712          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 713       } else if (dst.type() == RegType::sgpr) {
 714          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 715          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 716       } else {
 717          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 718          nir_print_instr(&instr->instr, stderr);
 719          fprintf(stderr, "\n");
 720       }
 721       break;
 722    }
 723    case nir_op_ineg: {
 724       Temp src = get_alu_src(ctx, instr->src[0]);
 725       if (dst.regClass() == v1) {
 726          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 727       } else if (dst.regClass() == s1) {
 728          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 729       } else if (dst.size() == 2) {
 730          Temp src0 = bld.tmp(dst.type(), 1);
 731          Temp src1 = bld.tmp(dst.type(), 1);
 732          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 733
 734          if (dst.regClass() == s2) {
 735             Temp carry = bld.tmp(s1);
 736             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 737             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 738             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 739          } else {
 740             Temp lower = bld.tmp(v1);
 741             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 742             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 743             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 744          }
 745       } else {
 746          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 747          nir_print_instr(&instr->instr, stderr);
 748          fprintf(stderr, "\n");
 749       }
 750       break;
 751    }
 752    case nir_op_iabs: {
 753       if (dst.regClass() == s1) {
 754          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 755       } else if (dst.regClass() == v1) {
 756          Temp src = get_alu_src(ctx, instr->src[0]);
 757          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 758       } else {
 759          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 760          nir_print_instr(&instr->instr, stderr);
 761          fprintf(stderr, "\n");
 762       }
 763       break;
 764    }
 765    case nir_op_isign: {
 766       Temp src = get_alu_src(ctx, instr->src[0]);
 767       if (dst.regClass() == s1) {
 768          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 769          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 770          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 771       } else if (dst.regClass() == s2) {
 772          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 773          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 774          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 775       } else if (dst.regClass() == v1) {
 776          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 777          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 778          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 779       } else if (dst.regClass() == v2) {
 780          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 781          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 782          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 783          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 784          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 785          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 786       } else {
 787          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 788          nir_print_instr(&instr->instr, stderr);
 789          fprintf(stderr, "\n");
 790       }
 791       break;
 792    }
 793    case nir_op_imax: {
 794       if (dst.regClass() == v1) {
 795          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 796       } else if (dst.regClass() == s1) {
 797          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 798       } else {
 799          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 800          nir_print_instr(&instr->instr, stderr);
 801          fprintf(stderr, "\n");
 802       }
 803       break;
 804    }
 805    case nir_op_umax: {
 806       if (dst.regClass() == v1) {
 807          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 808       } else if (dst.regClass() == s1) {
 809          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 810       } else {
 811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 812          nir_print_instr(&instr->instr, stderr);
 813          fprintf(stderr, "\n");
 814       }
 815       break;
 816    }
 817    case nir_op_imin: {
 818       if (dst.regClass() == v1) {
 819          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 820       } else if (dst.regClass() == s1) {
 821          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 822       } else {
 823          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 824          nir_print_instr(&instr->instr, stderr);
 825          fprintf(stderr, "\n");
 826       }
 827       break;
 828    }
 829    case nir_op_umin: {
 830       if (dst.regClass() == v1) {
 831          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 832       } else if (dst.regClass() == s1) {
 833          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 834       } else {
 835          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 836          nir_print_instr(&instr->instr, stderr);
 837          fprintf(stderr, "\n");
 838       }
 839       break;
 840    }
 841    case nir_op_ior: {
 842       if (instr->dest.dest.ssa.bit_size == 1) {
 843          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 844       } else if (dst.regClass() == v1) {
 845          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 846       } else if (dst.regClass() == s1) {
 847          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 848       } else if (dst.regClass() == s2) {
 849          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 850       } else {
 851          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 852          nir_print_instr(&instr->instr, stderr);
 853          fprintf(stderr, "\n");
 854       }
 855       break;
 856    }
 857    case nir_op_iand: {
 858       if (instr->dest.dest.ssa.bit_size == 1) {
 859          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 860       } else if (dst.regClass() == v1) {
 861          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 862       } else if (dst.regClass() == s1) {
 863          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 864       } else if (dst.regClass() == s2) {
 865          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 866       } else {
 867          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 868          nir_print_instr(&instr->instr, stderr);
 869          fprintf(stderr, "\n");
 870       }
 871       break;
 872    }
 873    case nir_op_ixor: {
 874       if (instr->dest.dest.ssa.bit_size == 1) {
 875          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 876       } else if (dst.regClass() == v1) {
 877          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 878       } else if (dst.regClass() == s1) {
 879          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 880       } else if (dst.regClass() == s2) {
 881          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 882       } else {
 883          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 884          nir_print_instr(&instr->instr, stderr);
 885          fprintf(stderr, "\n");
 886       }
 887       break;
 888    }
 889    case nir_op_ushr: {
 890       if (dst.regClass() == v1) {
 891          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 892       } else if (dst.regClass() == v2) {
 893          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 894                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 895       } else if (dst.regClass() == s2) {
 896          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 897       } else if (dst.regClass() == s1) {
 898          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 899       } else {
 900          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 901          nir_print_instr(&instr->instr, stderr);
 902          fprintf(stderr, "\n");
 903       }
 904       break;
 905    }
 906    case nir_op_ishl: {
 907       if (dst.regClass() == v1) {
 908          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 909       } else if (dst.regClass() == v2) {
 910          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 911                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 912       } else if (dst.regClass() == s1) {
 913          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 914       } else if (dst.regClass() == s2) {
 915          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 916       } else {
 917          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 918          nir_print_instr(&instr->instr, stderr);
 919          fprintf(stderr, "\n");
 920       }
 921       break;
 922    }
 923    case nir_op_ishr: {
 924       if (dst.regClass() == v1) {
 925          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 926       } else if (dst.regClass() == v2) {
 927          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 928                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 929       } else if (dst.regClass() == s1) {
 930          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 931       } else if (dst.regClass() == s2) {
 932          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 933       } else {
 934          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 935          nir_print_instr(&instr->instr, stderr);
 936          fprintf(stderr, "\n");
 937       }
 938       break;
 939    }
 940    case nir_op_find_lsb: {
 941       Temp src = get_alu_src(ctx, instr->src[0]);
 942       if (src.regClass() == s1) {
 943          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 944       } else if (src.regClass() == v1) {
 945          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 946       } else if (src.regClass() == s2) {
 947          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 948       } else {
 949          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 950          nir_print_instr(&instr->instr, stderr);
 951          fprintf(stderr, "\n");
 952       }
 953       break;
 954    }
 955    case nir_op_ufind_msb:
 956    case nir_op_ifind_msb: {
 957       Temp src = get_alu_src(ctx, instr->src[0]);
 958       if (src.regClass() == s1 || src.regClass() == s2) {
 959          aco_opcode op = src.regClass() == s2 ?
 960                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 961                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 962          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 963
 964          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 965                                         Operand(src.size() * 32u - 1u), msb_rev);
 966          Temp msb = sub.def(0).getTemp();
 967          Temp carry = sub.def(1).getTemp();
 968
 969          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 970       } else if (src.regClass() == v1) {
 971          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 972          Temp msb_rev = bld.tmp(v1);
 973          emit_vop1_instruction(ctx, instr, op, msb_rev);
 974          Temp msb = bld.tmp(v1);
 975          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 976          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 977       } else {
 978          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 979          nir_print_instr(&instr->instr, stderr);
 980          fprintf(stderr, "\n");
 981       }
 982       break;
 983    }
 984    case nir_op_bitfield_reverse: {
 985       if (dst.regClass() == s1) {
 986          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 987       } else if (dst.regClass() == v1) {
 988          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 989       } else {
 990          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 991          nir_print_instr(&instr->instr, stderr);
 992          fprintf(stderr, "\n");
 993       }
 994       break;
 995    }
 996    case nir_op_iadd: {
 997       if (dst.regClass() == s1) {
 998          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
 999          break;
1000       }
1001
1002       Temp src0 = get_alu_src(ctx, instr->src[0]);
1003       Temp src1 = get_alu_src(ctx, instr->src[1]);
1004       if (dst.regClass() == v1) {
1005          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1006          break;
1007       }
1008
1009       assert(src0.size() == 2 && src1.size() == 2);
1010       Temp src00 = bld.tmp(src0.type(), 1);
1011       Temp src01 = bld.tmp(dst.type(), 1);
1012       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1013       Temp src10 = bld.tmp(src1.type(), 1);
1014       Temp src11 = bld.tmp(dst.type(), 1);
1015       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1016
1017       if (dst.regClass() == s2) {
1018          Temp carry = bld.tmp(s1);
1019          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1020          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1021          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1022       } else if (dst.regClass() == v2) {
1023          Temp dst0 = bld.tmp(v1);
1024          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1025          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1026          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1027       } else {
1028          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1029          nir_print_instr(&instr->instr, stderr);
1030          fprintf(stderr, "\n");
1031       }
1032       break;
1033    }
1034    case nir_op_uadd_sat: {
1035       Temp src0 = get_alu_src(ctx, instr->src[0]);
1036       Temp src1 = get_alu_src(ctx, instr->src[1]);
1037       if (dst.regClass() == s1) {
1038          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1039          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1040                   src0, src1);
1041          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1042       } else if (dst.regClass() == v1) {
1043          if (ctx->options->chip_class >= GFX9) {
1044             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1045             add->operands[0] = Operand(src0);
1046             add->operands[1] = Operand(src1);
1047             add->definitions[0] = Definition(dst);
1048             add->clamp = 1;
1049             ctx->block->instructions.emplace_back(std::move(add));
1050          } else {
1051             if (src1.regClass() != v1)
1052                std::swap(src0, src1);
1053             assert(src1.regClass() == v1);
1054             Temp tmp = bld.tmp(v1);
1055             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1056             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1057          }
1058       } else {
1059          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1060          nir_print_instr(&instr->instr, stderr);
1061          fprintf(stderr, "\n");
1062       }
1063       break;
1064    }
1065    case nir_op_uadd_carry: {
1066       Temp src0 = get_alu_src(ctx, instr->src[0]);
1067       Temp src1 = get_alu_src(ctx, instr->src[1]);
1068       if (dst.regClass() == s1) {
1069          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1070          break;
1071       }
1072       if (dst.regClass() == v1) {
1073          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1074          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1075          break;
1076       }
1077
1078       Temp src00 = bld.tmp(src0.type(), 1);
1079       Temp src01 = bld.tmp(dst.type(), 1);
1080       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1081       Temp src10 = bld.tmp(src1.type(), 1);
1082       Temp src11 = bld.tmp(dst.type(), 1);
1083       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1084       if (dst.regClass() == s2) {
1085          Temp carry = bld.tmp(s1);
1086          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1087          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1088          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1089       } else if (dst.regClass() == v2) {
1090          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1091          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1092          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1093          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1094       } else {
1095          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1096          nir_print_instr(&instr->instr, stderr);
1097          fprintf(stderr, "\n");
1098       }
1099       break;
1100    }
1101    case nir_op_isub: {
1102       if (dst.regClass() == s1) {
1103          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1104          break;
1105       }
1106
1107       Temp src0 = get_alu_src(ctx, instr->src[0]);
1108       Temp src1 = get_alu_src(ctx, instr->src[1]);
1109       if (dst.regClass() == v1) {
1110          bld.vsub32(Definition(dst), src0, src1);
1111          break;
1112       }
1113
1114       Temp src00 = bld.tmp(src0.type(), 1);
1115       Temp src01 = bld.tmp(dst.type(), 1);
1116       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1117       Temp src10 = bld.tmp(src1.type(), 1);
1118       Temp src11 = bld.tmp(dst.type(), 1);
1119       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1120       if (dst.regClass() == s2) {
1121          Temp carry = bld.tmp(s1);
1122          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1123          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1124          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1125       } else if (dst.regClass() == v2) {
1126          Temp lower = bld.tmp(v1);
1127          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1128          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1129          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1130       } else {
1131          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1132          nir_print_instr(&instr->instr, stderr);
1133          fprintf(stderr, "\n");
1134       }
1135       break;
1136    }
1137    case nir_op_usub_borrow: {
1138       Temp src0 = get_alu_src(ctx, instr->src[0]);
1139       Temp src1 = get_alu_src(ctx, instr->src[1]);
1140       if (dst.regClass() == s1) {
1141          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1142          break;
1143       } else if (dst.regClass() == v1) {
1144          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1145          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1146          break;
1147       }
1148
1149       Temp src00 = bld.tmp(src0.type(), 1);
1150       Temp src01 = bld.tmp(dst.type(), 1);
1151       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1152       Temp src10 = bld.tmp(src1.type(), 1);
1153       Temp src11 = bld.tmp(dst.type(), 1);
1154       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1155       if (dst.regClass() == s2) {
1156          Temp borrow = bld.tmp(s1);
1157          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1158          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1159          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1160       } else if (dst.regClass() == v2) {
1161          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1162          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1163          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1164          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1165       } else {
1166          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1167          nir_print_instr(&instr->instr, stderr);
1168          fprintf(stderr, "\n");
1169       }
1170       break;
1171    }
1172    case nir_op_imul: {
1173       if (dst.regClass() == v1) {
1174          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1175                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1176       } else if (dst.regClass() == s1) {
1177          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1178       } else {
1179          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1180          nir_print_instr(&instr->instr, stderr);
1181          fprintf(stderr, "\n");
1182       }
1183       break;
1184    }
1185    case nir_op_umul_high: {
1186       if (dst.regClass() == v1) {
1187          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1188       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1189          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1190       } else if (dst.regClass() == s1) {
1191          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1192                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1193          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1194       } else {
1195          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1196          nir_print_instr(&instr->instr, stderr);
1197          fprintf(stderr, "\n");
1198       }
1199       break;
1200    }
1201    case nir_op_imul_high: {
1202       if (dst.regClass() == v1) {
1203          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1204       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1205          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1206       } else if (dst.regClass() == s1) {
1207          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1208                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1209          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1210       } else {
1211          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1212          nir_print_instr(&instr->instr, stderr);
1213          fprintf(stderr, "\n");
1214       }
1215       break;
1216    }
1217    case nir_op_fmul: {
1218       if (dst.size() == 1) {
1219          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1220       } else if (dst.size() == 2) {
1221          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1222                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1223       } else {
1224          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1225          nir_print_instr(&instr->instr, stderr);
1226          fprintf(stderr, "\n");
1227       }
1228       break;
1229    }
1230    case nir_op_fadd: {
1231       if (dst.size() == 1) {
1232          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1233       } else if (dst.size() == 2) {
1234          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1235                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1236       } else {
1237          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1238          nir_print_instr(&instr->instr, stderr);
1239          fprintf(stderr, "\n");
1240       }
1241       break;
1242    }
1243    case nir_op_fsub: {
1244       Temp src0 = get_alu_src(ctx, instr->src[0]);
1245       Temp src1 = get_alu_src(ctx, instr->src[1]);
1246       if (dst.size() == 1) {
1247          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1248             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1249          else
1250             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1251       } else if (dst.size() == 2) {
1252          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1253                                      get_alu_src(ctx, instr->src[0]),
1254                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1255          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1256          sub->neg[1] = true;
1257       } else {
1258          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1259          nir_print_instr(&instr->instr, stderr);
1260          fprintf(stderr, "\n");
1261       }
1262       break;
1263    }
1264    case nir_op_fmax: {
1265       if (dst.size() == 1) {
1266          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1267       } else if (dst.size() == 2) {
1268          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1269                   get_alu_src(ctx, instr->src[0]),
1270                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1271       } else {
1272          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1273          nir_print_instr(&instr->instr, stderr);
1274          fprintf(stderr, "\n");
1275       }
1276       break;
1277    }
1278    case nir_op_fmin: {
1279       if (dst.size() == 1) {
1280          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1281       } else if (dst.size() == 2) {
1282          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1283                   get_alu_src(ctx, instr->src[0]),
1284                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1285       } else {
1286          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1287          nir_print_instr(&instr->instr, stderr);
1288          fprintf(stderr, "\n");
1289       }
1290       break;
1291    }
1292    case nir_op_fmax3: {
1293       if (dst.size() == 1) {
1294          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1295       } else {
1296          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1297          nir_print_instr(&instr->instr, stderr);
1298          fprintf(stderr, "\n");
1299       }
1300       break;
1301    }
1302    case nir_op_fmin3: {
1303       if (dst.size() == 1) {
1304          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1305       } else {
1306          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1307          nir_print_instr(&instr->instr, stderr);
1308          fprintf(stderr, "\n");
1309       }
1310       break;
1311    }
1312    case nir_op_fmed3: {
1313       if (dst.size() == 1) {
1314          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1315       } else {
1316          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1317          nir_print_instr(&instr->instr, stderr);
1318          fprintf(stderr, "\n");
1319       }
1320       break;
1321    }
1322    case nir_op_umax3: {
1323       if (dst.size() == 1) {
1324          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1325       } else {
1326          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1327          nir_print_instr(&instr->instr, stderr);
1328          fprintf(stderr, "\n");
1329       }
1330       break;
1331    }
1332    case nir_op_umin3: {
1333       if (dst.size() == 1) {
1334          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1335       } else {
1336          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1337          nir_print_instr(&instr->instr, stderr);
1338          fprintf(stderr, "\n");
1339       }
1340       break;
1341    }
1342    case nir_op_umed3: {
1343       if (dst.size() == 1) {
1344          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1345       } else {
1346          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1347          nir_print_instr(&instr->instr, stderr);
1348          fprintf(stderr, "\n");
1349       }
1350       break;
1351    }
1352    case nir_op_imax3: {
1353       if (dst.size() == 1) {
1354          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1355       } else {
1356          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1357          nir_print_instr(&instr->instr, stderr);
1358          fprintf(stderr, "\n");
1359       }
1360       break;
1361    }
1362    case nir_op_imin3: {
1363       if (dst.size() == 1) {
1364          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1365       } else {
1366          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1367          nir_print_instr(&instr->instr, stderr);
1368          fprintf(stderr, "\n");
1369       }
1370       break;
1371    }
1372    case nir_op_imed3: {
1373       if (dst.size() == 1) {
1374          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1375       } else {
1376          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1377          nir_print_instr(&instr->instr, stderr);
1378          fprintf(stderr, "\n");
1379       }
1380       break;
1381    }
1382    case nir_op_cube_face_coord: {
1383       Temp in = get_alu_src(ctx, instr->src[0], 3);
1384       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1385                       emit_extract_vector(ctx, in, 1, v1),
1386                       emit_extract_vector(ctx, in, 2, v1) };
1387       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1388       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1389       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1390       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1391       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1392       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1393       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1394       break;
1395    }
1396    case nir_op_cube_face_index: {
1397       Temp in = get_alu_src(ctx, instr->src[0], 3);
1398       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1399                       emit_extract_vector(ctx, in, 1, v1),
1400                       emit_extract_vector(ctx, in, 2, v1) };
1401       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1402       break;
1403    }
1404    case nir_op_bcsel: {
1405       emit_bcsel(ctx, instr, dst);
1406       break;
1407    }
1408    case nir_op_frsq: {
1409       if (dst.size() == 1) {
1410          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1411       } else if (dst.size() == 2) {
1412          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1413       } else {
1414          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1415          nir_print_instr(&instr->instr, stderr);
1416          fprintf(stderr, "\n");
1417       }
1418       break;
1419    }
1420    case nir_op_fneg: {
1421       Temp src = get_alu_src(ctx, instr->src[0]);
1422       if (dst.size() == 1) {
1423          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1424       } else if (dst.size() == 2) {
1425          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1426          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1427          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1428          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1429       } else {
1430          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1431          nir_print_instr(&instr->instr, stderr);
1432          fprintf(stderr, "\n");
1433       }
1434       break;
1435    }
1436    case nir_op_fabs: {
1437       Temp src = get_alu_src(ctx, instr->src[0]);
1438       if (dst.size() == 1) {
1439          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1440       } else if (dst.size() == 2) {
1441          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1442          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1443          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1444          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1445       } else {
1446          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1447          nir_print_instr(&instr->instr, stderr);
1448          fprintf(stderr, "\n");
1449       }
1450       break;
1451    }
1452    case nir_op_fsat: {
1453       Temp src = get_alu_src(ctx, instr->src[0]);
1454       if (dst.size() == 1) {
1455          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1456       } else if (dst.size() == 2) {
1457          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1458          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1459          vop3->clamp = true;
1460       } else {
1461          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1462          nir_print_instr(&instr->instr, stderr);
1463          fprintf(stderr, "\n");
1464       }
1465       break;
1466    }
1467    case nir_op_flog2: {
1468       if (dst.size() == 1) {
1469          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1470       } else {
1471          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1472          nir_print_instr(&instr->instr, stderr);
1473          fprintf(stderr, "\n");
1474       }
1475       break;
1476    }
1477    case nir_op_frcp: {
1478       if (dst.size() == 1) {
1479          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1480       } else if (dst.size() == 2) {
1481          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1482       } else {
1483          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1484          nir_print_instr(&instr->instr, stderr);
1485          fprintf(stderr, "\n");
1486       }
1487       break;
1488    }
1489    case nir_op_fexp2: {
1490       if (dst.size() == 1) {
1491          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fsqrt: {
1500       if (dst.size() == 1) {
1501          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1502       } else if (dst.size() == 2) {
1503          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1504       } else {
1505          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1506          nir_print_instr(&instr->instr, stderr);
1507          fprintf(stderr, "\n");
1508       }
1509       break;
1510    }
1511    case nir_op_ffract: {
1512       if (dst.size() == 1) {
1513          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1514       } else if (dst.size() == 2) {
1515          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1516       } else {
1517          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1518          nir_print_instr(&instr->instr, stderr);
1519          fprintf(stderr, "\n");
1520       }
1521       break;
1522    }
1523    case nir_op_ffloor: {
1524       if (dst.size() == 1) {
1525          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1526       } else if (dst.size() == 2) {
1527          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1528       } else {
1529          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1530          nir_print_instr(&instr->instr, stderr);
1531          fprintf(stderr, "\n");
1532       }
1533       break;
1534    }
1535    case nir_op_fceil: {
1536       if (dst.size() == 1) {
1537          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1538       } else if (dst.size() == 2) {
1539          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1540       } else {
1541          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1542          nir_print_instr(&instr->instr, stderr);
1543          fprintf(stderr, "\n");
1544       }
1545       break;
1546    }
1547    case nir_op_ftrunc: {
1548       if (dst.size() == 1) {
1549          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1550       } else if (dst.size() == 2) {
1551          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1552       } else {
1553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1554          nir_print_instr(&instr->instr, stderr);
1555          fprintf(stderr, "\n");
1556       }
1557       break;
1558    }
1559    case nir_op_fround_even: {
1560       if (dst.size() == 1) {
1561          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1562       } else if (dst.size() == 2) {
1563          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1564       } else {
1565          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1566          nir_print_instr(&instr->instr, stderr);
1567          fprintf(stderr, "\n");
1568       }
1569       break;
1570    }
1571    case nir_op_fsin:
1572    case nir_op_fcos: {
1573       Temp src = get_alu_src(ctx, instr->src[0]);
1574       aco_ptr<Instruction> norm;
1575       if (dst.size() == 1) {
1576          Temp tmp;
1577          Operand half_pi(0x3e22f983u);
1578          if (src.type() == RegType::sgpr)
1579             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1580          else
1581             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1582
1583          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1584          if (ctx->options->chip_class < GFX9)
1585             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1586
1587          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1588          bld.vop1(opcode, Definition(dst), tmp);
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_ldexp: {
1597       if (dst.size() == 1) {
1598          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1599                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1600                   get_alu_src(ctx, instr->src[1]));
1601       } else if (dst.size() == 2) {
1602          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1603                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1604                   get_alu_src(ctx, instr->src[1]));
1605       } else {
1606          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1607          nir_print_instr(&instr->instr, stderr);
1608          fprintf(stderr, "\n");
1609       }
1610       break;
1611    }
1612    case nir_op_frexp_sig: {
1613       if (dst.size() == 1) {
1614          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1615                   get_alu_src(ctx, instr->src[0]));
1616       } else if (dst.size() == 2) {
1617          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1618                   get_alu_src(ctx, instr->src[0]));
1619       } else {
1620          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1621          nir_print_instr(&instr->instr, stderr);
1622          fprintf(stderr, "\n");
1623       }
1624       break;
1625    }
1626    case nir_op_frexp_exp: {
1627       if (instr->src[0].src.ssa->bit_size == 32) {
1628          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1629                   get_alu_src(ctx, instr->src[0]));
1630       } else if (instr->src[0].src.ssa->bit_size == 64) {
1631          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1632                   get_alu_src(ctx, instr->src[0]));
1633       } else {
1634          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1635          nir_print_instr(&instr->instr, stderr);
1636          fprintf(stderr, "\n");
1637       }
1638       break;
1639    }
1640    case nir_op_fsign: {
1641       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1642       if (dst.size() == 1) {
1643          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1644          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1645          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1646          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1647       } else if (dst.size() == 2) {
1648          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1649          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1650          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, src, cond);
1651
1652          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1653          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1654          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1655
1656          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1657       } else {
1658          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1659          nir_print_instr(&instr->instr, stderr);
1660          fprintf(stderr, "\n");
1661       }
1662       break;
1663    }
1664    case nir_op_f2f32: {
1665       if (instr->src[0].src.ssa->bit_size == 64) {
1666          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_f2f64: {
1675       if (instr->src[0].src.ssa->bit_size == 32) {
1676          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1677       } else {
1678          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1679          nir_print_instr(&instr->instr, stderr);
1680          fprintf(stderr, "\n");
1681       }
1682       break;
1683    }
1684    case nir_op_i2f32: {
1685       assert(dst.size() == 1);
1686       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1687       break;
1688    }
1689    case nir_op_i2f64: {
1690       if (instr->src[0].src.ssa->bit_size == 32) {
1691          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1692       } else if (instr->src[0].src.ssa->bit_size == 64) {
1693          Temp src = get_alu_src(ctx, instr->src[0]);
1694          RegClass rc = RegClass(src.type(), 1);
1695          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1696          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1697          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1698          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1699          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1700          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1701
1702       } else {
1703          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1704          nir_print_instr(&instr->instr, stderr);
1705          fprintf(stderr, "\n");
1706       }
1707       break;
1708    }
1709    case nir_op_u2f32: {
1710       assert(dst.size() == 1);
1711       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1712       break;
1713    }
1714    case nir_op_u2f64: {
1715       if (instr->src[0].src.ssa->bit_size == 32) {
1716          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1717       } else if (instr->src[0].src.ssa->bit_size == 64) {
1718          Temp src = get_alu_src(ctx, instr->src[0]);
1719          RegClass rc = RegClass(src.type(), 1);
1720          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1721          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1722          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1723          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1724          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1725          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1726       } else {
1727          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1728          nir_print_instr(&instr->instr, stderr);
1729          fprintf(stderr, "\n");
1730       }
1731       break;
1732    }
1733    case nir_op_f2i32: {
1734       Temp src = get_alu_src(ctx, instr->src[0]);
1735       if (instr->src[0].src.ssa->bit_size == 32) {
1736          if (dst.type() == RegType::vgpr)
1737             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1738          else
1739             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1740                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1741
1742       } else if (instr->src[0].src.ssa->bit_size == 64) {
1743          if (dst.type() == RegType::vgpr)
1744             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1745          else
1746             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1747                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1748
1749       } else {
1750          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1751          nir_print_instr(&instr->instr, stderr);
1752          fprintf(stderr, "\n");
1753       }
1754       break;
1755    }
1756    case nir_op_f2u32: {
1757       Temp src = get_alu_src(ctx, instr->src[0]);
1758       if (instr->src[0].src.ssa->bit_size == 32) {
1759          if (dst.type() == RegType::vgpr)
1760             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1761          else
1762             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1763                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1764
1765       } else if (instr->src[0].src.ssa->bit_size == 64) {
1766          if (dst.type() == RegType::vgpr)
1767             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1768          else
1769             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1770                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1771
1772       } else {
1773          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1774          nir_print_instr(&instr->instr, stderr);
1775          fprintf(stderr, "\n");
1776       }
1777       break;
1778    }
1779    case nir_op_f2i64: {
1780       Temp src = get_alu_src(ctx, instr->src[0]);
1781       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1782          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1783          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1784          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1785          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1786          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1787          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1788          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1789          Temp new_exponent = bld.tmp(v1);
1790          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1791          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1792          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1793          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1794          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1795          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1796          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1797          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1798          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1799          Temp new_lower = bld.tmp(v1);
1800          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1801          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1802          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1803
1804       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1805          if (src.type() == RegType::vgpr)
1806             src = bld.as_uniform(src);
1807          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1808          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1809          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1810          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1811          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1812          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1813          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1814          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1815          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1816          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1817          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1818          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1819          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1820          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1821          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1822          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1823          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1824          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1825          Temp borrow = bld.tmp(s1);
1826          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1827          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1828          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1829
1830       } else if (instr->src[0].src.ssa->bit_size == 64) {
1831          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1832          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1833          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1834          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1835          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1836          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1837          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1838          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1839          if (dst.type() == RegType::sgpr) {
1840             lower = bld.as_uniform(lower);
1841             upper = bld.as_uniform(upper);
1842          }
1843          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1844
1845       } else {
1846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1847          nir_print_instr(&instr->instr, stderr);
1848          fprintf(stderr, "\n");
1849       }
1850       break;
1851    }
1852    case nir_op_f2u64: {
1853       Temp src = get_alu_src(ctx, instr->src[0]);
1854       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1855          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1856          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1857          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1858          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1859          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1860          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1861          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1862          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1863          Temp new_exponent = bld.tmp(v1);
1864          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1865          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1866          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1867          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1868          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1869          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1870          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1871          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1872          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1873
1874       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1875          if (src.type() == RegType::vgpr)
1876             src = bld.as_uniform(src);
1877          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1878          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1879          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1880          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1881          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1882          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1883          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1884          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1885          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1886          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1887          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1888          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1889          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1890          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1891          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1892          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1893          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1894          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1895
1896       } else if (instr->src[0].src.ssa->bit_size == 64) {
1897          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1898          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1899          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1900          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1901          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1902          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1903          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1904          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1905          if (dst.type() == RegType::sgpr) {
1906             lower = bld.as_uniform(lower);
1907             upper = bld.as_uniform(upper);
1908          }
1909          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1910
1911       } else {
1912          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1913          nir_print_instr(&instr->instr, stderr);
1914          fprintf(stderr, "\n");
1915       }
1916       break;
1917    }
1918    case nir_op_b2f32: {
1919       Temp src = get_alu_src(ctx, instr->src[0]);
1920       if (dst.regClass() == s1) {
1921          src = as_uniform_bool(ctx, src);
1922          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1923       } else if (dst.regClass() == v1) {
1924          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u),
1925                       as_divergent_bool(ctx, src, true));
1926       } else {
1927          unreachable("Wrong destination register class for nir_op_b2f32.");
1928       }
1929       break;
1930    }
1931    case nir_op_b2f64: {
1932       Temp src = get_alu_src(ctx, instr->src[0]);
1933       if (dst.regClass() == s2) {
1934          src = as_uniform_bool(ctx, src);
1935          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1936       } else if (dst.regClass() == v2) {
1937          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1938          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one,
1939                       as_divergent_bool(ctx, src, true));
1940          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1941       } else {
1942          unreachable("Wrong destination register class for nir_op_b2f64.");
1943       }
1944       break;
1945    }
1946    case nir_op_i2i32: {
1947       Temp src = get_alu_src(ctx, instr->src[0]);
1948       if (instr->src[0].src.ssa->bit_size == 64) {
1949          /* we can actually just say dst = src, as it would map the lower register */
1950          emit_extract_vector(ctx, src, 0, dst);
1951       } else {
1952          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1953          nir_print_instr(&instr->instr, stderr);
1954          fprintf(stderr, "\n");
1955       }
1956       break;
1957    }
1958    case nir_op_u2u32: {
1959       Temp src = get_alu_src(ctx, instr->src[0]);
1960       if (instr->src[0].src.ssa->bit_size == 16) {
1961          if (dst.regClass() == s1) {
1962             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1963          } else {
1964             // TODO: do better with SDWA
1965             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1966          }
1967       } else if (instr->src[0].src.ssa->bit_size == 64) {
1968          /* we can actually just say dst = src, as it would map the lower register */
1969          emit_extract_vector(ctx, src, 0, dst);
1970       } else {
1971          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1972          nir_print_instr(&instr->instr, stderr);
1973          fprintf(stderr, "\n");
1974       }
1975       break;
1976    }
1977    case nir_op_i2i64: {
1978       Temp src = get_alu_src(ctx, instr->src[0]);
1979       if (instr->src[0].src.ssa->bit_size == 32) {
1980          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1981       } else {
1982          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1983          nir_print_instr(&instr->instr, stderr);
1984          fprintf(stderr, "\n");
1985       }
1986       break;
1987    }
1988    case nir_op_u2u64: {
1989       Temp src = get_alu_src(ctx, instr->src[0]);
1990       if (instr->src[0].src.ssa->bit_size == 32) {
1991          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1992       } else {
1993          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1994          nir_print_instr(&instr->instr, stderr);
1995          fprintf(stderr, "\n");
1996       }
1997       break;
1998    }
1999    case nir_op_b2i32: {
2000       Temp src = get_alu_src(ctx, instr->src[0]);
2001       if (dst.regClass() == s1) {
2002          if (src.regClass() == s1) {
2003             bld.copy(Definition(dst), src);
2004          } else {
2005             // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2006             assert(src.regClass() == s2);
2007             bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(dst)), Operand(0u), src);
2008          }
2009       } else {
2010          assert(dst.regClass() == v1 && src.regClass() == s2);
2011          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2012       }
2013       break;
2014    }
2015    case nir_op_i2b1: {
2016       Temp src = get_alu_src(ctx, instr->src[0]);
2017       if (dst.regClass() == s2) {
2018          assert(src.regClass() == v1 || src.regClass() == v2);
2019          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2020                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2021       } else {
2022          assert(src.regClass() == s1 && dst.regClass() == s1);
2023          bld.sopc(aco_opcode::s_cmp_lg_u32, bld.scc(Definition(dst)), Operand(0u), src);
2024       }
2025       break;
2026    }
2027    case nir_op_pack_64_2x32_split: {
2028       Temp src0 = get_alu_src(ctx, instr->src[0]);
2029       Temp src1 = get_alu_src(ctx, instr->src[1]);
2030
2031       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2032       break;
2033    }
2034    case nir_op_unpack_64_2x32_split_x:
2035       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2036       break;
2037    case nir_op_unpack_64_2x32_split_y:
2038       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2039       break;
2040    case nir_op_pack_half_2x16: {
2041       Temp src = get_alu_src(ctx, instr->src[0], 2);
2042
2043       if (dst.regClass() == v1) {
2044          Temp src0 = bld.tmp(v1);
2045          Temp src1 = bld.tmp(v1);
2046          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2047          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2048
2049       } else {
2050          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2051          nir_print_instr(&instr->instr, stderr);
2052          fprintf(stderr, "\n");
2053       }
2054       break;
2055    }
2056    case nir_op_unpack_half_2x16_split_x: {
2057       if (dst.regClass() == v1) {
2058          Builder bld(ctx->program, ctx->block);
2059          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2060       } else {
2061          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2062          nir_print_instr(&instr->instr, stderr);
2063          fprintf(stderr, "\n");
2064       }
2065       break;
2066    }
2067    case nir_op_unpack_half_2x16_split_y: {
2068       if (dst.regClass() == v1) {
2069          Builder bld(ctx->program, ctx->block);
2070          /* TODO: use SDWA here */
2071          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2072                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2073       } else {
2074          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2075          nir_print_instr(&instr->instr, stderr);
2076          fprintf(stderr, "\n");
2077       }
2078       break;
2079    }
2080    case nir_op_fquantize2f16: {
2081       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2082
2083       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2084
2085       Temp cmp_res = bld.tmp(s2);
2086       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2087
2088       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2089
2090       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2091       break;
2092    }
2093    case nir_op_bfm: {
2094       Temp bits = get_alu_src(ctx, instr->src[0]);
2095       Temp offset = get_alu_src(ctx, instr->src[1]);
2096
2097       if (dst.regClass() == s1) {
2098          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2099       } else if (dst.regClass() == v1) {
2100          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2101       } else {
2102          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2103          nir_print_instr(&instr->instr, stderr);
2104          fprintf(stderr, "\n");
2105       }
2106       break;
2107    }
2108    case nir_op_bitfield_select: {
2109       /* (mask & insert) | (~mask & base) */
2110       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2111       Temp insert = get_alu_src(ctx, instr->src[1]);
2112       Temp base = get_alu_src(ctx, instr->src[2]);
2113
2114       /* dst = (insert & bitmask) | (base & ~bitmask) */
2115       if (dst.regClass() == s1) {
2116          aco_ptr<Instruction> sop2;
2117          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2118          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2119          Operand lhs;
2120          if (const_insert && const_bitmask) {
2121             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2122          } else {
2123             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2124             lhs = Operand(insert);
2125          }
2126
2127          Operand rhs;
2128          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2129          if (const_base && const_bitmask) {
2130             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2131          } else {
2132             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2133             rhs = Operand(base);
2134          }
2135
2136          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2137
2138       } else if (dst.regClass() == v1) {
2139          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2140             base = as_vgpr(ctx, base);
2141          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2142             insert = as_vgpr(ctx, insert);
2143
2144          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2145
2146       } else {
2147          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2148          nir_print_instr(&instr->instr, stderr);
2149          fprintf(stderr, "\n");
2150       }
2151       break;
2152    }
2153    case nir_op_ubfe:
2154    case nir_op_ibfe: {
2155       Temp base = get_alu_src(ctx, instr->src[0]);
2156       Temp offset = get_alu_src(ctx, instr->src[1]);
2157       Temp bits = get_alu_src(ctx, instr->src[2]);
2158
2159       if (dst.type() == RegType::sgpr) {
2160          Operand extract;
2161          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2162          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2163          if (const_offset && const_bits) {
2164             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2165             extract = Operand(const_extract);
2166          } else {
2167             Operand width;
2168             if (const_bits) {
2169                width = Operand(const_bits->u32 << 16);
2170             } else {
2171                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2172             }
2173             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2174          }
2175
2176          aco_opcode opcode;
2177          if (dst.regClass() == s1) {
2178             if (instr->op == nir_op_ubfe)
2179                opcode = aco_opcode::s_bfe_u32;
2180             else
2181                opcode = aco_opcode::s_bfe_i32;
2182          } else if (dst.regClass() == s2) {
2183             if (instr->op == nir_op_ubfe)
2184                opcode = aco_opcode::s_bfe_u64;
2185             else
2186                opcode = aco_opcode::s_bfe_i64;
2187          } else {
2188             unreachable("Unsupported BFE bit size");
2189          }
2190
2191          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2192
2193       } else {
2194          aco_opcode opcode;
2195          if (dst.regClass() == v1) {
2196             if (instr->op == nir_op_ubfe)
2197                opcode = aco_opcode::v_bfe_u32;
2198             else
2199                opcode = aco_opcode::v_bfe_i32;
2200          } else {
2201             unreachable("Unsupported BFE bit size");
2202          }
2203
2204          emit_vop3a_instruction(ctx, instr, opcode, dst);
2205       }
2206       break;
2207    }
2208    case nir_op_bit_count: {
2209       Temp src = get_alu_src(ctx, instr->src[0]);
2210       if (src.regClass() == s1) {
2211          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2212       } else if (src.regClass() == v1) {
2213          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2214       } else if (src.regClass() == v2) {
2215          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2216                   emit_extract_vector(ctx, src, 1, v1),
2217                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2218                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2219       } else if (src.regClass() == s2) {
2220          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2221       } else {
2222          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2223          nir_print_instr(&instr->instr, stderr);
2224          fprintf(stderr, "\n");
2225       }
2226       break;
2227    }
2228    case nir_op_flt: {
2229       if (instr->src[0].src.ssa->bit_size == 32)
2230          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f32, dst);
2231       else if (instr->src[0].src.ssa->bit_size == 64)
2232          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f64, dst);
2233       break;
2234    }
2235    case nir_op_fge: {
2236       if (instr->src[0].src.ssa->bit_size == 32)
2237          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f32, dst);
2238       else if (instr->src[0].src.ssa->bit_size == 64)
2239          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f64, dst);
2240       break;
2241    }
2242    case nir_op_feq: {
2243       if (instr->src[0].src.ssa->bit_size == 32)
2244          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f32, dst);
2245       else if (instr->src[0].src.ssa->bit_size == 64)
2246          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f64, dst);
2247       break;
2248    }
2249    case nir_op_fne: {
2250       if (instr->src[0].src.ssa->bit_size == 32)
2251          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f32, dst);
2252       else if (instr->src[0].src.ssa->bit_size == 64)
2253          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f64, dst);
2254       break;
2255    }
2256    case nir_op_ilt: {
2257       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2258          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i32, dst);
2259       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2260          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_i32, dst);
2261       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2262          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i64, dst);
2263       break;
2264    }
2265    case nir_op_ige: {
2266       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2267          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i32, dst);
2268       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2269          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_i32, dst);
2270       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2271          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i64, dst);
2272       break;
2273    }
2274    case nir_op_ieq: {
2275       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2276          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i32, dst);
2277       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2278          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_i32, dst);
2279       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2280          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i64, dst);
2281       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2282          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_u64, dst);
2283       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2284          Temp src0 = get_alu_src(ctx, instr->src[0]);
2285          Temp src1 = get_alu_src(ctx, instr->src[1]);
2286          bld.sopc(aco_opcode::s_cmp_eq_i32, bld.scc(Definition(dst)),
2287                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2288       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2289          Temp src0 = get_alu_src(ctx, instr->src[0]);
2290          Temp src1 = get_alu_src(ctx, instr->src[1]);
2291          bld.sop2(aco_opcode::s_xnor_b64, Definition(dst), bld.def(s1, scc),
2292                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2293       } else {
2294          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2295          nir_print_instr(&instr->instr, stderr);
2296          fprintf(stderr, "\n");
2297       }
2298       break;
2299    }
2300    case nir_op_ine: {
2301       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2302          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i32, dst);
2303       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2304          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i64, dst);
2305       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2306          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_i32, dst);
2307       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2308          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_u64, dst);
2309       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2310          Temp src0 = get_alu_src(ctx, instr->src[0]);
2311          Temp src1 = get_alu_src(ctx, instr->src[1]);
2312          bld.sopc(aco_opcode::s_cmp_lg_i32, bld.scc(Definition(dst)),
2313                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2314       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2315          Temp src0 = get_alu_src(ctx, instr->src[0]);
2316          Temp src1 = get_alu_src(ctx, instr->src[1]);
2317          bld.sop2(aco_opcode::s_xor_b64, Definition(dst), bld.def(s1, scc),
2318                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2319       } else {
2320          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2321          nir_print_instr(&instr->instr, stderr);
2322          fprintf(stderr, "\n");
2323       }
2324       break;
2325    }
2326    case nir_op_ult: {
2327       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2328          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u32, dst);
2329       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2330          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_u32, dst);
2331       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2332          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u64, dst);
2333       break;
2334    }
2335    case nir_op_uge: {
2336       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2337          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u32, dst);
2338       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2339          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_u32, dst);
2340       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2341          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u64, dst);
2342       break;
2343    }
2344    case nir_op_fddx:
2345    case nir_op_fddy:
2346    case nir_op_fddx_fine:
2347    case nir_op_fddy_fine:
2348    case nir_op_fddx_coarse:
2349    case nir_op_fddy_coarse: {
2350       Definition tl = bld.def(v1);
2351       uint16_t dpp_ctrl;
2352       if (instr->op == nir_op_fddx_fine) {
2353          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2354          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2355       } else if (instr->op == nir_op_fddy_fine) {
2356          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2357          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2358       } else {
2359          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2360          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2361             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2362          else
2363             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2364       }
2365
2366       Definition tmp = bld.def(v1);
2367       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2368       emit_wqm(ctx, tmp.getTemp(), dst, true);
2369       break;
2370    }
2371    default:
2372       fprintf(stderr, "Unknown NIR ALU instr: ");
2373       nir_print_instr(&instr->instr, stderr);
2374       fprintf(stderr, "\n");
2375    }
2376 }
2377
2378 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2379 {
2380    Temp dst = get_ssa_temp(ctx, &instr->def);
2381
2382    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2383    // which get truncated the lsb if double and msb if int
2384    // for now, we only use s_mov_b64 with 64bit inline constants
2385    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2386    assert(dst.type() == RegType::sgpr);
2387
2388    if (dst.size() == 1)
2389    {
2390       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(instr->value[0].u32));
2391    } else {
2392       assert(dst.size() != 1);
2393       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2394       if (instr->def.bit_size == 64)
2395          for (unsigned i = 0; i < dst.size(); i++)
2396             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2397       else {
2398          for (unsigned i = 0; i < dst.size(); i++)
2399             vec->operands[i] = Operand{instr->value[i].u32};
2400       }
2401       vec->definitions[0] = Definition(dst);
2402       ctx->block->instructions.emplace_back(std::move(vec));
2403    }
2404 }
2405
2406 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2407 {
2408    uint32_t new_mask = 0;
2409    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2410       if (mask & (1u << i))
2411          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2412    return new_mask;
2413 }
2414
2415 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2416 {
2417    /* This wouldn't work inside control flow or with indirect offsets but
2418     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2419
2420    unsigned write_mask = nir_intrinsic_write_mask(instr);
2421    unsigned component = nir_intrinsic_component(instr);
2422    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2423    unsigned idx = nir_intrinsic_base(instr) + component;
2424
2425    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2426    if (off_instr->type != nir_instr_type_load_const) {
2427       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2428       nir_print_instr(off_instr, stderr);
2429       fprintf(stderr, "\n");
2430    }
2431    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2432
2433    if (instr->src[0].ssa->bit_size == 64)
2434       write_mask = widen_mask(write_mask, 2);
2435
2436    for (unsigned i = 0; i < 8; ++i) {
2437       if (write_mask & (1 << i)) {
2438          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2439          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2440       }
2441       idx++;
2442    }
2443 }
2444
2445 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2446 {
2447    unsigned write_mask = nir_intrinsic_write_mask(instr);
2448    Operand values[4];
2449    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2450    for (unsigned i = 0; i < 4; ++i) {
2451       if (write_mask & (1 << i)) {
2452          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2453          values[i] = Operand(tmp);
2454       } else {
2455          values[i] = Operand(v1);
2456       }
2457    }
2458
2459    unsigned index = nir_intrinsic_base(instr) / 4;
2460    unsigned target, col_format;
2461    unsigned enabled_channels = 0xF;
2462    aco_opcode compr_op = (aco_opcode)0;
2463
2464    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2465    assert(offset && "Non-const offsets on exports not yet supported");
2466    index += offset->u32;
2467
2468    assert(index != FRAG_RESULT_COLOR);
2469
2470    /* Unlike vertex shader exports, it's fine to use multiple exports to
2471     * export separate channels of one target. So shaders which export both
2472     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2473     * TODO: combine the exports in those cases and create better code
2474     */
2475
2476    if (index == FRAG_RESULT_SAMPLE_MASK) {
2477
2478       if (ctx->program->info->ps.writes_z) {
2479          target = V_008DFC_SQ_EXP_MRTZ;
2480          enabled_channels = 0x4;
2481          col_format = (unsigned) -1;
2482
2483          values[2] = values[0];
2484          values[0] = Operand(v1);
2485       } else {
2486          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2487          exp->valid_mask = false;
2488          exp->done = false;
2489          exp->compressed = true;
2490          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2491          exp->enabled_mask = 0xc;
2492          for (int i = 0; i < 4; i++)
2493             exp->operands[i] = Operand(v1);
2494          exp->operands[1] = Operand(values[0]);
2495          ctx->block->instructions.emplace_back(std::move(exp));
2496          return;
2497       }
2498
2499    } else if (index == FRAG_RESULT_DEPTH) {
2500
2501       target = V_008DFC_SQ_EXP_MRTZ;
2502       enabled_channels = 0x1;
2503       col_format = (unsigned) -1;
2504
2505    } else if (index == FRAG_RESULT_STENCIL) {
2506
2507       if (ctx->program->info->ps.writes_z) {
2508          target = V_008DFC_SQ_EXP_MRTZ;
2509          enabled_channels = 0x2;
2510          col_format = (unsigned) -1;
2511
2512          values[1] = values[0];
2513          values[0] = Operand(v1);
2514       } else {
2515          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2516          shift->operands[0] = Operand((uint32_t) 16);
2517          shift->operands[1] = values[0];
2518          Temp tmp = {ctx->program->allocateId(), v1};
2519          shift->definitions[0] = Definition(tmp);
2520          ctx->block->instructions.emplace_back(std::move(shift));
2521
2522          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2523          exp->valid_mask = false;
2524          exp->done = false;
2525          exp->compressed = true;
2526          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2527          exp->enabled_mask = 0x3;
2528          exp->operands[0] = Operand(tmp);
2529          for (int i = 1; i < 4; i++)
2530             exp->operands[i] = Operand(v1);
2531          ctx->block->instructions.emplace_back(std::move(exp));
2532          return;
2533       }
2534
2535    } else {
2536       index -= FRAG_RESULT_DATA0;
2537       target = V_008DFC_SQ_EXP_MRT + index;
2538       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2539    }
2540    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2541    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2542    assert(!is_int8 && !is_int10);
2543
2544    switch (col_format)
2545    {
2546    case V_028714_SPI_SHADER_ZERO:
2547       enabled_channels = 0; /* writemask */
2548       target = V_008DFC_SQ_EXP_NULL;
2549       break;
2550
2551    case V_028714_SPI_SHADER_32_R:
2552       enabled_channels = 1;
2553       break;
2554
2555    case V_028714_SPI_SHADER_32_GR:
2556       enabled_channels = 0x3;
2557       break;
2558
2559    case V_028714_SPI_SHADER_32_AR:
2560       if (ctx->options->chip_class >= GFX10) {
2561          /* Special case: on GFX10, the outputs are different for 32_AR */
2562          enabled_channels = 0x3;
2563          values[1] = values[3];
2564       } else {
2565          enabled_channels = 0x9;
2566       }
2567       break;
2568
2569    case V_028714_SPI_SHADER_FP16_ABGR:
2570       enabled_channels = 0x5;
2571       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2572       break;
2573
2574    case V_028714_SPI_SHADER_UNORM16_ABGR:
2575       enabled_channels = 0x5;
2576       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2577       break;
2578
2579    case V_028714_SPI_SHADER_SNORM16_ABGR:
2580       enabled_channels = 0x5;
2581       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2582       break;
2583
2584    case V_028714_SPI_SHADER_UINT16_ABGR:
2585       enabled_channels = 0x5;
2586       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2587       break;
2588
2589    case V_028714_SPI_SHADER_SINT16_ABGR:
2590       enabled_channels = 0x5;
2591       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2592       break;
2593
2594    case V_028714_SPI_SHADER_32_ABGR:
2595       enabled_channels = 0xF;
2596       break;
2597
2598    default:
2599       break;
2600    }
2601
2602    if (target == V_008DFC_SQ_EXP_NULL)
2603       return;
2604
2605    if ((bool)compr_op)
2606    {
2607       for (int i = 0; i < 2; i++)
2608       {
2609          /* check if at least one of the values to be compressed is enabled */
2610          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2611          if (enabled) {
2612             enabled_channels |= enabled << (i*2);
2613             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2614             Temp tmp{ctx->program->allocateId(), v1};
2615             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2616             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2617             compr->definitions[0] = Definition(tmp);
2618             values[i] = Operand(tmp);
2619             ctx->block->instructions.emplace_back(std::move(compr));
2620          } else {
2621             values[i] = Operand(v1);
2622          }
2623       }
2624    }
2625
2626    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2627    exp->valid_mask = false;
2628    exp->done = false;
2629    exp->compressed = (bool) compr_op;
2630    exp->dest = target;
2631    exp->enabled_mask = enabled_channels;
2632    if ((bool) compr_op) {
2633       for (int i = 0; i < 2; i++)
2634          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2635       exp->operands[2] = Operand(v1);
2636       exp->operands[3] = Operand(v1);
2637    } else {
2638       for (int i = 0; i < 4; i++)
2639          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2640    }
2641
2642    ctx->block->instructions.emplace_back(std::move(exp));
2643 }
2644
2645 Operand load_lds_size_m0(isel_context *ctx)
2646 {
2647    /* TODO: m0 does not need to be initialized on GFX9+ */
2648    Builder bld(ctx->program, ctx->block);
2649    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2650 }
2651
2652 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2653               Temp address, unsigned base_offset, unsigned align)
2654 {
2655    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2656
2657    Builder bld(ctx->program, ctx->block);
2658
2659    Operand m = load_lds_size_m0(ctx);
2660
2661    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2662    unsigned bytes_read = 0;
2663    unsigned result_size = 0;
2664    unsigned total_bytes = num_components * elem_size_bytes;
2665    std::array<Temp, 4> result;
2666
2667    while (bytes_read < total_bytes) {
2668       unsigned todo = total_bytes - bytes_read;
2669       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2670       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2671
2672       aco_opcode op = aco_opcode::last_opcode;
2673       bool read2 = false;
2674       if (todo >= 16 && aligned16) {
2675          op = aco_opcode::ds_read_b128;
2676          todo = 16;
2677       } else if (todo >= 16 && aligned8) {
2678          op = aco_opcode::ds_read2_b64;
2679          read2 = true;
2680          todo = 16;
2681       } else if (todo >= 12 && aligned16) {
2682          op = aco_opcode::ds_read_b96;
2683          todo = 12;
2684       } else if (todo >= 8 && aligned8) {
2685          op = aco_opcode::ds_read_b64;
2686          todo = 8;
2687       } else if (todo >= 8) {
2688          op = aco_opcode::ds_read2_b32;
2689          read2 = true;
2690          todo = 8;
2691       } else if (todo >= 4) {
2692          op = aco_opcode::ds_read_b32;
2693          todo = 4;
2694       } else {
2695          assert(false);
2696       }
2697       assert(todo % elem_size_bytes == 0);
2698       unsigned num_elements = todo / elem_size_bytes;
2699       unsigned offset = base_offset + bytes_read;
2700       unsigned max_offset = read2 ? 1019 : 65535;
2701
2702       Temp address_offset = address;
2703       if (offset > max_offset) {
2704          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2705          offset = bytes_read;
2706       }
2707       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2708
2709       Temp res;
2710       if (num_components == 1 && dst.type() == RegType::vgpr)
2711          res = dst;
2712       else
2713          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2714
2715       if (read2)
2716          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2717       else
2718          res = bld.ds(op, Definition(res), address_offset, m, offset);
2719
2720       if (num_components == 1) {
2721          assert(todo == total_bytes);
2722          if (dst.type() == RegType::sgpr)
2723             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2724          return;
2725       }
2726
2727       if (dst.type() == RegType::sgpr)
2728          res = bld.as_uniform(res);
2729
2730       if (num_elements == 1) {
2731          result[result_size++] = res;
2732       } else {
2733          assert(res != dst && res.size() % num_elements == 0);
2734          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2735          split->operands[0] = Operand(res);
2736          for (unsigned i = 0; i < num_elements; i++)
2737             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2738          ctx->block->instructions.emplace_back(std::move(split));
2739       }
2740
2741       bytes_read += todo;
2742    }
2743
2744    assert(result_size == num_components && result_size > 1);
2745    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2746    for (unsigned i = 0; i < result_size; i++)
2747       vec->operands[i] = Operand(result[i]);
2748    vec->definitions[0] = Definition(dst);
2749    ctx->block->instructions.emplace_back(std::move(vec));
2750    ctx->allocated_vec.emplace(dst.id(), result);
2751 }
2752
2753 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2754 {
2755    if (start == 0 && size == data.size())
2756       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2757
2758    unsigned size_hint = 1;
2759    auto it = ctx->allocated_vec.find(data.id());
2760    if (it != ctx->allocated_vec.end())
2761       size_hint = it->second[0].size();
2762    if (size % size_hint || start % size_hint)
2763       size_hint = 1;
2764
2765    start /= size_hint;
2766    size /= size_hint;
2767
2768    Temp elems[size];
2769    for (unsigned i = 0; i < size; i++)
2770       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2771
2772    if (size == 1)
2773       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2774
2775    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2776    for (unsigned i = 0; i < size; i++)
2777       vec->operands[i] = Operand(elems[i]);
2778    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2779    vec->definitions[0] = Definition(res);
2780    ctx->block->instructions.emplace_back(std::move(vec));
2781    return res;
2782 }
2783
2784 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2785 {
2786    Builder bld(ctx->program, ctx->block);
2787    unsigned bytes_written = 0;
2788    while (bytes_written < total_size * 4) {
2789       unsigned todo = total_size * 4 - bytes_written;
2790       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2791       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2792
2793       aco_opcode op = aco_opcode::last_opcode;
2794       bool write2 = false;
2795       unsigned size = 0;
2796       if (todo >= 16 && aligned16) {
2797          op = aco_opcode::ds_write_b128;
2798          size = 4;
2799       } else if (todo >= 16 && aligned8) {
2800          op = aco_opcode::ds_write2_b64;
2801          write2 = true;
2802          size = 4;
2803       } else if (todo >= 12 && aligned16) {
2804          op = aco_opcode::ds_write_b96;
2805          size = 3;
2806       } else if (todo >= 8 && aligned8) {
2807          op = aco_opcode::ds_write_b64;
2808          size = 2;
2809       } else if (todo >= 8) {
2810          op = aco_opcode::ds_write2_b32;
2811          write2 = true;
2812          size = 2;
2813       } else if (todo >= 4) {
2814          op = aco_opcode::ds_write_b32;
2815          size = 1;
2816       } else {
2817          assert(false);
2818       }
2819
2820       unsigned offset = offset0 + offset1 + bytes_written;
2821       unsigned max_offset = write2 ? 1020 : 65535;
2822       Temp address_offset = address;
2823       if (offset > max_offset) {
2824          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2825          offset = offset1 + bytes_written;
2826       }
2827       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2828
2829       if (write2) {
2830          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2831          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2832          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2833       } else {
2834          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2835          bld.ds(op, address_offset, val, m, offset);
2836       }
2837
2838       bytes_written += size * 4;
2839    }
2840 }
2841
2842 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2843                Temp address, unsigned base_offset, unsigned align)
2844 {
2845    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2846
2847    Operand m = load_lds_size_m0(ctx);
2848
2849    /* we need at most two stores for 32bit variables */
2850    int start[2], count[2];
2851    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2852    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2853    assert(wrmask == 0);
2854
2855    /* one combined store is sufficient */
2856    if (count[0] == count[1]) {
2857       Builder bld(ctx->program, ctx->block);
2858
2859       Temp address_offset = address;
2860       if ((base_offset >> 2) + start[1] > 255) {
2861          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2862          base_offset = 0;
2863       }
2864
2865       assert(count[0] == 1);
2866       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2867       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2868       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2869       base_offset = base_offset / elem_size_bytes;
2870       bld.ds(op, address_offset, val0, val1, m,
2871              base_offset + start[0], base_offset + start[1]);
2872       return;
2873    }
2874
2875    for (unsigned i = 0; i < 2; i++) {
2876       if (count[i] == 0)
2877          continue;
2878
2879       unsigned elem_size_words = elem_size_bytes / 4;
2880       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2881                       base_offset, start[i] * elem_size_bytes, align);
2882    }
2883    return;
2884 }
2885
2886 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2887 {
2888    if (ctx->stage == vertex_vs) {
2889       visit_store_vs_output(ctx, instr);
2890    } else if (ctx->stage == fragment_fs) {
2891       visit_store_fs_output(ctx, instr);
2892    } else {
2893       unreachable("Shader stage not implemented");
2894    }
2895 }
2896
2897 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2898 {
2899    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2900    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2901
2902    Builder bld(ctx->program, ctx->block);
2903    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2904    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2905 }
2906
2907 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2908 {
2909    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2910    for (unsigned i = 0; i < num_components; i++)
2911       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2912
2913    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2914       assert(num_components == 4);
2915       Builder bld(ctx->program, ctx->block);
2916       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2917    }
2918
2919    for (Operand& op : vec->operands)
2920       op = op.isUndefined() ? Operand(0u) : op;
2921
2922    vec->definitions[0] = Definition(dst);
2923    ctx->block->instructions.emplace_back(std::move(vec));
2924    emit_split_vector(ctx, dst, num_components);
2925    return;
2926 }
2927
2928 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2929 {
2930    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2931    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2932    unsigned idx = nir_intrinsic_base(instr);
2933    unsigned component = nir_intrinsic_component(instr);
2934    Temp prim_mask = ctx->prim_mask;
2935
2936    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2937    if (offset) {
2938       assert(offset->u32 == 0);
2939    } else {
2940       /* the lower 15bit of the prim_mask contain the offset into LDS
2941        * while the upper bits contain the number of prims */
2942       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2943       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2944       Builder bld(ctx->program, ctx->block);
2945       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2946       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2947       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2948       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2949       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2950    }
2951
2952    if (instr->dest.ssa.num_components == 1) {
2953       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2954    } else {
2955       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2956       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2957       {
2958          Temp tmp = {ctx->program->allocateId(), v1};
2959          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2960          vec->operands[i] = Operand(tmp);
2961       }
2962       vec->definitions[0] = Definition(dst);
2963       ctx->block->instructions.emplace_back(std::move(vec));
2964    }
2965 }
2966
2967 unsigned get_num_channels_from_data_format(unsigned data_format)
2968 {
2969    switch (data_format) {
2970    case V_008F0C_BUF_DATA_FORMAT_8:
2971    case V_008F0C_BUF_DATA_FORMAT_16:
2972    case V_008F0C_BUF_DATA_FORMAT_32:
2973       return 1;
2974    case V_008F0C_BUF_DATA_FORMAT_8_8:
2975    case V_008F0C_BUF_DATA_FORMAT_16_16:
2976    case V_008F0C_BUF_DATA_FORMAT_32_32:
2977       return 2;
2978    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2979    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2980    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2981       return 3;
2982    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2983    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2984    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2985    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2986    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2987       return 4;
2988    default:
2989       break;
2990    }
2991
2992    return 4;
2993 }
2994
2995 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2996  * so we may need to fix it up. */
2997 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
2998 {
2999    Builder bld(ctx->program, ctx->block);
3000
3001    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3002       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3003
3004    /* For the integer-like cases, do a natural sign extension.
3005     *
3006     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3007     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3008     * exponent.
3009     */
3010    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3011    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3012
3013    /* Convert back to the right type. */
3014    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3015       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3016       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
3017       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3018    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3019       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3020    }
3021
3022    return alpha;
3023 }
3024
3025 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3026 {
3027    Builder bld(ctx->program, ctx->block);
3028    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3029    if (ctx->stage & sw_vs) {
3030
3031       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3032       if (off_instr->type != nir_instr_type_load_const) {
3033          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3034          nir_print_instr(off_instr, stderr);
3035          fprintf(stderr, "\n");
3036       }
3037       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3038
3039       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
3040
3041       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3042       unsigned component = nir_intrinsic_component(instr);
3043       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3044       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3045       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3046       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3047
3048       unsigned dfmt = attrib_format & 0xf;
3049
3050       unsigned nfmt = (attrib_format >> 4) & 0x7;
3051       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3052       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3053       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3054       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3055       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3056       if (post_shuffle)
3057          num_channels = MAX2(num_channels, 3);
3058
3059       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3060
3061       Temp index;
3062       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3063          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3064          if (divisor) {
3065             ctx->needs_instance_id = true;
3066
3067             if (divisor != 1) {
3068                Temp divided = bld.tmp(v1);
3069                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
3070                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
3071             } else {
3072                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
3073             }
3074          } else {
3075             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
3076          }
3077       } else {
3078          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
3079       }
3080
3081       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3082          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3083          attrib_offset = attrib_offset % attrib_stride;
3084       }
3085
3086       Operand soffset(0u);
3087       if (attrib_offset >= 4096) {
3088          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3089          attrib_offset = 0;
3090       }
3091
3092       aco_opcode opcode;
3093       switch (num_channels) {
3094       case 1:
3095          opcode = aco_opcode::tbuffer_load_format_x;
3096          break;
3097       case 2:
3098          opcode = aco_opcode::tbuffer_load_format_xy;
3099          break;
3100       case 3:
3101          opcode = aco_opcode::tbuffer_load_format_xyz;
3102          break;
3103       case 4:
3104          opcode = aco_opcode::tbuffer_load_format_xyzw;
3105          break;
3106       default:
3107          unreachable("Unimplemented load_input vector size");
3108       }
3109
3110       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3111
3112       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3113       mubuf->operands[0] = Operand(index);
3114       mubuf->operands[1] = Operand(list);
3115       mubuf->operands[2] = soffset;
3116       mubuf->definitions[0] = Definition(tmp);
3117       mubuf->idxen = true;
3118       mubuf->can_reorder = true;
3119       mubuf->dfmt = dfmt;
3120       mubuf->nfmt = nfmt;
3121       assert(attrib_offset < 4096);
3122       mubuf->offset = attrib_offset;
3123       ctx->block->instructions.emplace_back(std::move(mubuf));
3124
3125       emit_split_vector(ctx, tmp, tmp.size());
3126
3127       if (tmp.id() != dst.id()) {
3128          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3129                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3130
3131          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3132          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3133          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3134
3135          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3136          for (unsigned i = 0; i < dst.size(); i++) {
3137             unsigned idx = i + component;
3138             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3139                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3140                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3141             } else if (idx < num_channels) {
3142                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3143             } else if (is_float && idx == 3) {
3144                vec->operands[i] = Operand(0x3f800000u);
3145             } else if (!is_float && idx == 3) {
3146                vec->operands[i] = Operand(1u);
3147             } else {
3148                vec->operands[i] = Operand(0u);
3149             }
3150          }
3151          vec->definitions[0] = Definition(dst);
3152          ctx->block->instructions.emplace_back(std::move(vec));
3153          emit_split_vector(ctx, dst, dst.size());
3154       }
3155
3156    } else if (ctx->stage == fragment_fs) {
3157       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3158       if (off_instr->type != nir_instr_type_load_const ||
3159           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3160          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3161          nir_print_instr(off_instr, stderr);
3162          fprintf(stderr, "\n");
3163       }
3164
3165       Temp prim_mask = ctx->prim_mask;
3166       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3167       if (offset) {
3168          assert(offset->u32 == 0);
3169       } else {
3170          /* the lower 15bit of the prim_mask contain the offset into LDS
3171           * while the upper bits contain the number of prims */
3172          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3173          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3174          Builder bld(ctx->program, ctx->block);
3175          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3176          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3177          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3178          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3179          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3180       }
3181
3182       unsigned idx = nir_intrinsic_base(instr);
3183       unsigned component = nir_intrinsic_component(instr);
3184
3185       if (dst.size() == 1) {
3186          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3187       } else {
3188          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3189          for (unsigned i = 0; i < dst.size(); i++)
3190             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3191          vec->definitions[0] = Definition(dst);
3192          bld.insert(std::move(vec));
3193       }
3194
3195    } else {
3196       unreachable("Shader stage not implemented");
3197    }
3198 }
3199
3200 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3201 {
3202    if (ctx->program->info->need_indirect_descriptor_sets) {
3203       Builder bld(ctx->program, ctx->block);
3204       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
3205       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3206    }
3207
3208    return ctx->descriptor_sets[desc_set];
3209 }
3210
3211
3212 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3213 {
3214    Builder bld(ctx->program, ctx->block);
3215    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3216    if (!ctx->divergent_vals[instr->dest.ssa.index])
3217       index = bld.as_uniform(index);
3218    unsigned desc_set = nir_intrinsic_desc_set(instr);
3219    unsigned binding = nir_intrinsic_binding(instr);
3220
3221    Temp desc_ptr;
3222    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3223    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3224    unsigned offset = layout->binding[binding].offset;
3225    unsigned stride;
3226    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3227        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3228       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3229       desc_ptr = ctx->push_constants;
3230       offset = pipeline_layout->push_constant_size + 16 * idx;
3231       stride = 16;
3232    } else {
3233       desc_ptr = load_desc_ptr(ctx, desc_set);
3234       stride = layout->binding[binding].size;
3235    }
3236
3237    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3238    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3239    if (stride != 1) {
3240       if (nir_const_index) {
3241          const_index = const_index * stride;
3242       } else if (index.type() == RegType::vgpr) {
3243          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3244          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3245       } else {
3246          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3247       }
3248    }
3249    if (offset) {
3250       if (nir_const_index) {
3251          const_index = const_index + offset;
3252       } else if (index.type() == RegType::vgpr) {
3253          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3254       } else {
3255          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3256       }
3257    }
3258
3259    if (nir_const_index && const_index == 0) {
3260       index = desc_ptr;
3261    } else if (index.type() == RegType::vgpr) {
3262       index = bld.vadd32(bld.def(v1),
3263                          nir_const_index ? Operand(const_index) : Operand(index),
3264                          Operand(desc_ptr));
3265    } else {
3266       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3267                        nir_const_index ? Operand(const_index) : Operand(index),
3268                        Operand(desc_ptr));
3269    }
3270
3271    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3272 }
3273
3274 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3275                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3276 {
3277    Builder bld(ctx->program, ctx->block);
3278
3279    unsigned num_bytes = dst.size() * 4;
3280    bool dlc = glc && ctx->options->chip_class >= GFX10;
3281
3282    aco_opcode op;
3283    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3284       if (ctx->options->chip_class < GFX8)
3285          offset = as_vgpr(ctx, offset);
3286
3287       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3288       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3289       unsigned const_offset = 0;
3290
3291       Temp lower = Temp();
3292       if (num_bytes > 16) {
3293          assert(num_components == 3 || num_components == 4);
3294          op = aco_opcode::buffer_load_dwordx4;
3295          lower = bld.tmp(v4);
3296          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3297          mubuf->definitions[0] = Definition(lower);
3298          mubuf->operands[0] = vaddr;
3299          mubuf->operands[1] = Operand(rsrc);
3300          mubuf->operands[2] = soffset;
3301          mubuf->offen = (offset.type() == RegType::vgpr);
3302          mubuf->glc = glc;
3303          mubuf->dlc = dlc;
3304          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3305          mubuf->can_reorder = readonly;
3306          bld.insert(std::move(mubuf));
3307          emit_split_vector(ctx, lower, 2);
3308          num_bytes -= 16;
3309          const_offset = 16;
3310       }
3311
3312       switch (num_bytes) {
3313          case 4:
3314             op = aco_opcode::buffer_load_dword;
3315             break;
3316          case 8:
3317             op = aco_opcode::buffer_load_dwordx2;
3318             break;
3319          case 12:
3320             op = aco_opcode::buffer_load_dwordx3;
3321             break;
3322          case 16:
3323             op = aco_opcode::buffer_load_dwordx4;
3324             break;
3325          default:
3326             unreachable("Load SSBO not implemented for this size.");
3327       }
3328       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3329       mubuf->operands[0] = vaddr;
3330       mubuf->operands[1] = Operand(rsrc);
3331       mubuf->operands[2] = soffset;
3332       mubuf->offen = (offset.type() == RegType::vgpr);
3333       mubuf->glc = glc;
3334       mubuf->dlc = dlc;
3335       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3336       mubuf->can_reorder = readonly;
3337       mubuf->offset = const_offset;
3338       aco_ptr<Instruction> instr = std::move(mubuf);
3339
3340       if (dst.size() > 4) {
3341          assert(lower != Temp());
3342          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3343          instr->definitions[0] = Definition(upper);
3344          bld.insert(std::move(instr));
3345          if (dst.size() == 8)
3346             emit_split_vector(ctx, upper, 2);
3347          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3348          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3349          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3350          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3351          if (dst.size() == 8)
3352             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3353       }
3354
3355       if (dst.type() == RegType::sgpr) {
3356          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3357          instr->definitions[0] = Definition(vec);
3358          bld.insert(std::move(instr));
3359          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3360       } else {
3361          instr->definitions[0] = Definition(dst);
3362          bld.insert(std::move(instr));
3363       }
3364    } else {
3365       switch (num_bytes) {
3366          case 4:
3367             op = aco_opcode::s_buffer_load_dword;
3368             break;
3369          case 8:
3370             op = aco_opcode::s_buffer_load_dwordx2;
3371             break;
3372          case 12:
3373          case 16:
3374             op = aco_opcode::s_buffer_load_dwordx4;
3375             break;
3376          case 24:
3377          case 32:
3378             op = aco_opcode::s_buffer_load_dwordx8;
3379             break;
3380          default:
3381             unreachable("Load SSBO not implemented for this size.");
3382       }
3383       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3384       load->operands[0] = Operand(rsrc);
3385       load->operands[1] = Operand(bld.as_uniform(offset));
3386       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3387       load->definitions[0] = Definition(dst);
3388       load->glc = glc;
3389       load->dlc = dlc;
3390       load->barrier = readonly ? barrier_none : barrier_buffer;
3391       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3392       assert(ctx->options->chip_class >= GFX8 || !glc);
3393
3394       /* trim vector */
3395       if (dst.size() == 3) {
3396          Temp vec = bld.tmp(s4);
3397          load->definitions[0] = Definition(vec);
3398          bld.insert(std::move(load));
3399          emit_split_vector(ctx, vec, 4);
3400
3401          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3402                     emit_extract_vector(ctx, vec, 0, s1),
3403                     emit_extract_vector(ctx, vec, 1, s1),
3404                     emit_extract_vector(ctx, vec, 2, s1));
3405       } else if (dst.size() == 6) {
3406          Temp vec = bld.tmp(s8);
3407          load->definitions[0] = Definition(vec);
3408          bld.insert(std::move(load));
3409          emit_split_vector(ctx, vec, 4);
3410
3411          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3412                     emit_extract_vector(ctx, vec, 0, s2),
3413                     emit_extract_vector(ctx, vec, 1, s2),
3414                     emit_extract_vector(ctx, vec, 2, s2));
3415       } else {
3416          bld.insert(std::move(load));
3417       }
3418
3419    }
3420    emit_split_vector(ctx, dst, num_components);
3421 }
3422
3423 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3424 {
3425    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3426    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3427
3428    Builder bld(ctx->program, ctx->block);
3429
3430    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3431    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3432    unsigned binding = nir_intrinsic_binding(idx_instr);
3433    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3434
3435    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3436       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3437                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3438                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3439                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3440       if (ctx->options->chip_class >= GFX10) {
3441          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3442                       S_008F0C_OOB_SELECT(3) |
3443                       S_008F0C_RESOURCE_LEVEL(1);
3444       } else {
3445          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3446                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3447       }
3448       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3449                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3450                                      Operand(0xFFFFFFFFu),
3451                                      Operand(desc_type));
3452       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3453                         rsrc, upper_dwords);
3454    } else {
3455       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3456       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3457    }
3458
3459    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3460 }
3461
3462 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3463 {
3464    Builder bld(ctx->program, ctx->block);
3465    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3466
3467    unsigned offset = nir_intrinsic_base(instr);
3468    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3469    if (index_cv && instr->dest.ssa.bit_size == 32) {
3470
3471       unsigned count = instr->dest.ssa.num_components;
3472       unsigned start = (offset + index_cv->u32) / 4u;
3473       start -= ctx->base_inline_push_consts;
3474       if (start + count <= ctx->num_inline_push_consts) {
3475          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3476          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3477          for (unsigned i = 0; i < count; ++i) {
3478             elems[i] = ctx->inline_push_consts[start + i];
3479             vec->operands[i] = Operand{elems[i]};
3480          }
3481          vec->definitions[0] = Definition(dst);
3482          ctx->block->instructions.emplace_back(std::move(vec));
3483          ctx->allocated_vec.emplace(dst.id(), elems);
3484          return;
3485       }
3486    }
3487
3488    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3489    if (offset != 0) // TODO check if index != 0 as well
3490       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3491    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3492    Temp vec = dst;
3493    bool trim = false;
3494    aco_opcode op;
3495
3496    switch (dst.size()) {
3497    case 1:
3498       op = aco_opcode::s_load_dword;
3499       break;
3500    case 2:
3501       op = aco_opcode::s_load_dwordx2;
3502       break;
3503    case 3:
3504       vec = bld.tmp(s4);
3505       trim = true;
3506    case 4:
3507       op = aco_opcode::s_load_dwordx4;
3508       break;
3509    case 6:
3510       vec = bld.tmp(s8);
3511       trim = true;
3512    case 8:
3513       op = aco_opcode::s_load_dwordx8;
3514       break;
3515    default:
3516       unreachable("unimplemented or forbidden load_push_constant.");
3517    }
3518
3519    bld.smem(op, Definition(vec), ptr, index);
3520
3521    if (trim) {
3522       emit_split_vector(ctx, vec, 4);
3523       RegClass rc = dst.size() == 3 ? s1 : s2;
3524       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3525                  emit_extract_vector(ctx, vec, 0, rc),
3526                  emit_extract_vector(ctx, vec, 1, rc),
3527                  emit_extract_vector(ctx, vec, 2, rc));
3528
3529    }
3530    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3531 }
3532
3533 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3534 {
3535    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3536
3537    Builder bld(ctx->program, ctx->block);
3538
3539    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3540                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3541                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3542                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3543    if (ctx->options->chip_class >= GFX10) {
3544       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3545                    S_008F0C_OOB_SELECT(3) |
3546                    S_008F0C_RESOURCE_LEVEL(1);
3547    } else {
3548       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3549                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3550    }
3551
3552    unsigned base = nir_intrinsic_base(instr);
3553    unsigned range = nir_intrinsic_range(instr);
3554
3555    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3556    if (base && offset.type() == RegType::sgpr)
3557       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3558    else if (base && offset.type() == RegType::vgpr)
3559       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3560
3561    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3562                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3563                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3564                           Operand(desc_type));
3565
3566    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3567 }
3568
3569 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3570 {
3571    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3572       ctx->cf_info.exec_potentially_empty = true;
3573
3574    ctx->program->needs_exact = true;
3575
3576    // TODO: optimize uniform conditions
3577    Builder bld(ctx->program, ctx->block);
3578    Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
3579    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3580    bld.pseudo(aco_opcode::p_discard_if, src);
3581    ctx->block->kind |= block_kind_uses_discard_if;
3582    return;
3583 }
3584
3585 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3586 {
3587    Builder bld(ctx->program, ctx->block);
3588
3589    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3590       ctx->cf_info.exec_potentially_empty = true;
3591
3592    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3593                     ctx->cf_info.parent_loop.has_divergent_continue;
3594
3595    if (ctx->block->loop_nest_depth &&
3596        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3597       /* we handle discards the same way as jump instructions */
3598       append_logical_end(ctx->block);
3599
3600       /* in loops, discard behaves like break */
3601       Block *linear_target = ctx->cf_info.parent_loop.exit;
3602       ctx->block->kind |= block_kind_discard;
3603
3604       if (!divergent) {
3605          /* uniform discard - loop ends here */
3606          assert(nir_instr_is_last(&instr->instr));
3607          ctx->block->kind |= block_kind_uniform;
3608          ctx->cf_info.has_branch = true;
3609          bld.branch(aco_opcode::p_branch);
3610          add_linear_edge(ctx->block->index, linear_target);
3611          return;
3612       }
3613
3614       /* we add a break right behind the discard() instructions */
3615       ctx->block->kind |= block_kind_break;
3616       unsigned idx = ctx->block->index;
3617
3618       /* remove critical edges from linear CFG */
3619       bld.branch(aco_opcode::p_branch);
3620       Block* break_block = ctx->program->create_and_insert_block();
3621       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3622       break_block->kind |= block_kind_uniform;
3623       add_linear_edge(idx, break_block);
3624       add_linear_edge(break_block->index, linear_target);
3625       bld.reset(break_block);
3626       bld.branch(aco_opcode::p_branch);
3627
3628       Block* continue_block = ctx->program->create_and_insert_block();
3629       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3630       add_linear_edge(idx, continue_block);
3631       append_logical_start(continue_block);
3632       ctx->block = continue_block;
3633
3634       return;
3635    }
3636
3637    /* it can currently happen that NIR doesn't remove the unreachable code */
3638    if (!nir_instr_is_last(&instr->instr)) {
3639       ctx->program->needs_exact = true;
3640       /* save exec somewhere temporarily so that it doesn't get
3641        * overwritten before the discard from outer exec masks */
3642       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3643       bld.pseudo(aco_opcode::p_discard_if, cond);
3644       ctx->block->kind |= block_kind_uses_discard_if;
3645       return;
3646    }
3647
3648    /* This condition is incorrect for uniformly branched discards in a loop
3649     * predicated by a divergent condition, but the above code catches that case
3650     * and the discard would end up turning into a discard_if.
3651     * For example:
3652     * if (divergent) {
3653     *    while (...) {
3654     *       if (uniform) {
3655     *          discard;
3656     *       }
3657     *    }
3658     * }
3659     */
3660    if (!ctx->cf_info.parent_if.is_divergent) {
3661       /* program just ends here */
3662       ctx->block->kind |= block_kind_uniform;
3663       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3664               0 /* enabled mask */, 9 /* dest */,
3665               false /* compressed */, true/* done */, true /* valid mask */);
3666       bld.sopp(aco_opcode::s_endpgm);
3667       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3668    } else {
3669       ctx->block->kind |= block_kind_discard;
3670       /* branch and linear edge is added by visit_if() */
3671    }
3672 }
3673
3674 enum aco_descriptor_type {
3675    ACO_DESC_IMAGE,
3676    ACO_DESC_FMASK,
3677    ACO_DESC_SAMPLER,
3678    ACO_DESC_BUFFER,
3679    ACO_DESC_PLANE_0,
3680    ACO_DESC_PLANE_1,
3681    ACO_DESC_PLANE_2,
3682 };
3683
3684 static bool
3685 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3686    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3687       return false;
3688    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3689    return dim == ac_image_cube ||
3690           dim == ac_image_1darray ||
3691           dim == ac_image_2darray ||
3692           dim == ac_image_2darraymsaa;
3693 }
3694
3695 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3696                       enum aco_descriptor_type desc_type,
3697                       const nir_tex_instr *tex_instr, bool image, bool write)
3698 {
3699 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3700    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3701    if (it != ctx->tex_desc.end())
3702       return it->second;
3703 */
3704    Temp index = Temp();
3705    bool index_set = false;
3706    unsigned constant_index = 0;
3707    unsigned descriptor_set;
3708    unsigned base_index;
3709    Builder bld(ctx->program, ctx->block);
3710
3711    if (!deref_instr) {
3712       assert(tex_instr && !image);
3713       descriptor_set = 0;
3714       base_index = tex_instr->sampler_index;
3715    } else {
3716       while(deref_instr->deref_type != nir_deref_type_var) {
3717          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3718          if (!array_size)
3719             array_size = 1;
3720
3721          assert(deref_instr->deref_type == nir_deref_type_array);
3722          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3723          if (const_value) {
3724             constant_index += array_size * const_value->u32;
3725          } else {
3726             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3727             if (indirect.type() == RegType::vgpr)
3728                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3729
3730             if (array_size != 1)
3731                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3732
3733             if (!index_set) {
3734                index = indirect;
3735                index_set = true;
3736             } else {
3737                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3738             }
3739          }
3740
3741          deref_instr = nir_src_as_deref(deref_instr->parent);
3742       }
3743       descriptor_set = deref_instr->var->data.descriptor_set;
3744       base_index = deref_instr->var->data.binding;
3745    }
3746
3747    Temp list = load_desc_ptr(ctx, descriptor_set);
3748    list = convert_pointer_to_64_bit(ctx, list);
3749
3750    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3751    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3752    unsigned offset = binding->offset;
3753    unsigned stride = binding->size;
3754    aco_opcode opcode;
3755    RegClass type;
3756
3757    assert(base_index < layout->binding_count);
3758
3759    switch (desc_type) {
3760    case ACO_DESC_IMAGE:
3761       type = s8;
3762       opcode = aco_opcode::s_load_dwordx8;
3763       break;
3764    case ACO_DESC_FMASK:
3765       type = s8;
3766       opcode = aco_opcode::s_load_dwordx8;
3767       offset += 32;
3768       break;
3769    case ACO_DESC_SAMPLER:
3770       type = s4;
3771       opcode = aco_opcode::s_load_dwordx4;
3772       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3773          offset += radv_combined_image_descriptor_sampler_offset(binding);
3774       break;
3775    case ACO_DESC_BUFFER:
3776       type = s4;
3777       opcode = aco_opcode::s_load_dwordx4;
3778       break;
3779    case ACO_DESC_PLANE_0:
3780    case ACO_DESC_PLANE_1:
3781       type = s8;
3782       opcode = aco_opcode::s_load_dwordx8;
3783       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3784       break;
3785    case ACO_DESC_PLANE_2:
3786       type = s4;
3787       opcode = aco_opcode::s_load_dwordx4;
3788       offset += 64;
3789       break;
3790    default:
3791       unreachable("invalid desc_type\n");
3792    }
3793
3794    offset += constant_index * stride;
3795
3796    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3797       (!index_set || binding->immutable_samplers_equal)) {
3798       if (binding->immutable_samplers_equal)
3799          constant_index = 0;
3800
3801       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3802       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3803                         Operand(samplers[constant_index * 4 + 0]),
3804                         Operand(samplers[constant_index * 4 + 1]),
3805                         Operand(samplers[constant_index * 4 + 2]),
3806                         Operand(samplers[constant_index * 4 + 3]));
3807    }
3808
3809    Operand off;
3810    if (!index_set) {
3811       off = Operand(offset);
3812    } else {
3813       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3814                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3815    }
3816
3817    Temp res = bld.smem(opcode, bld.def(type), list, off);
3818
3819    if (desc_type == ACO_DESC_PLANE_2) {
3820       Temp components[8];
3821       for (unsigned i = 0; i < 8; i++)
3822          components[i] = bld.tmp(s1);
3823       bld.pseudo(aco_opcode::p_split_vector,
3824                  Definition(components[0]),
3825                  Definition(components[1]),
3826                  Definition(components[2]),
3827                  Definition(components[3]),
3828                  res);
3829
3830       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3831       bld.pseudo(aco_opcode::p_split_vector,
3832                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3833                  Definition(components[4]),
3834                  Definition(components[5]),
3835                  Definition(components[6]),
3836                  Definition(components[7]),
3837                  desc2);
3838
3839       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3840                        components[0], components[1], components[2], components[3],
3841                        components[4], components[5], components[6], components[7]);
3842    }
3843
3844    return res;
3845 }
3846
3847 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3848 {
3849    switch (dim) {
3850    case GLSL_SAMPLER_DIM_BUF:
3851       return 1;
3852    case GLSL_SAMPLER_DIM_1D:
3853       return array ? 2 : 1;
3854    case GLSL_SAMPLER_DIM_2D:
3855       return array ? 3 : 2;
3856    case GLSL_SAMPLER_DIM_MS:
3857       return array ? 4 : 3;
3858    case GLSL_SAMPLER_DIM_3D:
3859    case GLSL_SAMPLER_DIM_CUBE:
3860       return 3;
3861    case GLSL_SAMPLER_DIM_RECT:
3862    case GLSL_SAMPLER_DIM_SUBPASS:
3863       return 2;
3864    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3865       return 3;
3866    default:
3867       break;
3868    }
3869    return 0;
3870 }
3871
3872
3873 /* Adjust the sample index according to FMASK.
3874  *
3875  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3876  * which is the identity mapping. Each nibble says which physical sample
3877  * should be fetched to get that sample.
3878  *
3879  * For example, 0x11111100 means there are only 2 samples stored and
3880  * the second sample covers 3/4 of the pixel. When reading samples 0
3881  * and 1, return physical sample 0 (determined by the first two 0s
3882  * in FMASK), otherwise return physical sample 1.
3883  *
3884  * The sample index should be adjusted as follows:
3885  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3886  */
3887 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3888 {
3889    Builder bld(ctx->program, ctx->block);
3890    Temp fmask = bld.tmp(v1);
3891    unsigned dim = ctx->options->chip_class >= GFX10
3892                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3893                   : 0;
3894
3895    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3896    load->operands[0] = Operand(coords);
3897    load->operands[1] = Operand(fmask_desc_ptr);
3898    load->definitions[0] = Definition(fmask);
3899    load->glc = false;
3900    load->dlc = false;
3901    load->dmask = 0x1;
3902    load->unrm = true;
3903    load->da = da;
3904    load->dim = dim;
3905    load->can_reorder = true; /* fmask images shouldn't be modified */
3906    ctx->block->instructions.emplace_back(std::move(load));
3907
3908    Operand sample_index4;
3909    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3910       sample_index4 = Operand(sample_index.constantValue() << 2);
3911    } else if (sample_index.regClass() == s1) {
3912       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3913    } else {
3914       assert(sample_index.regClass() == v1);
3915       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3916    }
3917
3918    Temp final_sample;
3919    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3920       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3921    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3922       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3923    else
3924       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3925
3926    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3927     * resource descriptor is 0 (invalid),
3928     */
3929    Temp compare = bld.tmp(s2);
3930    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3931                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3932
3933    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3934
3935    /* Replace the MSAA sample index. */
3936    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3937 }
3938
3939 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3940 {
3941
3942    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3943    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3944    bool is_array = glsl_sampler_type_is_array(type);
3945    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3946    assert(!add_frag_pos && "Input attachments should be lowered.");
3947    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3948    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3949    int count = image_type_to_components_count(dim, is_array);
3950    std::vector<Operand> coords(count);
3951
3952    if (is_ms) {
3953       Operand sample_index;
3954       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3955       if (sample_cv)
3956          sample_index = Operand(sample_cv->u32);
3957       else
3958          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3959
3960       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3961          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3962          for (unsigned i = 0; i < vec->operands.size(); i++)
3963             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3964          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3965          vec->definitions[0] = Definition(fmask_load_address);
3966          ctx->block->instructions.emplace_back(std::move(vec));
3967
3968          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3969          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3970       }
3971       count--;
3972       coords[count] = sample_index;
3973    }
3974
3975    if (count == 1 && !gfx9_1d)
3976       return emit_extract_vector(ctx, src0, 0, v1);
3977
3978    if (gfx9_1d) {
3979       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3980       coords.resize(coords.size() + 1);
3981       coords[1] = Operand((uint32_t) 0);
3982       if (is_array)
3983          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3984    } else {
3985       for (int i = 0; i < count; i++)
3986          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3987    }
3988
3989    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3990    for (unsigned i = 0; i < coords.size(); i++)
3991       vec->operands[i] = coords[i];
3992    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3993    vec->definitions[0] = Definition(res);
3994    ctx->block->instructions.emplace_back(std::move(vec));
3995    return res;
3996 }
3997
3998
3999 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4000 {
4001    Builder bld(ctx->program, ctx->block);
4002    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4003    const struct glsl_type *type = glsl_without_array(var->type);
4004    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4005    bool is_array = glsl_sampler_type_is_array(type);
4006    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4007
4008    if (dim == GLSL_SAMPLER_DIM_BUF) {
4009       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4010       unsigned num_channels = util_last_bit(mask);
4011       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4012       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4013
4014       aco_opcode opcode;
4015       switch (num_channels) {
4016       case 1:
4017          opcode = aco_opcode::buffer_load_format_x;
4018          break;
4019       case 2:
4020          opcode = aco_opcode::buffer_load_format_xy;
4021          break;
4022       case 3:
4023          opcode = aco_opcode::buffer_load_format_xyz;
4024          break;
4025       case 4:
4026          opcode = aco_opcode::buffer_load_format_xyzw;
4027          break;
4028       default:
4029          unreachable(">4 channel buffer image load");
4030       }
4031       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4032       load->operands[0] = Operand(vindex);
4033       load->operands[1] = Operand(rsrc);
4034       load->operands[2] = Operand((uint32_t) 0);
4035       Temp tmp;
4036       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4037          tmp = dst;
4038       else
4039          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4040       load->definitions[0] = Definition(tmp);
4041       load->idxen = true;
4042       load->barrier = barrier_image;
4043       ctx->block->instructions.emplace_back(std::move(load));
4044
4045       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4046       return;
4047    }
4048
4049    Temp coords = get_image_coords(ctx, instr, type);
4050    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4051
4052    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4053    unsigned num_components = util_bitcount(dmask);
4054    Temp tmp;
4055    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4056       tmp = dst;
4057    else
4058       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4059
4060    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4061    load->operands[0] = Operand(coords);
4062    load->operands[1] = Operand(resource);
4063    load->definitions[0] = Definition(tmp);
4064    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4065    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4066    load->dmask = dmask;
4067    load->unrm = true;
4068    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4069    load->barrier = barrier_image;
4070    ctx->block->instructions.emplace_back(std::move(load));
4071
4072    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4073    return;
4074 }
4075
4076 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4077 {
4078    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4079    const struct glsl_type *type = glsl_without_array(var->type);
4080    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4081    bool is_array = glsl_sampler_type_is_array(type);
4082    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4083
4084    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4085
4086    if (dim == GLSL_SAMPLER_DIM_BUF) {
4087       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4088       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4089       aco_opcode opcode;
4090       switch (data.size()) {
4091       case 1:
4092          opcode = aco_opcode::buffer_store_format_x;
4093          break;
4094       case 2:
4095          opcode = aco_opcode::buffer_store_format_xy;
4096          break;
4097       case 3:
4098          opcode = aco_opcode::buffer_store_format_xyz;
4099          break;
4100       case 4:
4101          opcode = aco_opcode::buffer_store_format_xyzw;
4102          break;
4103       default:
4104          unreachable(">4 channel buffer image store");
4105       }
4106       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4107       store->operands[0] = Operand(vindex);
4108       store->operands[1] = Operand(rsrc);
4109       store->operands[2] = Operand((uint32_t) 0);
4110       store->operands[3] = Operand(data);
4111       store->idxen = true;
4112       store->glc = glc;
4113       store->dlc = false;
4114       store->disable_wqm = true;
4115       store->barrier = barrier_image;
4116       ctx->program->needs_exact = true;
4117       ctx->block->instructions.emplace_back(std::move(store));
4118       return;
4119    }
4120
4121    assert(data.type() == RegType::vgpr);
4122    Temp coords = get_image_coords(ctx, instr, type);
4123    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4124
4125    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4126    store->operands[0] = Operand(coords);
4127    store->operands[1] = Operand(resource);
4128    store->operands[2] = Operand(s4);
4129    store->operands[3] = Operand(data);
4130    store->glc = glc;
4131    store->dlc = false;
4132    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4133    store->dmask = (1 << data.size()) - 1;
4134    store->unrm = true;
4135    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4136    store->disable_wqm = true;
4137    store->barrier = barrier_image;
4138    ctx->program->needs_exact = true;
4139    ctx->block->instructions.emplace_back(std::move(store));
4140    return;
4141 }
4142
4143 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4144 {
4145    /* return the previous value if dest is ever used */
4146    bool return_previous = false;
4147    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4148       return_previous = true;
4149       break;
4150    }
4151    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4152       return_previous = true;
4153       break;
4154    }
4155
4156    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4157    const struct glsl_type *type = glsl_without_array(var->type);
4158    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4159    bool is_array = glsl_sampler_type_is_array(type);
4160    Builder bld(ctx->program, ctx->block);
4161
4162    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4163    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4164
4165    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4166       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4167
4168    aco_opcode buf_op, image_op;
4169    switch (instr->intrinsic) {
4170       case nir_intrinsic_image_deref_atomic_add:
4171          buf_op = aco_opcode::buffer_atomic_add;
4172          image_op = aco_opcode::image_atomic_add;
4173          break;
4174       case nir_intrinsic_image_deref_atomic_umin:
4175          buf_op = aco_opcode::buffer_atomic_umin;
4176          image_op = aco_opcode::image_atomic_umin;
4177          break;
4178       case nir_intrinsic_image_deref_atomic_imin:
4179          buf_op = aco_opcode::buffer_atomic_smin;
4180          image_op = aco_opcode::image_atomic_smin;
4181          break;
4182       case nir_intrinsic_image_deref_atomic_umax:
4183          buf_op = aco_opcode::buffer_atomic_umax;
4184          image_op = aco_opcode::image_atomic_umax;
4185          break;
4186       case nir_intrinsic_image_deref_atomic_imax:
4187          buf_op = aco_opcode::buffer_atomic_smax;
4188          image_op = aco_opcode::image_atomic_smax;
4189          break;
4190       case nir_intrinsic_image_deref_atomic_and:
4191          buf_op = aco_opcode::buffer_atomic_and;
4192          image_op = aco_opcode::image_atomic_and;
4193          break;
4194       case nir_intrinsic_image_deref_atomic_or:
4195          buf_op = aco_opcode::buffer_atomic_or;
4196          image_op = aco_opcode::image_atomic_or;
4197          break;
4198       case nir_intrinsic_image_deref_atomic_xor:
4199          buf_op = aco_opcode::buffer_atomic_xor;
4200          image_op = aco_opcode::image_atomic_xor;
4201          break;
4202       case nir_intrinsic_image_deref_atomic_exchange:
4203          buf_op = aco_opcode::buffer_atomic_swap;
4204          image_op = aco_opcode::image_atomic_swap;
4205          break;
4206       case nir_intrinsic_image_deref_atomic_comp_swap:
4207          buf_op = aco_opcode::buffer_atomic_cmpswap;
4208          image_op = aco_opcode::image_atomic_cmpswap;
4209          break;
4210       default:
4211          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4212    }
4213
4214    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4215
4216    if (dim == GLSL_SAMPLER_DIM_BUF) {
4217       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4218       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4219       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4220       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4221       mubuf->operands[0] = Operand(vindex);
4222       mubuf->operands[1] = Operand(resource);
4223       mubuf->operands[2] = Operand((uint32_t)0);
4224       mubuf->operands[3] = Operand(data);
4225       if (return_previous)
4226          mubuf->definitions[0] = Definition(dst);
4227       mubuf->offset = 0;
4228       mubuf->idxen = true;
4229       mubuf->glc = return_previous;
4230       mubuf->dlc = false; /* Not needed for atomics */
4231       mubuf->disable_wqm = true;
4232       mubuf->barrier = barrier_image;
4233       ctx->program->needs_exact = true;
4234       ctx->block->instructions.emplace_back(std::move(mubuf));
4235       return;
4236    }
4237
4238    Temp coords = get_image_coords(ctx, instr, type);
4239    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4240    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4241    mimg->operands[0] = Operand(coords);
4242    mimg->operands[1] = Operand(resource);
4243    mimg->operands[2] = Operand(s4); /* no sampler */
4244    mimg->operands[3] = Operand(data);
4245    if (return_previous)
4246       mimg->definitions[0] = Definition(dst);
4247    mimg->glc = return_previous;
4248    mimg->dlc = false; /* Not needed for atomics */
4249    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4250    mimg->dmask = (1 << data.size()) - 1;
4251    mimg->unrm = true;
4252    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4253    mimg->disable_wqm = true;
4254    mimg->barrier = barrier_image;
4255    ctx->program->needs_exact = true;
4256    ctx->block->instructions.emplace_back(std::move(mimg));
4257    return;
4258 }
4259
4260 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4261 {
4262    if (in_elements && ctx->options->chip_class == GFX8) {
4263       Builder bld(ctx->program, ctx->block);
4264
4265       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4266       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4267       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4268       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4269
4270       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4271       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4272
4273       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4274       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4275       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4276
4277       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4278       /* idea
4279        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4280        * in case 12 (or 3?), we have to divide by 3:
4281        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4282        * use v_mul_hi_u32 with magic number to divide
4283        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4284        * disable v_skip
4285        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4286        */
4287
4288    } else {
4289       emit_extract_vector(ctx, desc, 2, dst);
4290    }
4291 }
4292
4293 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4294 {
4295    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4296    const struct glsl_type *type = glsl_without_array(var->type);
4297    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4298    bool is_array = glsl_sampler_type_is_array(type);
4299    Builder bld(ctx->program, ctx->block);
4300
4301    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4302       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4303       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4304    }
4305
4306    /* LOD */
4307    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4308
4309    /* Resource */
4310    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4311
4312    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4313
4314    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4315    mimg->operands[0] = Operand(lod);
4316    mimg->operands[1] = Operand(resource);
4317    unsigned& dmask = mimg->dmask;
4318    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4319    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4320    mimg->da = glsl_sampler_type_is_array(type);
4321    mimg->can_reorder = true;
4322    Definition& def = mimg->definitions[0];
4323    ctx->block->instructions.emplace_back(std::move(mimg));
4324
4325    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4326        glsl_sampler_type_is_array(type)) {
4327
4328       assert(instr->dest.ssa.num_components == 3);
4329       Temp tmp = {ctx->program->allocateId(), v3};
4330       def = Definition(tmp);
4331       emit_split_vector(ctx, tmp, 3);
4332
4333       /* divide 3rd value by 6 by multiplying with magic number */
4334       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4335       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4336
4337       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4338                  emit_extract_vector(ctx, tmp, 0, v1),
4339                  emit_extract_vector(ctx, tmp, 1, v1),
4340                  by_6);
4341
4342    } else if (ctx->options->chip_class == GFX9 &&
4343               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4344               glsl_sampler_type_is_array(type)) {
4345       assert(instr->dest.ssa.num_components == 2);
4346       def = Definition(dst);
4347       dmask = 0x5;
4348    } else {
4349       def = Definition(dst);
4350    }
4351
4352    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4353 }
4354
4355 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4356 {
4357    Builder bld(ctx->program, ctx->block);
4358    unsigned num_components = instr->num_components;
4359
4360    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4361    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4362    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4363
4364    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4365    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4366 }
4367
4368 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4369 {
4370    Builder bld(ctx->program, ctx->block);
4371    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4372    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4373    unsigned writemask = nir_intrinsic_write_mask(instr);
4374
4375    Temp offset;
4376    if (ctx->options->chip_class < GFX8)
4377       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4378    else
4379       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4380
4381    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4382    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4383
4384    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4385                ctx->options->chip_class >= GFX8;
4386    if (smem)
4387       offset = bld.as_uniform(offset);
4388    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4389
4390    while (writemask) {
4391       int start, count;
4392       u_bit_scan_consecutive_range(&writemask, &start, &count);
4393       if (count == 3 && smem) {
4394          writemask |= 1u << (start + 2);
4395          count = 2;
4396       }
4397       int num_bytes = count * elem_size_bytes;
4398
4399       if (num_bytes > 16) {
4400          assert(elem_size_bytes == 8);
4401          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4402          count = 2;
4403          num_bytes = 16;
4404       }
4405
4406       // TODO: check alignment of sub-dword stores
4407       // TODO: split 3 bytes. there is no store instruction for that
4408
4409       Temp write_data;
4410       if (count != instr->num_components) {
4411          emit_split_vector(ctx, data, instr->num_components);
4412          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4413          for (int i = 0; i < count; i++) {
4414             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4415             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4416          }
4417          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4418          vec->definitions[0] = Definition(write_data);
4419          ctx->block->instructions.emplace_back(std::move(vec));
4420       } else if (!smem && data.type() != RegType::vgpr) {
4421          assert(num_bytes % 4 == 0);
4422          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4423       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4424          assert(num_bytes % 4 == 0);
4425          write_data = bld.as_uniform(data);
4426       } else {
4427          write_data = data;
4428       }
4429
4430       aco_opcode vmem_op, smem_op;
4431       switch (num_bytes) {
4432          case 4:
4433             vmem_op = aco_opcode::buffer_store_dword;
4434             smem_op = aco_opcode::s_buffer_store_dword;
4435             break;
4436          case 8:
4437             vmem_op = aco_opcode::buffer_store_dwordx2;
4438             smem_op = aco_opcode::s_buffer_store_dwordx2;
4439             break;
4440          case 12:
4441             vmem_op = aco_opcode::buffer_store_dwordx3;
4442             smem_op = aco_opcode::last_opcode;
4443             assert(!smem);
4444             break;
4445          case 16:
4446             vmem_op = aco_opcode::buffer_store_dwordx4;
4447             smem_op = aco_opcode::s_buffer_store_dwordx4;
4448             break;
4449          default:
4450             unreachable("Store SSBO not implemented for this size.");
4451       }
4452       if (ctx->stage == fragment_fs)
4453          smem_op = aco_opcode::p_fs_buffer_store_smem;
4454
4455       if (smem) {
4456          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4457          store->operands[0] = Operand(rsrc);
4458          if (start) {
4459             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4460                                 offset, Operand(start * elem_size_bytes));
4461             store->operands[1] = Operand(off);
4462          } else {
4463             store->operands[1] = Operand(offset);
4464          }
4465          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4466             store->operands[1].setFixed(m0);
4467          store->operands[2] = Operand(write_data);
4468          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4469          store->dlc = false;
4470          store->disable_wqm = true;
4471          store->barrier = barrier_buffer;
4472          ctx->block->instructions.emplace_back(std::move(store));
4473          ctx->program->wb_smem_l1_on_end = true;
4474          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4475             ctx->block->kind |= block_kind_needs_lowering;
4476             ctx->program->needs_exact = true;
4477          }
4478       } else {
4479          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4480          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4481          store->operands[1] = Operand(rsrc);
4482          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4483          store->operands[3] = Operand(write_data);
4484          store->offset = start * elem_size_bytes;
4485          store->offen = (offset.type() == RegType::vgpr);
4486          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4487          store->dlc = false;
4488          store->disable_wqm = true;
4489          store->barrier = barrier_buffer;
4490          ctx->program->needs_exact = true;
4491          ctx->block->instructions.emplace_back(std::move(store));
4492       }
4493    }
4494 }
4495
4496 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4497 {
4498    /* return the previous value if dest is ever used */
4499    bool return_previous = false;
4500    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4501       return_previous = true;
4502       break;
4503    }
4504    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4505       return_previous = true;
4506       break;
4507    }
4508
4509    Builder bld(ctx->program, ctx->block);
4510    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4511
4512    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4513       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4514                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4515
4516    Temp offset;
4517    if (ctx->options->chip_class < GFX8)
4518       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4519    else
4520       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4521
4522    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4523    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4524
4525    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4526
4527    aco_opcode op32, op64;
4528    switch (instr->intrinsic) {
4529       case nir_intrinsic_ssbo_atomic_add:
4530          op32 = aco_opcode::buffer_atomic_add;
4531          op64 = aco_opcode::buffer_atomic_add_x2;
4532          break;
4533       case nir_intrinsic_ssbo_atomic_imin:
4534          op32 = aco_opcode::buffer_atomic_smin;
4535          op64 = aco_opcode::buffer_atomic_smin_x2;
4536          break;
4537       case nir_intrinsic_ssbo_atomic_umin:
4538          op32 = aco_opcode::buffer_atomic_umin;
4539          op64 = aco_opcode::buffer_atomic_umin_x2;
4540          break;
4541       case nir_intrinsic_ssbo_atomic_imax:
4542          op32 = aco_opcode::buffer_atomic_smax;
4543          op64 = aco_opcode::buffer_atomic_smax_x2;
4544          break;
4545       case nir_intrinsic_ssbo_atomic_umax:
4546          op32 = aco_opcode::buffer_atomic_umax;
4547          op64 = aco_opcode::buffer_atomic_umax_x2;
4548          break;
4549       case nir_intrinsic_ssbo_atomic_and:
4550          op32 = aco_opcode::buffer_atomic_and;
4551          op64 = aco_opcode::buffer_atomic_and_x2;
4552          break;
4553       case nir_intrinsic_ssbo_atomic_or:
4554          op32 = aco_opcode::buffer_atomic_or;
4555          op64 = aco_opcode::buffer_atomic_or_x2;
4556          break;
4557       case nir_intrinsic_ssbo_atomic_xor:
4558          op32 = aco_opcode::buffer_atomic_xor;
4559          op64 = aco_opcode::buffer_atomic_xor_x2;
4560          break;
4561       case nir_intrinsic_ssbo_atomic_exchange:
4562          op32 = aco_opcode::buffer_atomic_swap;
4563          op64 = aco_opcode::buffer_atomic_swap_x2;
4564          break;
4565       case nir_intrinsic_ssbo_atomic_comp_swap:
4566          op32 = aco_opcode::buffer_atomic_cmpswap;
4567          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4568          break;
4569       default:
4570          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4571    }
4572    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4573    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4574    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4575    mubuf->operands[1] = Operand(rsrc);
4576    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4577    mubuf->operands[3] = Operand(data);
4578    if (return_previous)
4579       mubuf->definitions[0] = Definition(dst);
4580    mubuf->offset = 0;
4581    mubuf->offen = (offset.type() == RegType::vgpr);
4582    mubuf->glc = return_previous;
4583    mubuf->dlc = false; /* Not needed for atomics */
4584    mubuf->disable_wqm = true;
4585    mubuf->barrier = barrier_buffer;
4586    ctx->program->needs_exact = true;
4587    ctx->block->instructions.emplace_back(std::move(mubuf));
4588 }
4589
4590 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4591
4592    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4593    Builder bld(ctx->program, ctx->block);
4594    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4595    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4596 }
4597
4598 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4599 {
4600    Builder bld(ctx->program, ctx->block);
4601    unsigned num_components = instr->num_components;
4602    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4603
4604    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4605    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4606
4607    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4608    bool dlc = glc && ctx->options->chip_class >= GFX10;
4609    aco_opcode op;
4610    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4611       bool global = ctx->options->chip_class >= GFX9;
4612       aco_opcode op;
4613       switch (num_bytes) {
4614       case 4:
4615          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4616          break;
4617       case 8:
4618          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4619          break;
4620       case 12:
4621          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4622          break;
4623       case 16:
4624          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4625          break;
4626       default:
4627          unreachable("load_global not implemented for this size.");
4628       }
4629       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4630       flat->operands[0] = Operand(addr);
4631       flat->operands[1] = Operand(s1);
4632       flat->glc = glc;
4633       flat->dlc = dlc;
4634
4635       if (dst.type() == RegType::sgpr) {
4636          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4637          flat->definitions[0] = Definition(vec);
4638          ctx->block->instructions.emplace_back(std::move(flat));
4639          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4640       } else {
4641          flat->definitions[0] = Definition(dst);
4642          ctx->block->instructions.emplace_back(std::move(flat));
4643       }
4644       emit_split_vector(ctx, dst, num_components);
4645    } else {
4646       switch (num_bytes) {
4647          case 4:
4648             op = aco_opcode::s_load_dword;
4649             break;
4650          case 8:
4651             op = aco_opcode::s_load_dwordx2;
4652             break;
4653          case 12:
4654          case 16:
4655             op = aco_opcode::s_load_dwordx4;
4656             break;
4657          default:
4658             unreachable("load_global not implemented for this size.");
4659       }
4660       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4661       load->operands[0] = Operand(addr);
4662       load->operands[1] = Operand(0u);
4663       load->definitions[0] = Definition(dst);
4664       load->glc = glc;
4665       load->dlc = dlc;
4666       load->barrier = barrier_buffer;
4667       assert(ctx->options->chip_class >= GFX8 || !glc);
4668
4669       if (dst.size() == 3) {
4670          /* trim vector */
4671          Temp vec = bld.tmp(s4);
4672          load->definitions[0] = Definition(vec);
4673          ctx->block->instructions.emplace_back(std::move(load));
4674          emit_split_vector(ctx, vec, 4);
4675
4676          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4677                     emit_extract_vector(ctx, vec, 0, s1),
4678                     emit_extract_vector(ctx, vec, 1, s1),
4679                     emit_extract_vector(ctx, vec, 2, s1));
4680       } else {
4681          ctx->block->instructions.emplace_back(std::move(load));
4682       }
4683    }
4684 }
4685
4686 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4687 {
4688    Builder bld(ctx->program, ctx->block);
4689    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4690
4691    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4692    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4693
4694    unsigned writemask = nir_intrinsic_write_mask(instr);
4695    while (writemask) {
4696       int start, count;
4697       u_bit_scan_consecutive_range(&writemask, &start, &count);
4698       unsigned num_bytes = count * elem_size_bytes;
4699
4700       Temp write_data = data;
4701       if (count != instr->num_components) {
4702          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4703          for (int i = 0; i < count; i++)
4704             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4705          write_data = bld.tmp(RegType::vgpr, count);
4706          vec->definitions[0] = Definition(write_data);
4707          ctx->block->instructions.emplace_back(std::move(vec));
4708       }
4709
4710       unsigned offset = start * elem_size_bytes;
4711       if (offset > 0 && ctx->options->chip_class < GFX9) {
4712          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4713          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4714          Temp carry = bld.tmp(s2);
4715          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4716
4717          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4718                   Operand(offset), addr0);
4719          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4720                   Operand(0u), addr1,
4721                   carry).def(1).setHint(vcc);
4722
4723          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4724
4725          offset = 0;
4726       }
4727
4728       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4729       bool global = ctx->options->chip_class >= GFX9;
4730       aco_opcode op;
4731       switch (num_bytes) {
4732       case 4:
4733          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4734          break;
4735       case 8:
4736          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4737          break;
4738       case 12:
4739          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4740          break;
4741       case 16:
4742          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4743          break;
4744       default:
4745          unreachable("store_global not implemented for this size.");
4746       }
4747       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4748       flat->operands[0] = Operand(addr);
4749       flat->operands[1] = Operand(s1);
4750       flat->operands[2] = Operand(data);
4751       flat->glc = glc;
4752       flat->dlc = false;
4753       flat->offset = offset;
4754       ctx->block->instructions.emplace_back(std::move(flat));
4755    }
4756 }
4757
4758 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4759    Builder bld(ctx->program, ctx->block);
4760    switch(instr->intrinsic) {
4761       case nir_intrinsic_group_memory_barrier:
4762       case nir_intrinsic_memory_barrier:
4763          bld.barrier(aco_opcode::p_memory_barrier_all);
4764          break;
4765       case nir_intrinsic_memory_barrier_atomic_counter:
4766          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4767          break;
4768       case nir_intrinsic_memory_barrier_buffer:
4769          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4770          break;
4771       case nir_intrinsic_memory_barrier_image:
4772          bld.barrier(aco_opcode::p_memory_barrier_image);
4773          break;
4774       case nir_intrinsic_memory_barrier_shared:
4775          bld.barrier(aco_opcode::p_memory_barrier_shared);
4776          break;
4777       default:
4778          unreachable("Unimplemented memory barrier intrinsic");
4779          break;
4780    }
4781 }
4782
4783 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4784 {
4785    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4786    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4787    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4788    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4789    Builder bld(ctx->program, ctx->block);
4790
4791    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4792    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4793    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4794 }
4795
4796 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4797 {
4798    unsigned writemask = nir_intrinsic_write_mask(instr);
4799    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4800    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4801    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4802    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4803
4804    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4805    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4806 }
4807
4808 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4809 {
4810    unsigned offset = nir_intrinsic_base(instr);
4811    Operand m = load_lds_size_m0(ctx);
4812    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4813    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4814
4815    unsigned num_operands = 3;
4816    aco_opcode op32, op64, op32_rtn, op64_rtn;
4817    switch(instr->intrinsic) {
4818       case nir_intrinsic_shared_atomic_add:
4819          op32 = aco_opcode::ds_add_u32;
4820          op64 = aco_opcode::ds_add_u64;
4821          op32_rtn = aco_opcode::ds_add_rtn_u32;
4822          op64_rtn = aco_opcode::ds_add_rtn_u64;
4823          break;
4824       case nir_intrinsic_shared_atomic_imin:
4825          op32 = aco_opcode::ds_min_i32;
4826          op64 = aco_opcode::ds_min_i64;
4827          op32_rtn = aco_opcode::ds_min_rtn_i32;
4828          op64_rtn = aco_opcode::ds_min_rtn_i64;
4829          break;
4830       case nir_intrinsic_shared_atomic_umin:
4831          op32 = aco_opcode::ds_min_u32;
4832          op64 = aco_opcode::ds_min_u64;
4833          op32_rtn = aco_opcode::ds_min_rtn_u32;
4834          op64_rtn = aco_opcode::ds_min_rtn_u64;
4835          break;
4836       case nir_intrinsic_shared_atomic_imax:
4837          op32 = aco_opcode::ds_max_i32;
4838          op64 = aco_opcode::ds_max_i64;
4839          op32_rtn = aco_opcode::ds_max_rtn_i32;
4840          op64_rtn = aco_opcode::ds_max_rtn_i64;
4841          break;
4842       case nir_intrinsic_shared_atomic_umax:
4843          op32 = aco_opcode::ds_max_u32;
4844          op64 = aco_opcode::ds_max_u64;
4845          op32_rtn = aco_opcode::ds_max_rtn_u32;
4846          op64_rtn = aco_opcode::ds_max_rtn_u64;
4847          break;
4848       case nir_intrinsic_shared_atomic_and:
4849          op32 = aco_opcode::ds_and_b32;
4850          op64 = aco_opcode::ds_and_b64;
4851          op32_rtn = aco_opcode::ds_and_rtn_b32;
4852          op64_rtn = aco_opcode::ds_and_rtn_b64;
4853          break;
4854       case nir_intrinsic_shared_atomic_or:
4855          op32 = aco_opcode::ds_or_b32;
4856          op64 = aco_opcode::ds_or_b64;
4857          op32_rtn = aco_opcode::ds_or_rtn_b32;
4858          op64_rtn = aco_opcode::ds_or_rtn_b64;
4859          break;
4860       case nir_intrinsic_shared_atomic_xor:
4861          op32 = aco_opcode::ds_xor_b32;
4862          op64 = aco_opcode::ds_xor_b64;
4863          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4864          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4865          break;
4866       case nir_intrinsic_shared_atomic_exchange:
4867          op32 = aco_opcode::ds_write_b32;
4868          op64 = aco_opcode::ds_write_b64;
4869          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4870          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4871          break;
4872       case nir_intrinsic_shared_atomic_comp_swap:
4873          op32 = aco_opcode::ds_cmpst_b32;
4874          op64 = aco_opcode::ds_cmpst_b64;
4875          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4876          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4877          num_operands = 4;
4878          break;
4879       default:
4880          unreachable("Unhandled shared atomic intrinsic");
4881    }
4882
4883    /* return the previous value if dest is ever used */
4884    bool return_previous = false;
4885    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4886       return_previous = true;
4887       break;
4888    }
4889    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4890       return_previous = true;
4891       break;
4892    }
4893
4894    aco_opcode op;
4895    if (data.size() == 1) {
4896       assert(instr->dest.ssa.bit_size == 32);
4897       op = return_previous ? op32_rtn : op32;
4898    } else {
4899       assert(instr->dest.ssa.bit_size == 64);
4900       op = return_previous ? op64_rtn : op64;
4901    }
4902
4903    if (offset > 65535) {
4904       Builder bld(ctx->program, ctx->block);
4905       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4906       offset = 0;
4907    }
4908
4909    aco_ptr<DS_instruction> ds;
4910    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4911    ds->operands[0] = Operand(address);
4912    ds->operands[1] = Operand(data);
4913    if (num_operands == 4)
4914       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4915    ds->operands[num_operands - 1] = m;
4916    ds->offset0 = offset;
4917    if (return_previous)
4918       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4919    ctx->block->instructions.emplace_back(std::move(ds));
4920 }
4921
4922 Temp get_scratch_resource(isel_context *ctx)
4923 {
4924    Builder bld(ctx->program, ctx->block);
4925    Temp scratch_addr = ctx->program->private_segment_buffer;
4926    if (ctx->stage != compute_cs)
4927       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
4928
4929    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
4930                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
4931
4932    if (ctx->program->chip_class >= GFX10) {
4933       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4934                    S_008F0C_OOB_SELECT(3) |
4935                    S_008F0C_RESOURCE_LEVEL(1);
4936    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
4937       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4938                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4939    }
4940
4941    /* older generations need element size = 16 bytes. element size removed in GFX9 */
4942    if (ctx->program->chip_class <= GFX8)
4943       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
4944
4945    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4946 }
4947
4948 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4949    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4950    Builder bld(ctx->program, ctx->block);
4951    Temp rsrc = get_scratch_resource(ctx);
4952    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4953    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4954
4955    aco_opcode op;
4956    switch (dst.size()) {
4957       case 1:
4958          op = aco_opcode::buffer_load_dword;
4959          break;
4960       case 2:
4961          op = aco_opcode::buffer_load_dwordx2;
4962          break;
4963       case 3:
4964          op = aco_opcode::buffer_load_dwordx3;
4965          break;
4966       case 4:
4967          op = aco_opcode::buffer_load_dwordx4;
4968          break;
4969       case 6:
4970       case 8: {
4971          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4972          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4973                                 bld.def(v4), offset, rsrc,
4974                                 ctx->program->scratch_offset, 0, true);
4975          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4976                                                   aco_opcode::buffer_load_dwordx4,
4977                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4978                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
4979          emit_split_vector(ctx, lower, 2);
4980          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4981          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4982          if (dst.size() == 8) {
4983             emit_split_vector(ctx, upper, 2);
4984             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4985             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4986          } else {
4987             elems[2] = upper;
4988          }
4989
4990          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4991                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4992          for (unsigned i = 0; i < dst.size() / 2; i++)
4993             vec->operands[i] = Operand(elems[i]);
4994          vec->definitions[0] = Definition(dst);
4995          bld.insert(std::move(vec));
4996          ctx->allocated_vec.emplace(dst.id(), elems);
4997          return;
4998       }
4999       default:
5000          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5001    }
5002
5003    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5004    emit_split_vector(ctx, dst, instr->num_components);
5005 }
5006
5007 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5008    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5009    Builder bld(ctx->program, ctx->block);
5010    Temp rsrc = get_scratch_resource(ctx);
5011    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5012    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5013
5014    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5015    unsigned writemask = nir_intrinsic_write_mask(instr);
5016
5017    while (writemask) {
5018       int start, count;
5019       u_bit_scan_consecutive_range(&writemask, &start, &count);
5020       int num_bytes = count * elem_size_bytes;
5021
5022       if (num_bytes > 16) {
5023          assert(elem_size_bytes == 8);
5024          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5025          count = 2;
5026          num_bytes = 16;
5027       }
5028
5029       // TODO: check alignment of sub-dword stores
5030       // TODO: split 3 bytes. there is no store instruction for that
5031
5032       Temp write_data;
5033       if (count != instr->num_components) {
5034          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5035          for (int i = 0; i < count; i++) {
5036             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5037             vec->operands[i] = Operand(elem);
5038          }
5039          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5040          vec->definitions[0] = Definition(write_data);
5041          ctx->block->instructions.emplace_back(std::move(vec));
5042       } else {
5043          write_data = data;
5044       }
5045
5046       aco_opcode op;
5047       switch (num_bytes) {
5048          case 4:
5049             op = aco_opcode::buffer_store_dword;
5050             break;
5051          case 8:
5052             op = aco_opcode::buffer_store_dwordx2;
5053             break;
5054          case 12:
5055             op = aco_opcode::buffer_store_dwordx3;
5056             break;
5057          case 16:
5058             op = aco_opcode::buffer_store_dwordx4;
5059             break;
5060          default:
5061             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5062       }
5063
5064       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5065    }
5066 }
5067
5068 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5069    uint8_t log2_ps_iter_samples;
5070    if (ctx->program->info->ps.force_persample) {
5071       log2_ps_iter_samples =
5072          util_logbase2(ctx->options->key.fs.num_samples);
5073    } else {
5074       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5075    }
5076
5077    /* The bit pattern matches that used by fixed function fragment
5078     * processing. */
5079    static const unsigned ps_iter_masks[] = {
5080       0xffff, /* not used */
5081       0x5555,
5082       0x1111,
5083       0x0101,
5084       0x0001,
5085    };
5086    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5087
5088    Builder bld(ctx->program, ctx->block);
5089
5090    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
5091    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5092    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5093    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5094    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
5095 }
5096
5097 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5098 {
5099    Builder bld(ctx->program, ctx->block);
5100
5101    if (cluster_size == 1) {
5102       return src;
5103    } if (op == nir_op_iand && cluster_size == 4) {
5104       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5105       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5106       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
5107                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
5108    } else if (op == nir_op_ior && cluster_size == 4) {
5109       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5110       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
5111                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
5112    } else if (op == nir_op_iand && cluster_size == 64) {
5113       //subgroupAnd(val) -> (exec & ~val) == 0
5114       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5115       return bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), tmp, Operand(0u));
5116    } else if (op == nir_op_ior && cluster_size == 64) {
5117       //subgroupOr(val) -> (val & exec) != 0
5118       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
5119    } else if (op == nir_op_ixor && cluster_size == 64) {
5120       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5121       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5122       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
5123       return bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5124    } else {
5125       //subgroupClustered{And,Or,Xor}(val, n) ->
5126       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5127       //cluster_offset = ~(n - 1) & lane_id
5128       //cluster_mask = ((1 << n) - 1)
5129       //subgroupClusteredAnd():
5130       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5131       //subgroupClusteredOr():
5132       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5133       //subgroupClusteredXor():
5134       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5135       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5136                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5137       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5138
5139       Temp tmp;
5140       if (op == nir_op_iand)
5141          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5142       else
5143          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5144
5145       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5146       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5147       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5148       if (cluster_mask != 0xffffffff)
5149          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5150
5151       Definition cmp_def = Definition();
5152       if (op == nir_op_iand) {
5153          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5154       } else if (op == nir_op_ior) {
5155          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5156       } else if (op == nir_op_ixor) {
5157          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5158                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5159          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5160       }
5161       cmp_def.setHint(vcc);
5162       return cmp_def.getTemp();
5163    }
5164 }
5165
5166 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5167 {
5168    Builder bld(ctx->program, ctx->block);
5169
5170    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5171    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5172    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5173    Temp tmp;
5174    if (op == nir_op_iand)
5175       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5176    else
5177       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5178
5179    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5180    Temp lo = lohi.def(0).getTemp();
5181    Temp hi = lohi.def(1).getTemp();
5182    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5183                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5184
5185    Definition cmp_def = Definition();
5186    if (op == nir_op_iand)
5187       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5188    else if (op == nir_op_ior)
5189       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5190    else if (op == nir_op_ixor)
5191       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5192                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5193    cmp_def.setHint(vcc);
5194    return cmp_def.getTemp();
5195 }
5196
5197 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5198 {
5199    Builder bld(ctx->program, ctx->block);
5200
5201    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5202    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5203    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5204    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5205    if (op == nir_op_iand)
5206       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5207    else if (op == nir_op_ior)
5208       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5209    else if (op == nir_op_ixor)
5210       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5211
5212    assert(false);
5213    return Temp();
5214 }
5215
5216 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5217 {
5218    Builder bld(ctx->program, ctx->block);
5219    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5220    if (src.regClass().type() == RegType::vgpr) {
5221       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5222    } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5223       bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(dst), Operand(0u), Operand(src));
5224    } else if (src.regClass() == s1) {
5225       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5226    } else if (src.regClass() == s2) {
5227       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5228    } else {
5229       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5230       nir_print_instr(&instr->instr, stderr);
5231       fprintf(stderr, "\n");
5232    }
5233 }
5234
5235 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5236 {
5237    Builder bld(ctx->program, ctx->block);
5238    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5239    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5240
5241    /* Build DD X/Y */
5242    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5243    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5244    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5245    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5246    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5247    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5248
5249    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5250    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5251    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5252    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5253    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5254    Temp wqm1 = bld.tmp(v1);
5255    emit_wqm(ctx, tmp1, wqm1, true);
5256    Temp wqm2 = bld.tmp(v1);
5257    emit_wqm(ctx, tmp2, wqm2, true);
5258    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5259    return;
5260 }
5261
5262 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5263 {
5264    Builder bld(ctx->program, ctx->block);
5265    switch(instr->intrinsic) {
5266    case nir_intrinsic_load_barycentric_sample:
5267    case nir_intrinsic_load_barycentric_pixel:
5268    case nir_intrinsic_load_barycentric_centroid: {
5269       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5270       fs_input input = get_interp_input(instr->intrinsic, mode);
5271
5272       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5273       if (input == fs_input::max_inputs) {
5274          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5275                     Operand(0u), Operand(0u));
5276       } else {
5277          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5278                     ctx->fs_inputs[input],
5279                     ctx->fs_inputs[input + 1]);
5280       }
5281       emit_split_vector(ctx, dst, 2);
5282       break;
5283    }
5284    case nir_intrinsic_load_barycentric_at_sample: {
5285       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5286       switch (ctx->options->key.fs.num_samples) {
5287          case 2: sample_pos_offset += 1 << 3; break;
5288          case 4: sample_pos_offset += 3 << 3; break;
5289          case 8: sample_pos_offset += 7 << 3; break;
5290          default: break;
5291       }
5292       Temp sample_pos;
5293       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5294       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5295       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5296       if (addr.type() == RegType::sgpr) {
5297          Operand offset;
5298          if (const_addr) {
5299             sample_pos_offset += const_addr->u32 << 3;
5300             offset = Operand(sample_pos_offset);
5301          } else if (ctx->options->chip_class >= GFX9) {
5302             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5303          } else {
5304             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5305             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5306          }
5307          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5308
5309       } else if (ctx->options->chip_class >= GFX9) {
5310          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5311          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5312       } else {
5313          /* addr += private_segment_buffer + sample_pos_offset */
5314          Temp tmp0 = bld.tmp(s1);
5315          Temp tmp1 = bld.tmp(s1);
5316          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5317          Definition scc_tmp = bld.def(s1, scc);
5318          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5319          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5320          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5321          Temp pck0 = bld.tmp(v1);
5322          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5323          tmp1 = as_vgpr(ctx, tmp1);
5324          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5325          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5326
5327          /* sample_pos = flat_load_dwordx2 addr */
5328          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5329       }
5330
5331       /* sample_pos -= 0.5 */
5332       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5333       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5334       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5335       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5336       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5337
5338       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5339       break;
5340    }
5341    case nir_intrinsic_load_barycentric_at_offset: {
5342       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5343       RegClass rc = RegClass(offset.type(), 1);
5344       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5345       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5346       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5347       break;
5348    }
5349    case nir_intrinsic_load_front_face: {
5350       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5351                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5352       break;
5353    }
5354    case nir_intrinsic_load_view_index:
5355    case nir_intrinsic_load_layer_id: {
5356       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5357          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5358          bld.copy(Definition(dst), Operand(ctx->view_index));
5359          break;
5360       }
5361
5362       unsigned idx = nir_intrinsic_base(instr);
5363       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5364                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5365       break;
5366    }
5367    case nir_intrinsic_load_frag_coord: {
5368       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5369       break;
5370    }
5371    case nir_intrinsic_load_sample_pos: {
5372       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5373       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5374       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5375                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5376                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5377       break;
5378    }
5379    case nir_intrinsic_load_interpolated_input:
5380       visit_load_interpolated_input(ctx, instr);
5381       break;
5382    case nir_intrinsic_store_output:
5383       visit_store_output(ctx, instr);
5384       break;
5385    case nir_intrinsic_load_input:
5386       visit_load_input(ctx, instr);
5387       break;
5388    case nir_intrinsic_load_ubo:
5389       visit_load_ubo(ctx, instr);
5390       break;
5391    case nir_intrinsic_load_push_constant:
5392       visit_load_push_constant(ctx, instr);
5393       break;
5394    case nir_intrinsic_load_constant:
5395       visit_load_constant(ctx, instr);
5396       break;
5397    case nir_intrinsic_vulkan_resource_index:
5398       visit_load_resource(ctx, instr);
5399       break;
5400    case nir_intrinsic_discard:
5401       visit_discard(ctx, instr);
5402       break;
5403    case nir_intrinsic_discard_if:
5404       visit_discard_if(ctx, instr);
5405       break;
5406    case nir_intrinsic_load_shared:
5407       visit_load_shared(ctx, instr);
5408       break;
5409    case nir_intrinsic_store_shared:
5410       visit_store_shared(ctx, instr);
5411       break;
5412    case nir_intrinsic_shared_atomic_add:
5413    case nir_intrinsic_shared_atomic_imin:
5414    case nir_intrinsic_shared_atomic_umin:
5415    case nir_intrinsic_shared_atomic_imax:
5416    case nir_intrinsic_shared_atomic_umax:
5417    case nir_intrinsic_shared_atomic_and:
5418    case nir_intrinsic_shared_atomic_or:
5419    case nir_intrinsic_shared_atomic_xor:
5420    case nir_intrinsic_shared_atomic_exchange:
5421    case nir_intrinsic_shared_atomic_comp_swap:
5422       visit_shared_atomic(ctx, instr);
5423       break;
5424    case nir_intrinsic_image_deref_load:
5425       visit_image_load(ctx, instr);
5426       break;
5427    case nir_intrinsic_image_deref_store:
5428       visit_image_store(ctx, instr);
5429       break;
5430    case nir_intrinsic_image_deref_atomic_add:
5431    case nir_intrinsic_image_deref_atomic_umin:
5432    case nir_intrinsic_image_deref_atomic_imin:
5433    case nir_intrinsic_image_deref_atomic_umax:
5434    case nir_intrinsic_image_deref_atomic_imax:
5435    case nir_intrinsic_image_deref_atomic_and:
5436    case nir_intrinsic_image_deref_atomic_or:
5437    case nir_intrinsic_image_deref_atomic_xor:
5438    case nir_intrinsic_image_deref_atomic_exchange:
5439    case nir_intrinsic_image_deref_atomic_comp_swap:
5440       visit_image_atomic(ctx, instr);
5441       break;
5442    case nir_intrinsic_image_deref_size:
5443       visit_image_size(ctx, instr);
5444       break;
5445    case nir_intrinsic_load_ssbo:
5446       visit_load_ssbo(ctx, instr);
5447       break;
5448    case nir_intrinsic_store_ssbo:
5449       visit_store_ssbo(ctx, instr);
5450       break;
5451    case nir_intrinsic_load_global:
5452       visit_load_global(ctx, instr);
5453       break;
5454    case nir_intrinsic_store_global:
5455       visit_store_global(ctx, instr);
5456       break;
5457    case nir_intrinsic_ssbo_atomic_add:
5458    case nir_intrinsic_ssbo_atomic_imin:
5459    case nir_intrinsic_ssbo_atomic_umin:
5460    case nir_intrinsic_ssbo_atomic_imax:
5461    case nir_intrinsic_ssbo_atomic_umax:
5462    case nir_intrinsic_ssbo_atomic_and:
5463    case nir_intrinsic_ssbo_atomic_or:
5464    case nir_intrinsic_ssbo_atomic_xor:
5465    case nir_intrinsic_ssbo_atomic_exchange:
5466    case nir_intrinsic_ssbo_atomic_comp_swap:
5467       visit_atomic_ssbo(ctx, instr);
5468       break;
5469    case nir_intrinsic_load_scratch:
5470       visit_load_scratch(ctx, instr);
5471       break;
5472    case nir_intrinsic_store_scratch:
5473       visit_store_scratch(ctx, instr);
5474       break;
5475    case nir_intrinsic_get_buffer_size:
5476       visit_get_buffer_size(ctx, instr);
5477       break;
5478    case nir_intrinsic_barrier: {
5479       unsigned* bsize = ctx->program->info->cs.block_size;
5480       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5481       if (workgroup_size > 64)
5482          bld.sopp(aco_opcode::s_barrier);
5483       break;
5484    }
5485    case nir_intrinsic_group_memory_barrier:
5486    case nir_intrinsic_memory_barrier:
5487    case nir_intrinsic_memory_barrier_atomic_counter:
5488    case nir_intrinsic_memory_barrier_buffer:
5489    case nir_intrinsic_memory_barrier_image:
5490    case nir_intrinsic_memory_barrier_shared:
5491       emit_memory_barrier(ctx, instr);
5492       break;
5493    case nir_intrinsic_load_num_work_groups:
5494    case nir_intrinsic_load_work_group_id:
5495    case nir_intrinsic_load_local_invocation_id: {
5496       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5497       Temp* ids;
5498       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5499          ids = ctx->num_workgroups;
5500       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5501          ids = ctx->workgroup_ids;
5502       else
5503          ids = ctx->local_invocation_ids;
5504       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5505                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5506                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5507                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5508       emit_split_vector(ctx, dst, 3);
5509       break;
5510    }
5511    case nir_intrinsic_load_local_invocation_index: {
5512       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5513                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5514       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5515       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5516       break;
5517    }
5518    case nir_intrinsic_load_subgroup_id: {
5519       if (ctx->stage == compute_cs) {
5520          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5521          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5522       } else {
5523          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5524       }
5525       break;
5526    }
5527    case nir_intrinsic_load_subgroup_invocation: {
5528       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5529                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5530       break;
5531    }
5532    case nir_intrinsic_load_num_subgroups: {
5533       if (ctx->stage == compute_cs)
5534          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5535       else
5536          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5537       break;
5538    }
5539    case nir_intrinsic_ballot: {
5540       Definition tmp = bld.def(s2);
5541       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5542       if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s2) {
5543          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5544       } else if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s1) {
5545          bld.sop2(aco_opcode::s_cselect_b64, tmp, Operand(exec, s2), Operand(0u), bld.scc(src));
5546       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5547          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5548       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5549          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5550       } else {
5551          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5552          nir_print_instr(&instr->instr, stderr);
5553          fprintf(stderr, "\n");
5554       }
5555       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5556       break;
5557    }
5558    case nir_intrinsic_shuffle: {
5559       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5560       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5561          emit_uniform_subgroup(ctx, instr, src);
5562       } else {
5563          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5564          assert(tid.regClass() == v1);
5565          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5566          if (src.regClass() == v1) {
5567             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5568          } else if (src.regClass() == v2) {
5569             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5570             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5571             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5572             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5573             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5574             emit_split_vector(ctx, dst, 2);
5575          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5576             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5577             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5578             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5579             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5580          } else {
5581             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5582             nir_print_instr(&instr->instr, stderr);
5583             fprintf(stderr, "\n");
5584          }
5585       }
5586       break;
5587    }
5588    case nir_intrinsic_load_sample_id: {
5589       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5590                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5591       break;
5592    }
5593    case nir_intrinsic_load_sample_mask_in: {
5594       visit_load_sample_mask_in(ctx, instr);
5595       break;
5596    }
5597    case nir_intrinsic_read_first_invocation: {
5598       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5599       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5600       if (src.regClass() == v1) {
5601          emit_wqm(ctx,
5602                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5603                   dst);
5604       } else if (src.regClass() == v2) {
5605          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5606          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5607          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5608          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5609          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5610          emit_split_vector(ctx, dst, 2);
5611       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5612          emit_wqm(ctx,
5613                   bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5614                            bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2))),
5615                   dst);
5616       } else if (src.regClass() == s1) {
5617          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5618       } else if (src.regClass() == s2) {
5619          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5620       } else {
5621          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5622          nir_print_instr(&instr->instr, stderr);
5623          fprintf(stderr, "\n");
5624       }
5625       break;
5626    }
5627    case nir_intrinsic_read_invocation: {
5628       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5629       Temp lane = get_ssa_temp(ctx, instr->src[1].ssa);
5630       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5631       assert(lane.regClass() == s1);
5632       if (src.regClass() == v1) {
5633          emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), src, lane), dst);
5634       } else if (src.regClass() == v2) {
5635          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5636          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5637          lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), lo, lane));
5638          hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), hi, lane));
5639          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5640          emit_split_vector(ctx, dst, 2);
5641       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5642          emit_wqm(ctx, bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, lane), dst);
5643       } else if (src.regClass() == s1) {
5644          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5645       } else if (src.regClass() == s2) {
5646          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5647       } else {
5648          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5649          nir_print_instr(&instr->instr, stderr);
5650          fprintf(stderr, "\n");
5651       }
5652       break;
5653    }
5654    case nir_intrinsic_vote_all: {
5655       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5656       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5657       assert(src.regClass() == s2);
5658       assert(dst.regClass() == s1);
5659
5660       Definition tmp = bld.def(s1);
5661       bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(tmp),
5662                bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)),
5663                Operand(exec, s2));
5664       emit_wqm(ctx, tmp.getTemp(), dst);
5665       break;
5666    }
5667    case nir_intrinsic_vote_any: {
5668       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5669       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5670       assert(src.regClass() == s2);
5671       assert(dst.regClass() == s1);
5672
5673       Definition tmp = bld.def(s1);
5674       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(tmp), src, Operand(exec, s2));
5675       emit_wqm(ctx, tmp.getTemp(), dst);
5676       break;
5677    }
5678    case nir_intrinsic_reduce:
5679    case nir_intrinsic_inclusive_scan:
5680    case nir_intrinsic_exclusive_scan: {
5681       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5682       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5683       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5684       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5685          nir_intrinsic_cluster_size(instr) : 0;
5686       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5687
5688       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5689          emit_uniform_subgroup(ctx, instr, src);
5690       } else if (instr->dest.ssa.bit_size == 1) {
5691          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5692             op = nir_op_iand;
5693          else if (op == nir_op_iadd)
5694             op = nir_op_ixor;
5695          else if (op == nir_op_umax || op == nir_op_imax)
5696             op = nir_op_ior;
5697          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5698
5699          switch (instr->intrinsic) {
5700          case nir_intrinsic_reduce:
5701             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5702             break;
5703          case nir_intrinsic_exclusive_scan:
5704             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5705             break;
5706          case nir_intrinsic_inclusive_scan:
5707             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5708             break;
5709          default:
5710             assert(false);
5711          }
5712       } else if (cluster_size == 1) {
5713          bld.copy(Definition(dst), src);
5714       } else {
5715          src = as_vgpr(ctx, src);
5716
5717          ReduceOp reduce_op;
5718          switch (op) {
5719          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5720             CASE(iadd)
5721             CASE(imul)
5722             CASE(fadd)
5723             CASE(fmul)
5724             CASE(imin)
5725             CASE(umin)
5726             CASE(fmin)
5727             CASE(imax)
5728             CASE(umax)
5729             CASE(fmax)
5730             CASE(iand)
5731             CASE(ior)
5732             CASE(ixor)
5733             default:
5734                unreachable("unknown reduction op");
5735          #undef CASE
5736          }
5737
5738          aco_opcode aco_op;
5739          switch (instr->intrinsic) {
5740             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5741             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5742             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5743             default:
5744                unreachable("unknown reduce intrinsic");
5745          }
5746
5747          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5748          reduce->operands[0] = Operand(src);
5749          // filled in by aco_reduce_assign.cpp, used internally as part of the
5750          // reduce sequence
5751          assert(dst.size() == 1 || dst.size() == 2);
5752          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5753          reduce->operands[2] = Operand(v1.as_linear());
5754
5755          Temp tmp_dst = bld.tmp(dst.regClass());
5756          reduce->definitions[0] = Definition(tmp_dst);
5757          reduce->definitions[1] = bld.def(s2); // used internally
5758          reduce->definitions[2] = Definition();
5759          reduce->definitions[3] = Definition(scc, s1);
5760          reduce->definitions[4] = Definition();
5761          reduce->reduce_op = reduce_op;
5762          reduce->cluster_size = cluster_size;
5763          ctx->block->instructions.emplace_back(std::move(reduce));
5764
5765          emit_wqm(ctx, tmp_dst, dst);
5766       }
5767       break;
5768    }
5769    case nir_intrinsic_quad_broadcast: {
5770       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5771       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5772          emit_uniform_subgroup(ctx, instr, src);
5773       } else {
5774          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5775          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5776          if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5777             uint32_t half_mask = 0x11111111u << lane;
5778             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5779             Temp tmp = bld.tmp(s2);
5780             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5781                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5782                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5783             emit_wqm(ctx, tmp, dst);
5784          } else if (instr->dest.ssa.bit_size == 32) {
5785             emit_wqm(ctx,
5786                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5787                                   dpp_quad_perm(lane, lane, lane, lane)),
5788                      dst);
5789          } else if (instr->dest.ssa.bit_size == 64) {
5790             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5791             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5792             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5793             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5794             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5795             emit_split_vector(ctx, dst, 2);
5796          } else {
5797             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5798             nir_print_instr(&instr->instr, stderr);
5799             fprintf(stderr, "\n");
5800          }
5801       }
5802       break;
5803    }
5804    case nir_intrinsic_quad_swap_horizontal:
5805    case nir_intrinsic_quad_swap_vertical:
5806    case nir_intrinsic_quad_swap_diagonal:
5807    case nir_intrinsic_quad_swizzle_amd: {
5808       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5809       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5810          emit_uniform_subgroup(ctx, instr, src);
5811          break;
5812       }
5813       uint16_t dpp_ctrl = 0;
5814       switch (instr->intrinsic) {
5815       case nir_intrinsic_quad_swap_horizontal:
5816          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5817          break;
5818       case nir_intrinsic_quad_swap_vertical:
5819          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5820          break;
5821       case nir_intrinsic_quad_swap_diagonal:
5822          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5823          break;
5824       case nir_intrinsic_quad_swizzle_amd: {
5825          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5826          break;
5827       }
5828       default:
5829          break;
5830       }
5831
5832       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5833       if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5834          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5835          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5836          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5837          emit_wqm(ctx, tmp, dst);
5838       } else if (instr->dest.ssa.bit_size == 32) {
5839          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5840          emit_wqm(ctx, tmp, dst);
5841       } else if (instr->dest.ssa.bit_size == 64) {
5842          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5843          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5844          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5845          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5846          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5847          emit_split_vector(ctx, dst, 2);
5848       } else {
5849          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5850          nir_print_instr(&instr->instr, stderr);
5851          fprintf(stderr, "\n");
5852       }
5853       break;
5854    }
5855    case nir_intrinsic_masked_swizzle_amd: {
5856       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5857       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5858          emit_uniform_subgroup(ctx, instr, src);
5859          break;
5860       }
5861       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5862       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5863       if (dst.regClass() == v1) {
5864          emit_wqm(ctx,
5865                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5866                   dst);
5867       } else if (dst.regClass() == v2) {
5868          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5869          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5870          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5871          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5872          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5873          emit_split_vector(ctx, dst, 2);
5874       } else {
5875          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5876          nir_print_instr(&instr->instr, stderr);
5877          fprintf(stderr, "\n");
5878       }
5879       break;
5880    }
5881    case nir_intrinsic_write_invocation_amd: {
5882       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5883       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5884       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5885       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5886       if (dst.regClass() == v1) {
5887          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5888          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5889       } else if (dst.regClass() == v2) {
5890          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5891          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5892          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5893          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5894          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5895          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5896          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5897          emit_split_vector(ctx, dst, 2);
5898       } else {
5899          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5900          nir_print_instr(&instr->instr, stderr);
5901          fprintf(stderr, "\n");
5902       }
5903       break;
5904    }
5905    case nir_intrinsic_mbcnt_amd: {
5906       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5907       RegClass rc = RegClass(src.type(), 1);
5908       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5909       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5910       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5911       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5912       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5913       emit_wqm(ctx, wqm_tmp, dst);
5914       break;
5915    }
5916    case nir_intrinsic_load_helper_invocation: {
5917       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5918       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5919       ctx->block->kind |= block_kind_needs_lowering;
5920       ctx->program->needs_exact = true;
5921       break;
5922    }
5923    case nir_intrinsic_is_helper_invocation: {
5924       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5925       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5926       ctx->block->kind |= block_kind_needs_lowering;
5927       ctx->program->needs_exact = true;
5928       break;
5929    }
5930    case nir_intrinsic_demote:
5931       bld.pseudo(aco_opcode::p_demote_to_helper);
5932       ctx->block->kind |= block_kind_uses_demote;
5933       ctx->program->needs_exact = true;
5934       break;
5935    case nir_intrinsic_demote_if: {
5936       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc),
5937                            as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false),
5938                            Operand(exec, s2));
5939       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5940       ctx->block->kind |= block_kind_uses_demote;
5941       ctx->program->needs_exact = true;
5942       break;
5943    }
5944    case nir_intrinsic_first_invocation: {
5945       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5946                get_ssa_temp(ctx, &instr->dest.ssa));
5947       break;
5948    }
5949    case nir_intrinsic_shader_clock:
5950       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
5951       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
5952       break;
5953    case nir_intrinsic_load_vertex_id_zero_base: {
5954       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5955       bld.copy(Definition(dst), ctx->vertex_id);
5956       break;
5957    }
5958    case nir_intrinsic_load_first_vertex: {
5959       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5960       bld.copy(Definition(dst), ctx->base_vertex);
5961       break;
5962    }
5963    case nir_intrinsic_load_base_instance: {
5964       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5965       bld.copy(Definition(dst), ctx->start_instance);
5966       break;
5967    }
5968    case nir_intrinsic_load_instance_id: {
5969       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5970       bld.copy(Definition(dst), ctx->instance_id);
5971       break;
5972    }
5973    case nir_intrinsic_load_draw_id: {
5974       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5975       bld.copy(Definition(dst), ctx->draw_id);
5976       break;
5977    }
5978    default:
5979       fprintf(stderr, "Unimplemented intrinsic instr: ");
5980       nir_print_instr(&instr->instr, stderr);
5981       fprintf(stderr, "\n");
5982       abort();
5983
5984       break;
5985    }
5986 }
5987
5988
5989 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5990                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5991                     enum glsl_base_type *stype)
5992 {
5993    nir_deref_instr *texture_deref_instr = NULL;
5994    nir_deref_instr *sampler_deref_instr = NULL;
5995    int plane = -1;
5996
5997    for (unsigned i = 0; i < instr->num_srcs; i++) {
5998       switch (instr->src[i].src_type) {
5999       case nir_tex_src_texture_deref:
6000          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6001          break;
6002       case nir_tex_src_sampler_deref:
6003          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6004          break;
6005       case nir_tex_src_plane:
6006          plane = nir_src_as_int(instr->src[i].src);
6007          break;
6008       default:
6009          break;
6010       }
6011    }
6012
6013    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6014
6015    if (!sampler_deref_instr)
6016       sampler_deref_instr = texture_deref_instr;
6017
6018    if (plane >= 0) {
6019       assert(instr->op != nir_texop_txf_ms &&
6020              instr->op != nir_texop_samples_identical);
6021       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6022       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6023    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6024       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6025    } else {
6026       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6027    }
6028    if (samp_ptr) {
6029       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6030       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6031          fprintf(stderr, "Unimplemented sampler descriptor: ");
6032          nir_print_instr(&instr->instr, stderr);
6033          fprintf(stderr, "\n");
6034          abort();
6035          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
6036       }
6037    }
6038    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6039                      instr->op == nir_texop_samples_identical))
6040       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6041 }
6042
6043 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6044                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6045 {
6046    Builder bld(ctx->program, ctx->block);
6047
6048    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6049    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6050    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6051
6052    Operand neg_one(0xbf800000u);
6053    Operand one(0x3f800000u);
6054    Operand two(0x40000000u);
6055    Operand four(0x40800000u);
6056
6057    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
6058    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6059    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6060
6061    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
6062    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6063    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
6064    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
6065
6066    // select sc
6067    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6068    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6069                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6070                        one, is_ma_y);
6071    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6072
6073    // select tc
6074    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6075    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6076    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6077
6078    // select ma
6079    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6080                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6081                   deriv_z, is_ma_z);
6082    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6083    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6084 }
6085
6086 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6087 {
6088    Builder bld(ctx->program, ctx->block);
6089    Temp coord_args[4], ma, tc, sc, id;
6090    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6091       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6092
6093    if (is_array) {
6094       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6095
6096       // see comment in ac_prepare_cube_coords()
6097       if (ctx->options->chip_class <= GFX8)
6098          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6099    }
6100
6101    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6102
6103    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6104    vop3a->operands[0] = Operand(ma);
6105    vop3a->abs[0] = true;
6106    Temp invma = bld.tmp(v1);
6107    vop3a->definitions[0] = Definition(invma);
6108    ctx->block->instructions.emplace_back(std::move(vop3a));
6109
6110    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6111    if (!is_deriv)
6112       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6113
6114    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6115    if (!is_deriv)
6116       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6117
6118    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6119
6120    if (is_deriv) {
6121       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6122       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6123
6124       for (unsigned i = 0; i < 2; i++) {
6125          // see comment in ac_prepare_cube_coords()
6126          Temp deriv_ma;
6127          Temp deriv_sc, deriv_tc;
6128          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6129                            &deriv_ma, &deriv_sc, &deriv_tc);
6130
6131          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6132
6133          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6134                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6135                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6136          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6137                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6138                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6139          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6140       }
6141
6142       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6143       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6144    }
6145
6146    if (is_array)
6147       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6148    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6149
6150 }
6151
6152 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6153 {
6154    Temp coord_vec[3];
6155    for (unsigned i = 0; i < coords.size(); i++)
6156       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6157
6158    Builder bld(ctx->program, ctx->block);
6159    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6160
6161    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6162    for (unsigned i = 0; i < coords.size(); i++)
6163       vec->operands[i] = Operand(coord_vec[i]);
6164    Temp res = bld.tmp(RegType::vgpr, coords.size());
6165    vec->definitions[0] = Definition(res);
6166    ctx->block->instructions.emplace_back(std::move(vec));
6167    return res;
6168 }
6169
6170 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6171 {
6172    if (vec->parent_instr->type != nir_instr_type_alu)
6173       return;
6174    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6175    if (vec_instr->op != nir_op_vec(vec->num_components))
6176       return;
6177
6178    for (unsigned i = 0; i < vec->num_components; i++) {
6179       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6180               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6181    }
6182 }
6183
6184 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6185 {
6186    Builder bld(ctx->program, ctx->block);
6187    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6188         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6189    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6190         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6191    nir_const_value *sample_index_cv = NULL;
6192    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6193    enum glsl_base_type stype;
6194    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6195
6196    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6197                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6198    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6199                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6200
6201    for (unsigned i = 0; i < instr->num_srcs; i++) {
6202       switch (instr->src[i].src_type) {
6203       case nir_tex_src_coord:
6204          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6205          break;
6206       case nir_tex_src_bias:
6207          if (instr->op == nir_texop_txb) {
6208             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6209             has_bias = true;
6210          }
6211          break;
6212       case nir_tex_src_lod: {
6213          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6214
6215          if (val && val->f32 <= 0.0) {
6216             level_zero = true;
6217          } else {
6218             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6219             has_lod = true;
6220          }
6221          break;
6222       }
6223       case nir_tex_src_comparator:
6224          if (instr->is_shadow) {
6225             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6226             has_compare = true;
6227          }
6228          break;
6229       case nir_tex_src_offset:
6230          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6231          get_const_vec(instr->src[i].src.ssa, const_offset);
6232          has_offset = true;
6233          break;
6234       case nir_tex_src_ddx:
6235          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6236          has_ddx = true;
6237          break;
6238       case nir_tex_src_ddy:
6239          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6240          has_ddy = true;
6241          break;
6242       case nir_tex_src_ms_index:
6243          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6244          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6245          has_sample_index = true;
6246          break;
6247       case nir_tex_src_texture_offset:
6248       case nir_tex_src_sampler_offset:
6249       default:
6250          break;
6251       }
6252    }
6253 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6254    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6255       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6256
6257    if (instr->op == nir_texop_texture_samples) {
6258       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6259
6260       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6261       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6262       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6263       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6264
6265       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6266                samples, Operand(1u), bld.scc(is_msaa));
6267       return;
6268    }
6269
6270    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6271       aco_ptr<Instruction> tmp_instr;
6272       Temp acc, pack = Temp();
6273
6274       uint32_t pack_const = 0;
6275       for (unsigned i = 0; i < offset.size(); i++) {
6276          if (!const_offset[i])
6277             continue;
6278          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6279       }
6280
6281       if (offset.type() == RegType::sgpr) {
6282          for (unsigned i = 0; i < offset.size(); i++) {
6283             if (const_offset[i])
6284                continue;
6285
6286             acc = emit_extract_vector(ctx, offset, i, s1);
6287             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6288
6289             if (i) {
6290                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6291             }
6292
6293             if (pack == Temp()) {
6294                pack = acc;
6295             } else {
6296                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6297             }
6298          }
6299
6300          if (pack_const && pack != Temp())
6301             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6302       } else {
6303          for (unsigned i = 0; i < offset.size(); i++) {
6304             if (const_offset[i])
6305                continue;
6306
6307             acc = emit_extract_vector(ctx, offset, i, v1);
6308             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6309
6310             if (i) {
6311                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6312             }
6313
6314             if (pack == Temp()) {
6315                pack = acc;
6316             } else {
6317                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6318             }
6319          }
6320
6321          if (pack_const && pack != Temp())
6322             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6323       }
6324       if (pack_const && pack == Temp())
6325          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6326       else if (pack == Temp())
6327          has_offset = false;
6328       else
6329          offset = pack;
6330    }
6331
6332    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6333       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6334
6335    /* pack derivatives */
6336    if (has_ddx || has_ddy) {
6337       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6338          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6339                              ddx, Operand(0u), ddy, Operand(0u));
6340       } else {
6341          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6342       }
6343       has_derivs = true;
6344    }
6345
6346    if (instr->coord_components > 1 &&
6347        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6348        instr->is_array &&
6349        instr->op != nir_texop_txf)
6350       coords = apply_round_slice(ctx, coords, 1);
6351
6352    if (instr->coord_components > 2 &&
6353       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6354        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6355        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6356        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6357        instr->is_array &&
6358        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6359       coords = apply_round_slice(ctx, coords, 2);
6360
6361    if (ctx->options->chip_class == GFX9 &&
6362        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6363        instr->op != nir_texop_lod && instr->coord_components) {
6364       assert(coords.size() > 0 && coords.size() < 3);
6365
6366       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6367       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6368       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6369       if (coords.size() > 1)
6370          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6371       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6372       vec->definitions[0] = Definition(coords);
6373       ctx->block->instructions.emplace_back(std::move(vec));
6374    }
6375
6376    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6377
6378    if (instr->op == nir_texop_samples_identical)
6379       resource = fmask_ptr;
6380
6381    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6382              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6383             instr->op != nir_texop_txs) {
6384       assert(has_sample_index);
6385       Operand op(sample_index);
6386       if (sample_index_cv)
6387          op = Operand(sample_index_cv->u32);
6388       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6389    }
6390
6391    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6392       Temp split_coords[coords.size()];
6393       emit_split_vector(ctx, coords, coords.size());
6394       for (unsigned i = 0; i < coords.size(); i++)
6395          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6396
6397       unsigned i = 0;
6398       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6399          Temp off = emit_extract_vector(ctx, offset, i, v1);
6400          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6401       }
6402
6403       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6404       for (unsigned i = 0; i < coords.size(); i++)
6405          vec->operands[i] = Operand(split_coords[i]);
6406       coords = bld.tmp(coords.regClass());
6407       vec->definitions[0] = Definition(coords);
6408       ctx->block->instructions.emplace_back(std::move(vec));
6409
6410       has_offset = false;
6411    }
6412
6413    /* Build tex instruction */
6414    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6415    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6416                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6417                   : 0;
6418    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6419    Temp tmp_dst = dst;
6420
6421    /* gather4 selects the component by dmask and always returns vec4 */
6422    if (instr->op == nir_texop_tg4) {
6423       assert(instr->dest.ssa.num_components == 4);
6424       if (instr->is_shadow)
6425          dmask = 1;
6426       else
6427          dmask = 1 << instr->component;
6428       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6429          tmp_dst = bld.tmp(v4);
6430    } else if (instr->op == nir_texop_samples_identical) {
6431       tmp_dst = bld.tmp(v1);
6432    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6433       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6434    }
6435
6436    aco_ptr<MIMG_instruction> tex;
6437    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6438       if (!has_lod)
6439          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6440
6441       bool div_by_6 = instr->op == nir_texop_txs &&
6442                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6443                       instr->is_array &&
6444                       (dmask & (1 << 2));
6445       if (tmp_dst.id() == dst.id() && div_by_6)
6446          tmp_dst = bld.tmp(tmp_dst.regClass());
6447
6448       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6449       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6450       tex->operands[1] = Operand(resource);
6451       if (ctx->options->chip_class == GFX9 &&
6452           instr->op == nir_texop_txs &&
6453           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6454           instr->is_array) {
6455          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6456       } else if (instr->op == nir_texop_query_levels) {
6457          tex->dmask = 1 << 3;
6458       } else {
6459          tex->dmask = dmask;
6460       }
6461       tex->da = da;
6462       tex->definitions[0] = Definition(tmp_dst);
6463       tex->dim = dim;
6464       tex->can_reorder = true;
6465       ctx->block->instructions.emplace_back(std::move(tex));
6466
6467       if (div_by_6) {
6468          /* divide 3rd value by 6 by multiplying with magic number */
6469          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6470          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6471          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6472          assert(instr->dest.ssa.num_components == 3);
6473          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6474          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6475                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6476                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6477                               by_6);
6478
6479       }
6480
6481       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6482       return;
6483    }
6484
6485    Temp tg4_compare_cube_wa64 = Temp();
6486
6487    if (tg4_integer_workarounds) {
6488       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6489       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6490       tex->operands[1] = Operand(resource);
6491       tex->dim = dim;
6492       tex->dmask = 0x3;
6493       tex->da = da;
6494       Temp size = bld.tmp(v2);
6495       tex->definitions[0] = Definition(size);
6496       tex->can_reorder = true;
6497       ctx->block->instructions.emplace_back(std::move(tex));
6498       emit_split_vector(ctx, size, size.size());
6499
6500       Temp half_texel[2];
6501       for (unsigned i = 0; i < 2; i++) {
6502          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6503          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6504          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6505          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6506       }
6507
6508       Temp orig_coords[2] = {
6509          emit_extract_vector(ctx, coords, 0, v1),
6510          emit_extract_vector(ctx, coords, 1, v1)};
6511       Temp new_coords[2] = {
6512          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6513          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6514       };
6515
6516       if (tg4_integer_cube_workaround) {
6517          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6518          Temp desc[resource.size()];
6519          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6520                                                                            Format::PSEUDO, 1, resource.size())};
6521          split->operands[0] = Operand(resource);
6522          for (unsigned i = 0; i < resource.size(); i++) {
6523             desc[i] = bld.tmp(s1);
6524             split->definitions[i] = Definition(desc[i]);
6525          }
6526          ctx->block->instructions.emplace_back(std::move(split));
6527
6528          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6529          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6530                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6531
6532          Temp nfmt;
6533          if (stype == GLSL_TYPE_UINT) {
6534             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6535                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6536                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6537                             bld.scc(compare_cube_wa));
6538          } else {
6539             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6540                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6541                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6542                             bld.scc(compare_cube_wa));
6543          }
6544          tg4_compare_cube_wa64 = as_divergent_bool(ctx, compare_cube_wa, true);
6545          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6546
6547          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6548                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6549          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6550
6551          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6552                                                                          Format::PSEUDO, resource.size(), 1)};
6553          for (unsigned i = 0; i < resource.size(); i++)
6554             vec->operands[i] = Operand(desc[i]);
6555          resource = bld.tmp(resource.regClass());
6556          vec->definitions[0] = Definition(resource);
6557          ctx->block->instructions.emplace_back(std::move(vec));
6558
6559          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6560                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6561          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6562                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6563       }
6564
6565       if (coords.size() == 3) {
6566          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6567                              new_coords[0], new_coords[1],
6568                              emit_extract_vector(ctx, coords, 2, v1));
6569       } else {
6570          assert(coords.size() == 2);
6571          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6572                              new_coords[0], new_coords[1]);
6573       }
6574    }
6575
6576    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6577        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6578        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6579       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6580
6581    std::vector<Operand> args;
6582    if (has_offset)
6583       args.emplace_back(Operand(offset));
6584    if (has_bias)
6585       args.emplace_back(Operand(bias));
6586    if (has_compare)
6587       args.emplace_back(Operand(compare));
6588    if (has_derivs)
6589       args.emplace_back(Operand(derivs));
6590    args.emplace_back(Operand(coords));
6591    if (has_sample_index)
6592       args.emplace_back(Operand(sample_index));
6593    if (has_lod)
6594       args.emplace_back(lod);
6595
6596    Operand arg;
6597    if (args.size() > 1) {
6598       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6599       unsigned size = 0;
6600       for (unsigned i = 0; i < args.size(); i++) {
6601          size += args[i].size();
6602          vec->operands[i] = args[i];
6603       }
6604       RegClass rc = RegClass(RegType::vgpr, size);
6605       Temp tmp = bld.tmp(rc);
6606       vec->definitions[0] = Definition(tmp);
6607       ctx->block->instructions.emplace_back(std::move(vec));
6608       arg = Operand(tmp);
6609    } else {
6610       assert(args[0].isTemp());
6611       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6612    }
6613
6614    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6615       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6616
6617       assert(coords.size() == 1);
6618       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6619       aco_opcode op;
6620       switch (last_bit) {
6621       case 1:
6622          op = aco_opcode::buffer_load_format_x; break;
6623       case 2:
6624          op = aco_opcode::buffer_load_format_xy; break;
6625       case 3:
6626          op = aco_opcode::buffer_load_format_xyz; break;
6627       case 4:
6628          op = aco_opcode::buffer_load_format_xyzw; break;
6629       default:
6630          unreachable("Tex instruction loads more than 4 components.");
6631       }
6632
6633       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6634       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6635          tmp_dst = dst;
6636       else
6637          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6638
6639       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6640       mubuf->operands[0] = Operand(coords);
6641       mubuf->operands[1] = Operand(resource);
6642       mubuf->operands[2] = Operand((uint32_t) 0);
6643       mubuf->definitions[0] = Definition(tmp_dst);
6644       mubuf->idxen = true;
6645       mubuf->can_reorder = true;
6646       ctx->block->instructions.emplace_back(std::move(mubuf));
6647
6648       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6649       return;
6650    }
6651
6652
6653    if (instr->op == nir_texop_txf ||
6654        instr->op == nir_texop_txf_ms ||
6655        instr->op == nir_texop_samples_identical) {
6656       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6657       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6658       tex->operands[0] = Operand(arg);
6659       tex->operands[1] = Operand(resource);
6660       tex->dim = dim;
6661       tex->dmask = dmask;
6662       tex->unrm = true;
6663       tex->da = da;
6664       tex->definitions[0] = Definition(tmp_dst);
6665       tex->can_reorder = true;
6666       ctx->block->instructions.emplace_back(std::move(tex));
6667
6668       if (instr->op == nir_texop_samples_identical) {
6669          assert(dmask == 1 && dst.regClass() == v1);
6670          assert(dst.id() != tmp_dst.id());
6671
6672          Temp tmp = bld.tmp(s2);
6673          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6674          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6675
6676       } else {
6677          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6678       }
6679       return;
6680    }
6681
6682    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6683    aco_opcode opcode = aco_opcode::image_sample;
6684    if (has_offset) { /* image_sample_*_o */
6685       if (has_compare) {
6686          opcode = aco_opcode::image_sample_c_o;
6687          if (has_derivs)
6688             opcode = aco_opcode::image_sample_c_d_o;
6689          if (has_bias)
6690             opcode = aco_opcode::image_sample_c_b_o;
6691          if (level_zero)
6692             opcode = aco_opcode::image_sample_c_lz_o;
6693          if (has_lod)
6694             opcode = aco_opcode::image_sample_c_l_o;
6695       } else {
6696          opcode = aco_opcode::image_sample_o;
6697          if (has_derivs)
6698             opcode = aco_opcode::image_sample_d_o;
6699          if (has_bias)
6700             opcode = aco_opcode::image_sample_b_o;
6701          if (level_zero)
6702             opcode = aco_opcode::image_sample_lz_o;
6703          if (has_lod)
6704             opcode = aco_opcode::image_sample_l_o;
6705       }
6706    } else { /* no offset */
6707       if (has_compare) {
6708          opcode = aco_opcode::image_sample_c;
6709          if (has_derivs)
6710             opcode = aco_opcode::image_sample_c_d;
6711          if (has_bias)
6712             opcode = aco_opcode::image_sample_c_b;
6713          if (level_zero)
6714             opcode = aco_opcode::image_sample_c_lz;
6715          if (has_lod)
6716             opcode = aco_opcode::image_sample_c_l;
6717       } else {
6718          opcode = aco_opcode::image_sample;
6719          if (has_derivs)
6720             opcode = aco_opcode::image_sample_d;
6721          if (has_bias)
6722             opcode = aco_opcode::image_sample_b;
6723          if (level_zero)
6724             opcode = aco_opcode::image_sample_lz;
6725          if (has_lod)
6726             opcode = aco_opcode::image_sample_l;
6727       }
6728    }
6729
6730    if (instr->op == nir_texop_tg4) {
6731       if (has_offset) {
6732          opcode = aco_opcode::image_gather4_lz_o;
6733          if (has_compare)
6734             opcode = aco_opcode::image_gather4_c_lz_o;
6735       } else {
6736          opcode = aco_opcode::image_gather4_lz;
6737          if (has_compare)
6738             opcode = aco_opcode::image_gather4_c_lz;
6739       }
6740    } else if (instr->op == nir_texop_lod) {
6741       opcode = aco_opcode::image_get_lod;
6742    }
6743
6744    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6745    tex->operands[0] = arg;
6746    tex->operands[1] = Operand(resource);
6747    tex->operands[2] = Operand(sampler);
6748    tex->dim = dim;
6749    tex->dmask = dmask;
6750    tex->da = da;
6751    tex->definitions[0] = Definition(tmp_dst);
6752    tex->can_reorder = true;
6753    ctx->block->instructions.emplace_back(std::move(tex));
6754
6755    if (tg4_integer_cube_workaround) {
6756       assert(tmp_dst.id() != dst.id());
6757       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6758
6759       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6760       Temp val[4];
6761       for (unsigned i = 0; i < dst.size(); i++) {
6762          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6763          Temp cvt_val;
6764          if (stype == GLSL_TYPE_UINT)
6765             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6766          else
6767             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6768          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6769       }
6770       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6771       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6772                            val[0], val[1], val[2], val[3]);
6773    }
6774    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6775    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6776
6777 }
6778
6779
6780 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6781 {
6782    Temp tmp = get_ssa_temp(ctx, ssa);
6783    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6784       return Operand(tmp.regClass());
6785    else
6786       return Operand(tmp);
6787 }
6788
6789 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6790 {
6791    aco_ptr<Pseudo_instruction> phi;
6792    unsigned num_src = exec_list_length(&instr->srcs);
6793    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6794
6795    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6796
6797    std::map<unsigned, nir_ssa_def*> phi_src;
6798    bool all_undef = true;
6799    nir_foreach_phi_src(src, instr) {
6800       phi_src[src->pred->index] = src->src.ssa;
6801       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6802          all_undef = false;
6803    }
6804    if (all_undef) {
6805       Builder bld(ctx->program, ctx->block);
6806       if (dst.regClass() == s1) {
6807          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6808       } else if (dst.regClass() == v1) {
6809          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6810       } else {
6811          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6812          for (unsigned i = 0; i < dst.size(); i++)
6813             vec->operands[i] = Operand(0u);
6814          vec->definitions[0] = Definition(dst);
6815          ctx->block->instructions.emplace_back(std::move(vec));
6816       }
6817       return;
6818    }
6819
6820    /* try to scalarize vector phis */
6821    if (dst.size() > 1) {
6822       // TODO: scalarize linear phis on divergent ifs
6823       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6824       std::array<Temp, 4> new_vec;
6825       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6826          Operand src = get_phi_operand(ctx, pair.second);
6827          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6828             can_scalarize = false;
6829             break;
6830          }
6831       }
6832       if (can_scalarize) {
6833          unsigned num_components = instr->dest.ssa.num_components;
6834          assert(dst.size() % num_components == 0);
6835          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6836
6837          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6838          for (unsigned k = 0; k < num_components; k++) {
6839             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6840             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6841             for (unsigned i = 0; i < num_src; i++) {
6842                Operand src = get_phi_operand(ctx, it->second);
6843                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6844                ++it;
6845             }
6846             Temp phi_dst = {ctx->program->allocateId(), rc};
6847             phi->definitions[0] = Definition(phi_dst);
6848             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6849             new_vec[k] = phi_dst;
6850             vec->operands[k] = Operand(phi_dst);
6851          }
6852          vec->definitions[0] = Definition(dst);
6853          ctx->block->instructions.emplace_back(std::move(vec));
6854          ctx->allocated_vec.emplace(dst.id(), new_vec);
6855          return;
6856       }
6857    }
6858
6859    unsigned extra_src = 0;
6860    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6861        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6862       extra_src++;
6863    }
6864
6865    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6866
6867    /* if we have a linear phi on a divergent if, we know that one src is undef */
6868    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6869       assert(extra_src == 0);
6870       Block* block;
6871       /* we place the phi either in the invert-block or in the current block */
6872       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6873          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6874          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6875          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6876          assert(block->kind & block_kind_invert);
6877          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6878       } else {
6879          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6880          block = ctx->block;
6881          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6882       }
6883       phi->operands[1] = Operand(dst.regClass());
6884       phi->definitions[0] = Definition(dst);
6885       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6886       return;
6887    }
6888
6889    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6890    for (unsigned i = 0; i < num_src; i++) {
6891       phi->operands[i] = get_phi_operand(ctx, it->second);
6892       ++it;
6893    }
6894    for (unsigned i = 0; i < extra_src; i++)
6895       phi->operands[num_src + i] = Operand(dst.regClass());
6896    phi->definitions[0] = Definition(dst);
6897    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6898 }
6899
6900
6901 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6902 {
6903    Temp dst = get_ssa_temp(ctx, &instr->def);
6904
6905    assert(dst.type() == RegType::sgpr);
6906
6907    if (dst.size() == 1) {
6908       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6909    } else {
6910       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6911       for (unsigned i = 0; i < dst.size(); i++)
6912          vec->operands[i] = Operand(0u);
6913       vec->definitions[0] = Definition(dst);
6914       ctx->block->instructions.emplace_back(std::move(vec));
6915    }
6916 }
6917
6918 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6919 {
6920    Builder bld(ctx->program, ctx->block);
6921    Block *logical_target;
6922    append_logical_end(ctx->block);
6923    unsigned idx = ctx->block->index;
6924
6925    switch (instr->type) {
6926    case nir_jump_break:
6927       logical_target = ctx->cf_info.parent_loop.exit;
6928       add_logical_edge(idx, logical_target);
6929       ctx->block->kind |= block_kind_break;
6930
6931       if (!ctx->cf_info.parent_if.is_divergent &&
6932           !ctx->cf_info.parent_loop.has_divergent_continue) {
6933          /* uniform break - directly jump out of the loop */
6934          ctx->block->kind |= block_kind_uniform;
6935          ctx->cf_info.has_branch = true;
6936          bld.branch(aco_opcode::p_branch);
6937          add_linear_edge(idx, logical_target);
6938          return;
6939       }
6940       ctx->cf_info.parent_loop.has_divergent_branch = true;
6941       break;
6942    case nir_jump_continue:
6943       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6944       add_logical_edge(idx, logical_target);
6945       ctx->block->kind |= block_kind_continue;
6946
6947       if (ctx->cf_info.parent_if.is_divergent) {
6948          /* for potential uniform breaks after this continue,
6949             we must ensure that they are handled correctly */
6950          ctx->cf_info.parent_loop.has_divergent_continue = true;
6951          ctx->cf_info.parent_loop.has_divergent_branch = true;
6952       } else {
6953          /* uniform continue - directly jump to the loop header */
6954          ctx->block->kind |= block_kind_uniform;
6955          ctx->cf_info.has_branch = true;
6956          bld.branch(aco_opcode::p_branch);
6957          add_linear_edge(idx, logical_target);
6958          return;
6959       }
6960       break;
6961    default:
6962       fprintf(stderr, "Unknown NIR jump instr: ");
6963       nir_print_instr(&instr->instr, stderr);
6964       fprintf(stderr, "\n");
6965       abort();
6966    }
6967
6968    /* remove critical edges from linear CFG */
6969    bld.branch(aco_opcode::p_branch);
6970    Block* break_block = ctx->program->create_and_insert_block();
6971    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6972    break_block->kind |= block_kind_uniform;
6973    add_linear_edge(idx, break_block);
6974    /* the loop_header pointer might be invalidated by this point */
6975    if (instr->type == nir_jump_continue)
6976       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6977    add_linear_edge(break_block->index, logical_target);
6978    bld.reset(break_block);
6979    bld.branch(aco_opcode::p_branch);
6980
6981    Block* continue_block = ctx->program->create_and_insert_block();
6982    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6983    add_linear_edge(idx, continue_block);
6984    append_logical_start(continue_block);
6985    ctx->block = continue_block;
6986    return;
6987 }
6988
6989 void visit_block(isel_context *ctx, nir_block *block)
6990 {
6991    nir_foreach_instr(instr, block) {
6992       switch (instr->type) {
6993       case nir_instr_type_alu:
6994          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6995          break;
6996       case nir_instr_type_load_const:
6997          visit_load_const(ctx, nir_instr_as_load_const(instr));
6998          break;
6999       case nir_instr_type_intrinsic:
7000          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7001          break;
7002       case nir_instr_type_tex:
7003          visit_tex(ctx, nir_instr_as_tex(instr));
7004          break;
7005       case nir_instr_type_phi:
7006          visit_phi(ctx, nir_instr_as_phi(instr));
7007          break;
7008       case nir_instr_type_ssa_undef:
7009          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7010          break;
7011       case nir_instr_type_deref:
7012          break;
7013       case nir_instr_type_jump:
7014          visit_jump(ctx, nir_instr_as_jump(instr));
7015          break;
7016       default:
7017          fprintf(stderr, "Unknown NIR instr type: ");
7018          nir_print_instr(instr, stderr);
7019          fprintf(stderr, "\n");
7020          //abort();
7021       }
7022    }
7023 }
7024
7025
7026
7027 static void visit_loop(isel_context *ctx, nir_loop *loop)
7028 {
7029    append_logical_end(ctx->block);
7030    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7031    Builder bld(ctx->program, ctx->block);
7032    bld.branch(aco_opcode::p_branch);
7033    unsigned loop_preheader_idx = ctx->block->index;
7034
7035    Block loop_exit = Block();
7036    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7037    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7038
7039    Block* loop_header = ctx->program->create_and_insert_block();
7040    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7041    loop_header->kind |= block_kind_loop_header;
7042    add_edge(loop_preheader_idx, loop_header);
7043    ctx->block = loop_header;
7044
7045    /* emit loop body */
7046    unsigned loop_header_idx = loop_header->index;
7047    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7048    append_logical_start(ctx->block);
7049    visit_cf_list(ctx, &loop->body);
7050
7051    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7052    if (!ctx->cf_info.has_branch) {
7053       append_logical_end(ctx->block);
7054       if (ctx->cf_info.exec_potentially_empty) {
7055          /* Discards can result in code running with an empty exec mask.
7056           * This would result in divergent breaks not ever being taken. As a
7057           * workaround, break the loop when the loop mask is empty instead of
7058           * always continuing. */
7059          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7060          unsigned block_idx = ctx->block->index;
7061
7062          /* create helper blocks to avoid critical edges */
7063          Block *break_block = ctx->program->create_and_insert_block();
7064          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7065          break_block->kind = block_kind_uniform;
7066          bld.reset(break_block);
7067          bld.branch(aco_opcode::p_branch);
7068          add_linear_edge(block_idx, break_block);
7069          add_linear_edge(break_block->index, &loop_exit);
7070
7071          Block *continue_block = ctx->program->create_and_insert_block();
7072          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7073          continue_block->kind = block_kind_uniform;
7074          bld.reset(continue_block);
7075          bld.branch(aco_opcode::p_branch);
7076          add_linear_edge(block_idx, continue_block);
7077          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7078
7079          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7080          ctx->block = &ctx->program->blocks[block_idx];
7081       } else {
7082          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7083          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7084             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7085          else
7086             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7087       }
7088
7089       bld.reset(ctx->block);
7090       bld.branch(aco_opcode::p_branch);
7091    }
7092
7093    /* fixup phis in loop header from unreachable blocks */
7094    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7095       bool linear = ctx->cf_info.has_branch;
7096       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7097       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7098          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7099              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7100             /* the last operand should be the one that needs to be removed */
7101             instr->operands.pop_back();
7102          } else if (!is_phi(instr)) {
7103             break;
7104          }
7105       }
7106    }
7107
7108    ctx->cf_info.has_branch = false;
7109
7110    // TODO: if the loop has not a single exit, we must add one °°
7111    /* emit loop successor block */
7112    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7113    append_logical_start(ctx->block);
7114
7115    #if 0
7116    // TODO: check if it is beneficial to not branch on continues
7117    /* trim linear phis in loop header */
7118    for (auto&& instr : loop_entry->instructions) {
7119       if (instr->opcode == aco_opcode::p_linear_phi) {
7120          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7121          new_phi->definitions[0] = instr->definitions[0];
7122          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7123             new_phi->operands[i] = instr->operands[i];
7124          /* check that the remaining operands are all the same */
7125          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7126             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7127          instr.swap(new_phi);
7128       } else if (instr->opcode == aco_opcode::p_phi) {
7129          continue;
7130       } else {
7131          break;
7132       }
7133    }
7134    #endif
7135 }
7136
7137 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7138 {
7139    ic->cond = cond;
7140
7141    append_logical_end(ctx->block);
7142    ctx->block->kind |= block_kind_branch;
7143
7144    /* branch to linear then block */
7145    assert(cond.regClass() == s2);
7146    aco_ptr<Pseudo_branch_instruction> branch;
7147    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7148    branch->operands[0] = Operand(cond);
7149    ctx->block->instructions.push_back(std::move(branch));
7150
7151    ic->BB_if_idx = ctx->block->index;
7152    ic->BB_invert = Block();
7153    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7154    /* Invert blocks are intentionally not marked as top level because they
7155     * are not part of the logical cfg. */
7156    ic->BB_invert.kind |= block_kind_invert;
7157    ic->BB_endif = Block();
7158    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7159    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7160
7161    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7162    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7163    ctx->cf_info.parent_if.is_divergent = true;
7164    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7165
7166    /** emit logical then block */
7167    Block* BB_then_logical = ctx->program->create_and_insert_block();
7168    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7169    add_edge(ic->BB_if_idx, BB_then_logical);
7170    ctx->block = BB_then_logical;
7171    append_logical_start(BB_then_logical);
7172 }
7173
7174 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7175 {
7176    Block *BB_then_logical = ctx->block;
7177    append_logical_end(BB_then_logical);
7178     /* branch from logical then block to invert block */
7179    aco_ptr<Pseudo_branch_instruction> branch;
7180    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7181    BB_then_logical->instructions.emplace_back(std::move(branch));
7182    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7183    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7184       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7185    BB_then_logical->kind |= block_kind_uniform;
7186    assert(!ctx->cf_info.has_branch);
7187    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7188    ctx->cf_info.parent_loop.has_divergent_branch = false;
7189
7190    /** emit linear then block */
7191    Block* BB_then_linear = ctx->program->create_and_insert_block();
7192    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7193    BB_then_linear->kind |= block_kind_uniform;
7194    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7195    /* branch from linear then block to invert block */
7196    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7197    BB_then_linear->instructions.emplace_back(std::move(branch));
7198    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7199
7200    /** emit invert merge block */
7201    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7202    ic->invert_idx = ctx->block->index;
7203
7204    /* branch to linear else block (skip else) */
7205    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7206    branch->operands[0] = Operand(ic->cond);
7207    ctx->block->instructions.push_back(std::move(branch));
7208
7209    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7210    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7211
7212    /** emit logical else block */
7213    Block* BB_else_logical = ctx->program->create_and_insert_block();
7214    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7215    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7216    add_linear_edge(ic->invert_idx, BB_else_logical);
7217    ctx->block = BB_else_logical;
7218    append_logical_start(BB_else_logical);
7219 }
7220
7221 static void end_divergent_if(isel_context *ctx, if_context *ic)
7222 {
7223    Block *BB_else_logical = ctx->block;
7224    append_logical_end(BB_else_logical);
7225
7226    /* branch from logical else block to endif block */
7227    aco_ptr<Pseudo_branch_instruction> branch;
7228    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7229    BB_else_logical->instructions.emplace_back(std::move(branch));
7230    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7231    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7232       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7233    BB_else_logical->kind |= block_kind_uniform;
7234
7235    assert(!ctx->cf_info.has_branch);
7236    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7237
7238
7239    /** emit linear else block */
7240    Block* BB_else_linear = ctx->program->create_and_insert_block();
7241    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7242    BB_else_linear->kind |= block_kind_uniform;
7243    add_linear_edge(ic->invert_idx, BB_else_linear);
7244
7245    /* branch from linear else block to endif block */
7246    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7247    BB_else_linear->instructions.emplace_back(std::move(branch));
7248    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7249
7250
7251    /** emit endif merge block */
7252    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7253    append_logical_start(ctx->block);
7254
7255
7256    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7257    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7258    /* uniform control flow never has an empty exec-mask */
7259    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7260       ctx->cf_info.exec_potentially_empty = false;
7261 }
7262
7263 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7264 {
7265    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7266    Builder bld(ctx->program, ctx->block);
7267    aco_ptr<Pseudo_branch_instruction> branch;
7268
7269    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7270       /**
7271        * Uniform conditionals are represented in the following way*) :
7272        *
7273        * The linear and logical CFG:
7274        *                        BB_IF
7275        *                        /    \
7276        *       BB_THEN (logical)      BB_ELSE (logical)
7277        *                        \    /
7278        *                        BB_ENDIF
7279        *
7280        * *) Exceptions may be due to break and continue statements within loops
7281        *    If a break/continue happens within uniform control flow, it branches
7282        *    to the loop exit/entry block. Otherwise, it branches to the next
7283        *    merge block.
7284        **/
7285       append_logical_end(ctx->block);
7286       ctx->block->kind |= block_kind_uniform;
7287
7288       /* emit branch */
7289       if (cond.regClass() == s2) {
7290          // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7291          cond = as_uniform_bool(ctx, cond);
7292       }
7293       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7294       branch->operands[0] = Operand(cond);
7295       branch->operands[0].setFixed(scc);
7296       ctx->block->instructions.emplace_back(std::move(branch));
7297
7298       unsigned BB_if_idx = ctx->block->index;
7299       Block BB_endif = Block();
7300       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7301       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7302
7303       /** emit then block */
7304       Block* BB_then = ctx->program->create_and_insert_block();
7305       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7306       add_edge(BB_if_idx, BB_then);
7307       append_logical_start(BB_then);
7308       ctx->block = BB_then;
7309       visit_cf_list(ctx, &if_stmt->then_list);
7310       BB_then = ctx->block;
7311       bool then_branch = ctx->cf_info.has_branch;
7312       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7313
7314       if (!then_branch) {
7315          append_logical_end(BB_then);
7316          /* branch from then block to endif block */
7317          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7318          BB_then->instructions.emplace_back(std::move(branch));
7319          add_linear_edge(BB_then->index, &BB_endif);
7320          if (!then_branch_divergent)
7321             add_logical_edge(BB_then->index, &BB_endif);
7322          BB_then->kind |= block_kind_uniform;
7323       }
7324
7325       ctx->cf_info.has_branch = false;
7326       ctx->cf_info.parent_loop.has_divergent_branch = false;
7327
7328       /** emit else block */
7329       Block* BB_else = ctx->program->create_and_insert_block();
7330       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7331       add_edge(BB_if_idx, BB_else);
7332       append_logical_start(BB_else);
7333       ctx->block = BB_else;
7334       visit_cf_list(ctx, &if_stmt->else_list);
7335       BB_else = ctx->block;
7336
7337       if (!ctx->cf_info.has_branch) {
7338          append_logical_end(BB_else);
7339          /* branch from then block to endif block */
7340          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7341          BB_else->instructions.emplace_back(std::move(branch));
7342          add_linear_edge(BB_else->index, &BB_endif);
7343          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7344             add_logical_edge(BB_else->index, &BB_endif);
7345          BB_else->kind |= block_kind_uniform;
7346       }
7347
7348       ctx->cf_info.has_branch &= then_branch;
7349       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7350
7351       /** emit endif merge block */
7352       if (!ctx->cf_info.has_branch) {
7353          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7354          append_logical_start(ctx->block);
7355       }
7356    } else { /* non-uniform condition */
7357       /**
7358        * To maintain a logical and linear CFG without critical edges,
7359        * non-uniform conditionals are represented in the following way*) :
7360        *
7361        * The linear CFG:
7362        *                        BB_IF
7363        *                        /    \
7364        *       BB_THEN (logical)      BB_THEN (linear)
7365        *                        \    /
7366        *                        BB_INVERT (linear)
7367        *                        /    \
7368        *       BB_ELSE (logical)      BB_ELSE (linear)
7369        *                        \    /
7370        *                        BB_ENDIF
7371        *
7372        * The logical CFG:
7373        *                        BB_IF
7374        *                        /    \
7375        *       BB_THEN (logical)      BB_ELSE (logical)
7376        *                        \    /
7377        *                        BB_ENDIF
7378        *
7379        * *) Exceptions may be due to break and continue statements within loops
7380        **/
7381
7382       if_context ic;
7383
7384       begin_divergent_if_then(ctx, &ic, cond);
7385       visit_cf_list(ctx, &if_stmt->then_list);
7386
7387       begin_divergent_if_else(ctx, &ic);
7388       visit_cf_list(ctx, &if_stmt->else_list);
7389
7390       end_divergent_if(ctx, &ic);
7391    }
7392 }
7393
7394 static void visit_cf_list(isel_context *ctx,
7395                           struct exec_list *list)
7396 {
7397    foreach_list_typed(nir_cf_node, node, node, list) {
7398       switch (node->type) {
7399       case nir_cf_node_block:
7400          visit_block(ctx, nir_cf_node_as_block(node));
7401          break;
7402       case nir_cf_node_if:
7403          visit_if(ctx, nir_cf_node_as_if(node));
7404          break;
7405       case nir_cf_node_loop:
7406          visit_loop(ctx, nir_cf_node_as_loop(node));
7407          break;
7408       default:
7409          unreachable("unimplemented cf list type");
7410       }
7411    }
7412 }
7413
7414 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7415 {
7416    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7417    uint64_t mask = ctx->vs_output.mask[slot];
7418    if (!is_pos && !mask)
7419       return;
7420    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7421       return;
7422    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7423    exp->enabled_mask = mask;
7424    for (unsigned i = 0; i < 4; ++i) {
7425       if (mask & (1 << i))
7426          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7427       else
7428          exp->operands[i] = Operand(v1);
7429    }
7430    exp->valid_mask = false;
7431    exp->done = false;
7432    exp->compressed = false;
7433    if (is_pos)
7434       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7435    else
7436       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7437    ctx->block->instructions.emplace_back(std::move(exp));
7438 }
7439
7440 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7441 {
7442    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7443    exp->enabled_mask = 0;
7444    for (unsigned i = 0; i < 4; ++i)
7445       exp->operands[i] = Operand(v1);
7446    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7447       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7448       exp->enabled_mask |= 0x1;
7449    }
7450    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7451       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7452       exp->enabled_mask |= 0x4;
7453    }
7454    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7455       if (ctx->options->chip_class < GFX9) {
7456          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7457          exp->enabled_mask |= 0x8;
7458       } else {
7459          Builder bld(ctx->program, ctx->block);
7460
7461          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7462                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7463          if (exp->operands[2].isTemp())
7464             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7465
7466          exp->operands[2] = Operand(out);
7467          exp->enabled_mask |= 0x4;
7468       }
7469    }
7470    exp->valid_mask = false;
7471    exp->done = false;
7472    exp->compressed = false;
7473    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7474    ctx->block->instructions.emplace_back(std::move(exp));
7475 }
7476
7477 static void create_vs_exports(isel_context *ctx)
7478 {
7479    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7480
7481    if (outinfo->export_prim_id) {
7482       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7483       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7484    }
7485
7486    if (ctx->options->key.has_multiview_view_index) {
7487       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7488       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7489    }
7490
7491    /* the order these position exports are created is important */
7492    int next_pos = 0;
7493    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7494    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7495       export_vs_psiz_layer_viewport(ctx, &next_pos);
7496    }
7497    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7498       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7499    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7500       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7501
7502    if (ctx->options->key.vs_common_out.export_clip_dists) {
7503       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7504          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7505       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7506          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7507    }
7508
7509    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7510       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7511           i != VARYING_SLOT_PRIMITIVE_ID)
7512          continue;
7513
7514       export_vs_varying(ctx, i, false, NULL);
7515    }
7516 }
7517
7518 static void emit_stream_output(isel_context *ctx,
7519                                Temp const *so_buffers,
7520                                Temp const *so_write_offset,
7521                                const struct radv_stream_output *output)
7522 {
7523    unsigned num_comps = util_bitcount(output->component_mask);
7524    unsigned loc = output->location;
7525    unsigned buf = output->buffer;
7526    unsigned offset = output->offset;
7527
7528    assert(num_comps && num_comps <= 4);
7529    if (!num_comps || num_comps > 4)
7530       return;
7531
7532    unsigned start = ffs(output->component_mask) - 1;
7533
7534    Temp out[4];
7535    bool all_undef = true;
7536    assert(ctx->stage == vertex_vs);
7537    for (unsigned i = 0; i < num_comps; i++) {
7538       out[i] = ctx->vs_output.outputs[loc][start + i];
7539       all_undef = all_undef && !out[i].id();
7540    }
7541    if (all_undef)
7542       return;
7543
7544    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7545    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7546    for (unsigned i = 0; i < num_comps; ++i)
7547       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7548    vec->definitions[0] = Definition(write_data);
7549    ctx->block->instructions.emplace_back(std::move(vec));
7550
7551    aco_opcode opcode;
7552    switch (num_comps) {
7553    case 1:
7554       opcode = aco_opcode::buffer_store_dword;
7555       break;
7556    case 2:
7557       opcode = aco_opcode::buffer_store_dwordx2;
7558       break;
7559    case 3:
7560       opcode = aco_opcode::buffer_store_dwordx3;
7561       break;
7562    case 4:
7563       opcode = aco_opcode::buffer_store_dwordx4;
7564       break;
7565    }
7566
7567    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7568    store->operands[0] = Operand(so_write_offset[buf]);
7569    store->operands[1] = Operand(so_buffers[buf]);
7570    store->operands[2] = Operand((uint32_t) 0);
7571    store->operands[3] = Operand(write_data);
7572    if (offset > 4095) {
7573       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7574       Builder bld(ctx->program, ctx->block);
7575       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7576    } else {
7577       store->offset = offset;
7578    }
7579    store->offen = true;
7580    store->glc = true;
7581    store->dlc = false;
7582    store->slc = true;
7583    store->can_reorder = true;
7584    ctx->block->instructions.emplace_back(std::move(store));
7585 }
7586
7587 static void emit_streamout(isel_context *ctx, unsigned stream)
7588 {
7589    Builder bld(ctx->program, ctx->block);
7590
7591    Temp so_buffers[4];
7592    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7593    for (unsigned i = 0; i < 4; i++) {
7594       unsigned stride = ctx->program->info->so.strides[i];
7595       if (!stride)
7596          continue;
7597
7598       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7599    }
7600
7601    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7602                                 ctx->streamout_config, Operand(0x70010u));
7603
7604    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7605                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7606
7607    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7608
7609    if_context ic;
7610    begin_divergent_if_then(ctx, &ic, can_emit);
7611
7612    bld.reset(ctx->block);
7613
7614    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7615
7616    Temp so_write_offset[4];
7617
7618    for (unsigned i = 0; i < 4; i++) {
7619       unsigned stride = ctx->program->info->so.strides[i];
7620       if (!stride)
7621          continue;
7622
7623       if (stride == 1) {
7624          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7625                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7626          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7627
7628          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7629       } else {
7630          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7631          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7632          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7633       }
7634    }
7635
7636    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7637       struct radv_stream_output *output =
7638          &ctx->program->info->so.outputs[i];
7639       if (stream != output->stream)
7640          continue;
7641
7642       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7643    }
7644
7645    begin_divergent_if_else(ctx, &ic);
7646    end_divergent_if(ctx, &ic);
7647 }
7648
7649 } /* end namespace */
7650
7651 void handle_bc_optimize(isel_context *ctx)
7652 {
7653    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7654    Builder bld(ctx->program, ctx->block);
7655    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7656    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7657    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7658    if (uses_center && uses_centroid) {
7659       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7660
7661       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7662          for (unsigned i = 0; i < 2; i++) {
7663             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7664                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7665                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7666                                       sel);
7667             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7668          }
7669       }
7670
7671       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7672          for (unsigned i = 0; i < 2; i++) {
7673             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7674                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7675                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7676                                       sel);
7677             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7678          }
7679       }
7680    }
7681 }
7682
7683 void select_program(Program *program,
7684                     unsigned shader_count,
7685                     struct nir_shader *const *shaders,
7686                     ac_shader_config* config,
7687                     struct radv_shader_info *info,
7688                     struct radv_nir_compiler_options *options)
7689 {
7690    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7691
7692    for (unsigned i = 0; i < shader_count; i++) {
7693       nir_shader *nir = shaders[i];
7694       init_context(&ctx, nir);
7695
7696       if (!i) {
7697          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7698          append_logical_start(ctx.block);
7699       }
7700
7701       if_context ic;
7702       if (shader_count >= 2) {
7703          Builder bld(ctx.program, ctx.block);
7704          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7705          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7706                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7707          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7708
7709          begin_divergent_if_then(&ctx, &ic, cond);
7710       }
7711
7712       if (i) {
7713          Builder bld(ctx.program, ctx.block);
7714          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7715          bld.sopp(aco_opcode::s_barrier);
7716       }
7717
7718       if (ctx.stage == fragment_fs)
7719          handle_bc_optimize(&ctx);
7720
7721       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7722       visit_cf_list(&ctx, &func->body);
7723
7724       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7725          emit_streamout(&ctx, 0);
7726
7727       if (ctx.stage == vertex_vs)
7728          create_vs_exports(&ctx);
7729
7730       if (shader_count >= 2) {
7731          begin_divergent_if_else(&ctx, &ic);
7732          end_divergent_if(&ctx, &ic);
7733       }
7734
7735       ralloc_free(ctx.divergent_vals);
7736    }
7737
7738    append_logical_end(ctx.block);
7739    ctx.block->kind |= block_kind_uniform;
7740    Builder bld(ctx.program, ctx.block);
7741    if (ctx.program->wb_smem_l1_on_end)
7742       bld.smem(aco_opcode::s_dcache_wb, false);
7743    bld.sopp(aco_opcode::s_endpgm);
7744
7745    /* cleanup CFG */
7746    for (Block& BB : program->blocks) {
7747       for (unsigned idx : BB.linear_preds)
7748          program->blocks[idx].linear_succs.emplace_back(BB.index);
7749       for (unsigned idx : BB.logical_preds)
7750          program->blocks[idx].logical_succs.emplace_back(BB.index);
7751    }
7752 }
7753 }