src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 127 {
 128    Builder bld(ctx->program, ctx->block);
 129
 130    if (!dst.id())
 131       dst = bld.tmp(src.regClass());
 132
 133    if (ctx->stage != fragment_fs) {
 134       if (!dst.id())
 135          return src;
 136
 137       if (src.type() == RegType::vgpr || src.size() > 1)
 138          bld.copy(Definition(dst), src);
 139       else
 140          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 141       return dst;
 142    }
 143
 144    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 145    ctx->program->needs_wqm |= program_needs_wqm;
 146    return dst;
 147 }
 148
 149 Temp as_vgpr(isel_context *ctx, Temp val)
 150 {
 151    if (val.type() == RegType::sgpr) {
 152       Builder bld(ctx->program, ctx->block);
 153       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 154    }
 155    assert(val.type() == RegType::vgpr);
 156    return val;
 157 }
 158
 159 //assumes a != 0xffffffff
 160 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 161 {
 162    assert(b != 0);
 163    Builder bld(ctx->program, ctx->block);
 164
 165    if (util_is_power_of_two_or_zero(b)) {
 166       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 167       return;
 168    }
 169
 170    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 171
 172    assert(info.multiplier <= 0xffffffff);
 173
 174    bool pre_shift = info.pre_shift != 0;
 175    bool increment = info.increment != 0;
 176    bool multiply = true;
 177    bool post_shift = info.post_shift != 0;
 178
 179    if (!pre_shift && !increment && !multiply && !post_shift) {
 180       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 181       return;
 182    }
 183
 184    Temp pre_shift_dst = a;
 185    if (pre_shift) {
 186       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 187       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 188    }
 189
 190    Temp increment_dst = pre_shift_dst;
 191    if (increment) {
 192       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 193       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 194    }
 195
 196    Temp multiply_dst = increment_dst;
 197    if (multiply) {
 198       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 199       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 200                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 201    }
 202
 203    if (post_shift) {
 204       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 205    }
 206 }
 207
 208 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 209 {
 210    Builder bld(ctx->program, ctx->block);
 211    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 212 }
 213
 214
 215 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 216 {
 217    /* no need to extract the whole vector */
 218    if (src.regClass() == dst_rc) {
 219       assert(idx == 0);
 220       return src;
 221    }
 222    assert(src.size() > idx);
 223    Builder bld(ctx->program, ctx->block);
 224    auto it = ctx->allocated_vec.find(src.id());
 225    /* the size check needs to be early because elements other than 0 may be garbage */
 226    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 227       if (it->second[idx].regClass() == dst_rc) {
 228          return it->second[idx];
 229       } else {
 230          assert(dst_rc.size() == it->second[idx].regClass().size());
 231          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 232          return bld.copy(bld.def(dst_rc), it->second[idx]);
 233       }
 234    }
 235
 236    if (src.size() == dst_rc.size()) {
 237       assert(idx == 0);
 238       return bld.copy(bld.def(dst_rc), src);
 239    } else {
 240       Temp dst = bld.tmp(dst_rc);
 241       emit_extract_vector(ctx, src, idx, dst);
 242       return dst;
 243    }
 244 }
 245
 246 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 247 {
 248    if (num_components == 1)
 249       return;
 250    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 251       return;
 252    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 253    split->operands[0] = Operand(vec_src);
 254    std::array<Temp,4> elems;
 255    for (unsigned i = 0; i < num_components; i++) {
 256       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 257       split->definitions[i] = Definition(elems[i]);
 258    }
 259    ctx->block->instructions.emplace_back(std::move(split));
 260    ctx->allocated_vec.emplace(vec_src.id(), elems);
 261 }
 262
 263 /* This vector expansion uses a mask to determine which elements in the new vector
 264  * come from the original vector. The other elements are undefined. */
 265 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 266 {
 267    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 268
 269    if (vec_src == dst)
 270       return;
 271
 272    Builder bld(ctx->program, ctx->block);
 273    if (num_components == 1) {
 274       if (dst.type() == RegType::sgpr)
 275          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 276       else
 277          bld.copy(Definition(dst), vec_src);
 278       return;
 279    }
 280
 281    unsigned component_size = dst.size() / num_components;
 282    std::array<Temp,4> elems;
 283
 284    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 285    vec->definitions[0] = Definition(dst);
 286    unsigned k = 0;
 287    for (unsigned i = 0; i < num_components; i++) {
 288       if (mask & (1 << i)) {
 289          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 290          if (dst.type() == RegType::sgpr)
 291             src = bld.as_uniform(src);
 292          vec->operands[i] = Operand(src);
 293       } else {
 294          vec->operands[i] = Operand(0u);
 295       }
 296       elems[i] = vec->operands[i].getTemp();
 297    }
 298    ctx->block->instructions.emplace_back(std::move(vec));
 299    ctx->allocated_vec.emplace(dst.id(), elems);
 300 }
 301
 302 Temp as_divergent_bool(isel_context *ctx, Temp val, bool vcc_hint)
 303 {
 304    if (val.regClass() == s2) {
 305       return val;
 306    } else {
 307       assert(val.regClass() == s1);
 308       Builder bld(ctx->program, ctx->block);
 309       Definition& def = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2),
 310                                  Operand((uint32_t) -1), Operand(0u), bld.scc(val)).def(0);
 311       if (vcc_hint)
 312          def.setHint(vcc);
 313       return def.getTemp();
 314    }
 315 }
 316
 317 Temp as_uniform_bool(isel_context *ctx, Temp val)
 318 {
 319    if (val.regClass() == s1) {
 320       return val;
 321    } else {
 322       assert(val.regClass() == s2);
 323       Builder bld(ctx->program, ctx->block);
 324       return bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), Operand(0u), Operand(val));
 325    }
 326 }
 327
 328 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 329 {
 330    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 331       return get_ssa_temp(ctx, src.src.ssa);
 332
 333    if (src.src.ssa->num_components == size) {
 334       bool identity_swizzle = true;
 335       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 336          if (src.swizzle[i] != i)
 337             identity_swizzle = false;
 338       }
 339       if (identity_swizzle)
 340          return get_ssa_temp(ctx, src.src.ssa);
 341    }
 342
 343    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 344    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 345    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 346    assert(vec.size() % elem_size == 0);
 347
 348    RegClass elem_rc = RegClass(vec.type(), elem_size);
 349    if (size == 1) {
 350       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 351    } else {
 352       assert(size <= 4);
 353       std::array<Temp,4> elems;
 354       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 355       for (unsigned i = 0; i < size; ++i) {
 356          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 357          vec_instr->operands[i] = Operand{elems[i]};
 358       }
 359       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 360       vec_instr->definitions[0] = Definition(dst);
 361       ctx->block->instructions.emplace_back(std::move(vec_instr));
 362       ctx->allocated_vec.emplace(dst.id(), elems);
 363       return dst;
 364    }
 365 }
 366
 367 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 368 {
 369    if (ptr.size() == 2)
 370       return ptr;
 371    Builder bld(ctx->program, ctx->block);
 372    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 373                      ptr, Operand((unsigned)ctx->options->address32_hi));
 374 }
 375
 376 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 377 {
 378    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 379    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 380    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 381    sop2->definitions[0] = Definition(dst);
 382    if (writes_scc)
 383       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 384    ctx->block->instructions.emplace_back(std::move(sop2));
 385 }
 386
 387 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 388 {
 389    Builder bld(ctx->program, ctx->block);
 390    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 391    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 392    if (src1.type() == RegType::sgpr) {
 393       if (commutative && src0.type() == RegType::vgpr) {
 394          Temp t = src0;
 395          src0 = src1;
 396          src1 = t;
 397       } else if (src0.type() == RegType::vgpr &&
 398                  op != aco_opcode::v_madmk_f32 &&
 399                  op != aco_opcode::v_madak_f32 &&
 400                  op != aco_opcode::v_madmk_f16 &&
 401                  op != aco_opcode::v_madak_f16) {
 402          /* If the instruction is not commutative, we emit a VOP3A instruction */
 403          bld.vop2_e64(op, Definition(dst), src0, src1);
 404          return;
 405       } else {
 406          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 407       }
 408    }
 409    bld.vop2(op, Definition(dst), src0, src1);
 410 }
 411
 412 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 413 {
 414    Temp src0 = get_alu_src(ctx, instr->src[0]);
 415    Temp src1 = get_alu_src(ctx, instr->src[1]);
 416    Temp src2 = get_alu_src(ctx, instr->src[2]);
 417
 418    /* ensure that the instruction has at most 1 sgpr operand
 419     * The optimizer will inline constants for us */
 420    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 421       src0 = as_vgpr(ctx, src0);
 422    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 423       src1 = as_vgpr(ctx, src1);
 424    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 425       src2 = as_vgpr(ctx, src2);
 426
 427    Builder bld(ctx->program, ctx->block);
 428    bld.vop3(op, Definition(dst), src0, src1, src2);
 429 }
 430
 431 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 432 {
 433    Builder bld(ctx->program, ctx->block);
 434    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 435 }
 436
 437 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 438 {
 439    Temp src0 = get_alu_src(ctx, instr->src[0]);
 440    Temp src1 = get_alu_src(ctx, instr->src[1]);
 441    aco_ptr<Instruction> vopc;
 442    if (src1.type() == RegType::sgpr) {
 443       if (src0.type() == RegType::vgpr) {
 444          /* to swap the operands, we might also have to change the opcode */
 445          switch (op) {
 446             case aco_opcode::v_cmp_lt_f32:
 447                op = aco_opcode::v_cmp_gt_f32;
 448                break;
 449             case aco_opcode::v_cmp_ge_f32:
 450                op = aco_opcode::v_cmp_le_f32;
 451                break;
 452             case aco_opcode::v_cmp_lt_i32:
 453                op = aco_opcode::v_cmp_gt_i32;
 454                break;
 455             case aco_opcode::v_cmp_ge_i32:
 456                op = aco_opcode::v_cmp_le_i32;
 457                break;
 458             case aco_opcode::v_cmp_lt_u32:
 459                op = aco_opcode::v_cmp_gt_u32;
 460                break;
 461             case aco_opcode::v_cmp_ge_u32:
 462                op = aco_opcode::v_cmp_le_u32;
 463                break;
 464             case aco_opcode::v_cmp_lt_f64:
 465                op = aco_opcode::v_cmp_gt_f64;
 466                break;
 467             case aco_opcode::v_cmp_ge_f64:
 468                op = aco_opcode::v_cmp_le_f64;
 469                break;
 470             case aco_opcode::v_cmp_lt_i64:
 471                op = aco_opcode::v_cmp_gt_i64;
 472                break;
 473             case aco_opcode::v_cmp_ge_i64:
 474                op = aco_opcode::v_cmp_le_i64;
 475                break;
 476             case aco_opcode::v_cmp_lt_u64:
 477                op = aco_opcode::v_cmp_gt_u64;
 478                break;
 479             case aco_opcode::v_cmp_ge_u64:
 480                op = aco_opcode::v_cmp_le_u64;
 481                break;
 482             default: /* eq and ne are commutative */
 483                break;
 484          }
 485          Temp t = src0;
 486          src0 = src1;
 487          src1 = t;
 488       } else {
 489          src1 = as_vgpr(ctx, src1);
 490       }
 491    }
 492    Builder bld(ctx->program, ctx->block);
 493    bld.vopc(op, Definition(dst), src0, src1).def(0).setHint(vcc);
 494 }
 495
 496 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 497 {
 498    if (dst.regClass() == s2) {
 499       emit_vopc_instruction(ctx, instr, op, dst);
 500       if (!ctx->divergent_vals[instr->dest.dest.ssa.index])
 501          emit_split_vector(ctx, dst, 2);
 502    } else if (dst.regClass() == s1) {
 503       Temp src0 = get_alu_src(ctx, instr->src[0]);
 504       Temp src1 = get_alu_src(ctx, instr->src[1]);
 505       assert(src0.type() == RegType::sgpr && src1.type() == RegType::sgpr);
 506
 507       Builder bld(ctx->program, ctx->block);
 508       bld.sopc(op, bld.scc(Definition(dst)), src0, src1);
 509
 510    } else {
 511       assert(false);
 512    }
 513 }
 514
 515 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 516 {
 517    Builder bld(ctx->program, ctx->block);
 518    Temp src0 = get_alu_src(ctx, instr->src[0]);
 519    Temp src1 = get_alu_src(ctx, instr->src[1]);
 520    if (dst.regClass() == s2) {
 521       bld.sop2(op64, Definition(dst), bld.def(s1, scc),
 522                as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
 523    } else {
 524       assert(dst.regClass() == s1);
 525       bld.sop2(op32, bld.def(s1), bld.scc(Definition(dst)),
 526                as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
 527    }
 528 }
 529
 530
 531 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 532 {
 533    Builder bld(ctx->program, ctx->block);
 534    Temp cond = get_alu_src(ctx, instr->src[0]);
 535    Temp then = get_alu_src(ctx, instr->src[1]);
 536    Temp els = get_alu_src(ctx, instr->src[2]);
 537
 538    if (dst.type() == RegType::vgpr) {
 539       cond = as_divergent_bool(ctx, cond, true);
 540
 541       aco_ptr<Instruction> bcsel;
 542       if (dst.size() == 1) {
 543          then = as_vgpr(ctx, then);
 544          els = as_vgpr(ctx, els);
 545
 546          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 547       } else if (dst.size() == 2) {
 548          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 549          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 550          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 551          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 552
 553          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 554          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 555
 556          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 557       } else {
 558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 559          nir_print_instr(&instr->instr, stderr);
 560          fprintf(stderr, "\n");
 561       }
 562       return;
 563    }
 564
 565    if (instr->dest.dest.ssa.bit_size != 1) { /* uniform condition and values in sgpr */
 566       if (dst.regClass() == s1 || dst.regClass() == s2) {
 567          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 568          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 569          bld.sop2(op, Definition(dst), then, els, bld.scc(as_uniform_bool(ctx, cond)));
 570       } else {
 571          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 572          nir_print_instr(&instr->instr, stderr);
 573          fprintf(stderr, "\n");
 574       }
 575       return;
 576    }
 577
 578    /* boolean bcsel */
 579    assert(instr->dest.dest.ssa.bit_size == 1);
 580
 581    if (dst.regClass() == s1)
 582       cond = as_uniform_bool(ctx, cond);
 583
 584    if (cond.regClass() == s1) { /* uniform selection */
 585       aco_opcode op;
 586       if (dst.regClass() == s2) {
 587          op = aco_opcode::s_cselect_b64;
 588          then = as_divergent_bool(ctx, then, false);
 589          els = as_divergent_bool(ctx, els, false);
 590       } else {
 591          assert(dst.regClass() == s1);
 592          op = aco_opcode::s_cselect_b32;
 593          then = as_uniform_bool(ctx, then);
 594          els = as_uniform_bool(ctx, els);
 595       }
 596       bld.sop2(op, Definition(dst), then, els, bld.scc(cond));
 597       return;
 598    }
 599
 600    /* divergent boolean bcsel
 601     * this implements bcsel on bools: dst = s0 ? s1 : s2
 602     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 603    assert (dst.regClass() == s2);
 604    then = as_divergent_bool(ctx, then, false);
 605    els = as_divergent_bool(ctx, els, false);
 606
 607    if (cond.id() != then.id())
 608       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 609
 610    if (cond.id() == els.id())
 611       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 612    else
 613       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 614                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 615 }
 616
 617 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 618 {
 619    if (!instr->dest.dest.is_ssa) {
 620       fprintf(stderr, "nir alu dst not in ssa: ");
 621       nir_print_instr(&instr->instr, stderr);
 622       fprintf(stderr, "\n");
 623       abort();
 624    }
 625    Builder bld(ctx->program, ctx->block);
 626    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 627    switch(instr->op) {
 628    case nir_op_vec2:
 629    case nir_op_vec3:
 630    case nir_op_vec4: {
 631       std::array<Temp,4> elems;
 632       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 633       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 634          elems[i] = get_alu_src(ctx, instr->src[i]);
 635          vec->operands[i] = Operand{elems[i]};
 636       }
 637       vec->definitions[0] = Definition(dst);
 638       ctx->block->instructions.emplace_back(std::move(vec));
 639       ctx->allocated_vec.emplace(dst.id(), elems);
 640       break;
 641    }
 642    case nir_op_mov: {
 643       Temp src = get_alu_src(ctx, instr->src[0]);
 644       aco_ptr<Instruction> mov;
 645       if (dst.type() == RegType::sgpr) {
 646          if (src.type() == RegType::vgpr)
 647             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 648          else if (src.regClass() == s1)
 649             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 650          else if (src.regClass() == s2)
 651             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 652          else
 653             unreachable("wrong src register class for nir_op_imov");
 654       } else if (dst.regClass() == v1) {
 655          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 656       } else if (dst.regClass() == v2) {
 657          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 658       } else {
 659          nir_print_instr(&instr->instr, stderr);
 660          unreachable("Should have been lowered to scalar.");
 661       }
 662       break;
 663    }
 664    case nir_op_inot: {
 665       Temp src = get_alu_src(ctx, instr->src[0]);
 666       /* uniform booleans */
 667       if (instr->dest.dest.ssa.bit_size == 1 && dst.regClass() == s1) {
 668          if (src.regClass() == s1) {
 669             /* in this case, src is either 1 or 0 */
 670             bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.scc(Definition(dst)), Operand(1u), src);
 671          } else {
 672             /* src is either exec_mask or 0 */
 673             assert(src.regClass() == s2);
 674             bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(Definition(dst)), Operand(0u), src);
 675          }
 676       } else if (dst.regClass() == v1) {
 677          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 678       } else if (dst.type() == RegType::sgpr) {
 679          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 680          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 681       } else {
 682          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 683          nir_print_instr(&instr->instr, stderr);
 684          fprintf(stderr, "\n");
 685       }
 686       break;
 687    }
 688    case nir_op_ineg: {
 689       Temp src = get_alu_src(ctx, instr->src[0]);
 690       if (dst.regClass() == v1) {
 691          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 692       } else if (dst.regClass() == s1) {
 693          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 694       } else if (dst.size() == 2) {
 695          Temp src0 = bld.tmp(dst.type(), 1);
 696          Temp src1 = bld.tmp(dst.type(), 1);
 697          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 698
 699          if (dst.regClass() == s2) {
 700             Temp carry = bld.tmp(s1);
 701             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 702             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 703             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 704          } else {
 705             Temp lower = bld.tmp(v1);
 706             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 707             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 708             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 709          }
 710       } else {
 711          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 712          nir_print_instr(&instr->instr, stderr);
 713          fprintf(stderr, "\n");
 714       }
 715       break;
 716    }
 717    case nir_op_iabs: {
 718       if (dst.regClass() == s1) {
 719          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 720       } else if (dst.regClass() == v1) {
 721          Temp src = get_alu_src(ctx, instr->src[0]);
 722          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 723       } else {
 724          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 725          nir_print_instr(&instr->instr, stderr);
 726          fprintf(stderr, "\n");
 727       }
 728       break;
 729    }
 730    case nir_op_isign: {
 731       Temp src = get_alu_src(ctx, instr->src[0]);
 732       if (dst.regClass() == s1) {
 733          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 734          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 735          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 736       } else if (dst.regClass() == s2) {
 737          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 738          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 739          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 740       } else if (dst.regClass() == v1) {
 741          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 742          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 743          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 744       } else if (dst.regClass() == v2) {
 745          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 746          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 747          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 748          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 749          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 750          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 751       } else {
 752          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 753          nir_print_instr(&instr->instr, stderr);
 754          fprintf(stderr, "\n");
 755       }
 756       break;
 757    }
 758    case nir_op_imax: {
 759       if (dst.regClass() == v1) {
 760          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 761       } else if (dst.regClass() == s1) {
 762          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 763       } else {
 764          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 765          nir_print_instr(&instr->instr, stderr);
 766          fprintf(stderr, "\n");
 767       }
 768       break;
 769    }
 770    case nir_op_umax: {
 771       if (dst.regClass() == v1) {
 772          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 773       } else if (dst.regClass() == s1) {
 774          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 775       } else {
 776          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 777          nir_print_instr(&instr->instr, stderr);
 778          fprintf(stderr, "\n");
 779       }
 780       break;
 781    }
 782    case nir_op_imin: {
 783       if (dst.regClass() == v1) {
 784          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 785       } else if (dst.regClass() == s1) {
 786          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 787       } else {
 788          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 789          nir_print_instr(&instr->instr, stderr);
 790          fprintf(stderr, "\n");
 791       }
 792       break;
 793    }
 794    case nir_op_umin: {
 795       if (dst.regClass() == v1) {
 796          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 797       } else if (dst.regClass() == s1) {
 798          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 799       } else {
 800          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 801          nir_print_instr(&instr->instr, stderr);
 802          fprintf(stderr, "\n");
 803       }
 804       break;
 805    }
 806    case nir_op_ior: {
 807       if (instr->dest.dest.ssa.bit_size == 1) {
 808          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 809       } else if (dst.regClass() == v1) {
 810          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 811       } else if (dst.regClass() == s1) {
 812          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 813       } else if (dst.regClass() == s2) {
 814          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 815       } else {
 816          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 817          nir_print_instr(&instr->instr, stderr);
 818          fprintf(stderr, "\n");
 819       }
 820       break;
 821    }
 822    case nir_op_iand: {
 823       if (instr->dest.dest.ssa.bit_size == 1) {
 824          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 825       } else if (dst.regClass() == v1) {
 826          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 827       } else if (dst.regClass() == s1) {
 828          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 829       } else if (dst.regClass() == s2) {
 830          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 831       } else {
 832          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 833          nir_print_instr(&instr->instr, stderr);
 834          fprintf(stderr, "\n");
 835       }
 836       break;
 837    }
 838    case nir_op_ixor: {
 839       if (instr->dest.dest.ssa.bit_size == 1) {
 840          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 841       } else if (dst.regClass() == v1) {
 842          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 843       } else if (dst.regClass() == s1) {
 844          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 845       } else if (dst.regClass() == s2) {
 846          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 847       } else {
 848          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 849          nir_print_instr(&instr->instr, stderr);
 850          fprintf(stderr, "\n");
 851       }
 852       break;
 853    }
 854    case nir_op_ushr: {
 855       if (dst.regClass() == v1) {
 856          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 857       } else if (dst.regClass() == v2) {
 858          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 859                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 860       } else if (dst.regClass() == s2) {
 861          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 862       } else if (dst.regClass() == s1) {
 863          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 864       } else {
 865          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 866          nir_print_instr(&instr->instr, stderr);
 867          fprintf(stderr, "\n");
 868       }
 869       break;
 870    }
 871    case nir_op_ishl: {
 872       if (dst.regClass() == v1) {
 873          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 874       } else if (dst.regClass() == v2) {
 875          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 876                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 877       } else if (dst.regClass() == s1) {
 878          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 879       } else if (dst.regClass() == s2) {
 880          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 881       } else {
 882          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 883          nir_print_instr(&instr->instr, stderr);
 884          fprintf(stderr, "\n");
 885       }
 886       break;
 887    }
 888    case nir_op_ishr: {
 889       if (dst.regClass() == v1) {
 890          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 891       } else if (dst.regClass() == v2) {
 892          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 893                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 894       } else if (dst.regClass() == s1) {
 895          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 896       } else if (dst.regClass() == s2) {
 897          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 898       } else {
 899          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 900          nir_print_instr(&instr->instr, stderr);
 901          fprintf(stderr, "\n");
 902       }
 903       break;
 904    }
 905    case nir_op_find_lsb: {
 906       Temp src = get_alu_src(ctx, instr->src[0]);
 907       if (src.regClass() == s1) {
 908          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 909       } else if (src.regClass() == v1) {
 910          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 911       } else if (src.regClass() == s2) {
 912          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 913       } else {
 914          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 915          nir_print_instr(&instr->instr, stderr);
 916          fprintf(stderr, "\n");
 917       }
 918       break;
 919    }
 920    case nir_op_ufind_msb:
 921    case nir_op_ifind_msb: {
 922       Temp src = get_alu_src(ctx, instr->src[0]);
 923       if (src.regClass() == s1 || src.regClass() == s2) {
 924          aco_opcode op = src.regClass() == s2 ?
 925                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 926                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 927          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 928
 929          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 930                                         Operand(src.size() * 32u - 1u), msb_rev);
 931          Temp msb = sub.def(0).getTemp();
 932          Temp carry = sub.def(1).getTemp();
 933
 934          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 935       } else if (src.regClass() == v1) {
 936          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 937          Temp msb_rev = bld.tmp(v1);
 938          emit_vop1_instruction(ctx, instr, op, msb_rev);
 939          Temp msb = bld.tmp(v1);
 940          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 941          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 942       } else {
 943          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 944          nir_print_instr(&instr->instr, stderr);
 945          fprintf(stderr, "\n");
 946       }
 947       break;
 948    }
 949    case nir_op_bitfield_reverse: {
 950       if (dst.regClass() == s1) {
 951          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 952       } else if (dst.regClass() == v1) {
 953          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 954       } else {
 955          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 956          nir_print_instr(&instr->instr, stderr);
 957          fprintf(stderr, "\n");
 958       }
 959       break;
 960    }
 961    case nir_op_iadd: {
 962       if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
 964          break;
 965       }
 966
 967       Temp src0 = get_alu_src(ctx, instr->src[0]);
 968       Temp src1 = get_alu_src(ctx, instr->src[1]);
 969       if (dst.regClass() == v1) {
 970          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
 971          break;
 972       }
 973
 974       assert(src0.size() == 2 && src1.size() == 2);
 975       Temp src00 = bld.tmp(src0.type(), 1);
 976       Temp src01 = bld.tmp(dst.type(), 1);
 977       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
 978       Temp src10 = bld.tmp(src1.type(), 1);
 979       Temp src11 = bld.tmp(dst.type(), 1);
 980       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
 981
 982       if (dst.regClass() == s2) {
 983          Temp carry = bld.tmp(s1);
 984          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
 985          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
 986          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 987       } else if (dst.regClass() == v2) {
 988          Temp dst0 = bld.tmp(v1);
 989          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
 990          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
 991          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 992       } else {
 993          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 994          nir_print_instr(&instr->instr, stderr);
 995          fprintf(stderr, "\n");
 996       }
 997       break;
 998    }
 999    case nir_op_uadd_sat: {
1000       Temp src0 = get_alu_src(ctx, instr->src[0]);
1001       Temp src1 = get_alu_src(ctx, instr->src[1]);
1002       if (dst.regClass() == s1) {
1003          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1004          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1005                   src0, src1);
1006          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1007       } else if (dst.regClass() == v1) {
1008          if (ctx->options->chip_class >= GFX9) {
1009             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1010             add->operands[0] = Operand(src0);
1011             add->operands[1] = Operand(src1);
1012             add->definitions[0] = Definition(dst);
1013             add->clamp = 1;
1014             ctx->block->instructions.emplace_back(std::move(add));
1015          } else {
1016             if (src1.regClass() != v1)
1017                std::swap(src0, src1);
1018             assert(src1.regClass() == v1);
1019             Temp tmp = bld.tmp(v1);
1020             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1021             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1022          }
1023       } else {
1024          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1025          nir_print_instr(&instr->instr, stderr);
1026          fprintf(stderr, "\n");
1027       }
1028       break;
1029    }
1030    case nir_op_uadd_carry: {
1031       Temp src0 = get_alu_src(ctx, instr->src[0]);
1032       Temp src1 = get_alu_src(ctx, instr->src[1]);
1033       if (dst.regClass() == s1) {
1034          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1035          break;
1036       }
1037       if (dst.regClass() == v1) {
1038          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1039          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1040          break;
1041       }
1042
1043       Temp src00 = bld.tmp(src0.type(), 1);
1044       Temp src01 = bld.tmp(dst.type(), 1);
1045       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1046       Temp src10 = bld.tmp(src1.type(), 1);
1047       Temp src11 = bld.tmp(dst.type(), 1);
1048       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1049       if (dst.regClass() == s2) {
1050          Temp carry = bld.tmp(s1);
1051          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1052          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1053          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1054       } else if (dst.regClass() == v2) {
1055          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1056          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1057          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1058          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1059       } else {
1060          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1061          nir_print_instr(&instr->instr, stderr);
1062          fprintf(stderr, "\n");
1063       }
1064       break;
1065    }
1066    case nir_op_isub: {
1067       if (dst.regClass() == s1) {
1068          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1069          break;
1070       }
1071
1072       Temp src0 = get_alu_src(ctx, instr->src[0]);
1073       Temp src1 = get_alu_src(ctx, instr->src[1]);
1074       if (dst.regClass() == v1) {
1075          bld.vsub32(Definition(dst), src0, src1);
1076          break;
1077       }
1078
1079       Temp src00 = bld.tmp(src0.type(), 1);
1080       Temp src01 = bld.tmp(dst.type(), 1);
1081       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1082       Temp src10 = bld.tmp(src1.type(), 1);
1083       Temp src11 = bld.tmp(dst.type(), 1);
1084       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1085       if (dst.regClass() == s2) {
1086          Temp carry = bld.tmp(s1);
1087          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1088          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1089          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1090       } else if (dst.regClass() == v2) {
1091          Temp lower = bld.tmp(v1);
1092          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1093          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1094          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1095       } else {
1096          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1097          nir_print_instr(&instr->instr, stderr);
1098          fprintf(stderr, "\n");
1099       }
1100       break;
1101    }
1102    case nir_op_usub_borrow: {
1103       Temp src0 = get_alu_src(ctx, instr->src[0]);
1104       Temp src1 = get_alu_src(ctx, instr->src[1]);
1105       if (dst.regClass() == s1) {
1106          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1107          break;
1108       } else if (dst.regClass() == v1) {
1109          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1110          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1111          break;
1112       }
1113
1114       Temp src00 = bld.tmp(src0.type(), 1);
1115       Temp src01 = bld.tmp(dst.type(), 1);
1116       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1117       Temp src10 = bld.tmp(src1.type(), 1);
1118       Temp src11 = bld.tmp(dst.type(), 1);
1119       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1120       if (dst.regClass() == s2) {
1121          Temp borrow = bld.tmp(s1);
1122          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1123          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1124          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1125       } else if (dst.regClass() == v2) {
1126          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1127          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1128          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1129          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1130       } else {
1131          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1132          nir_print_instr(&instr->instr, stderr);
1133          fprintf(stderr, "\n");
1134       }
1135       break;
1136    }
1137    case nir_op_imul: {
1138       if (dst.regClass() == v1) {
1139          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1140                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1141       } else if (dst.regClass() == s1) {
1142          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1143       } else {
1144          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1145          nir_print_instr(&instr->instr, stderr);
1146          fprintf(stderr, "\n");
1147       }
1148       break;
1149    }
1150    case nir_op_umul_high: {
1151       if (dst.regClass() == v1) {
1152          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1153       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1154          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1155       } else if (dst.regClass() == s1) {
1156          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1157                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1158          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1159       } else {
1160          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1161          nir_print_instr(&instr->instr, stderr);
1162          fprintf(stderr, "\n");
1163       }
1164       break;
1165    }
1166    case nir_op_imul_high: {
1167       if (dst.regClass() == v1) {
1168          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1169       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1170          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1171       } else if (dst.regClass() == s1) {
1172          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1173                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1174          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1175       } else {
1176          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1177          nir_print_instr(&instr->instr, stderr);
1178          fprintf(stderr, "\n");
1179       }
1180       break;
1181    }
1182    case nir_op_fmul: {
1183       if (dst.size() == 1) {
1184          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1185       } else if (dst.size() == 2) {
1186          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1187                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1188       } else {
1189          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1190          nir_print_instr(&instr->instr, stderr);
1191          fprintf(stderr, "\n");
1192       }
1193       break;
1194    }
1195    case nir_op_fadd: {
1196       if (dst.size() == 1) {
1197          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1198       } else if (dst.size() == 2) {
1199          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1200                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1201       } else {
1202          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1203          nir_print_instr(&instr->instr, stderr);
1204          fprintf(stderr, "\n");
1205       }
1206       break;
1207    }
1208    case nir_op_fsub: {
1209       Temp src0 = get_alu_src(ctx, instr->src[0]);
1210       Temp src1 = get_alu_src(ctx, instr->src[1]);
1211       if (dst.size() == 1) {
1212          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1213             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1214          else
1215             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1216       } else if (dst.size() == 2) {
1217          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1218                                      get_alu_src(ctx, instr->src[0]),
1219                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1220          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1221          sub->neg[1] = true;
1222       } else {
1223          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1224          nir_print_instr(&instr->instr, stderr);
1225          fprintf(stderr, "\n");
1226       }
1227       break;
1228    }
1229    case nir_op_fmax: {
1230       if (dst.size() == 1) {
1231          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1232       } else if (dst.size() == 2) {
1233          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1234                   get_alu_src(ctx, instr->src[0]),
1235                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1236       } else {
1237          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1238          nir_print_instr(&instr->instr, stderr);
1239          fprintf(stderr, "\n");
1240       }
1241       break;
1242    }
1243    case nir_op_fmin: {
1244       if (dst.size() == 1) {
1245          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1246       } else if (dst.size() == 2) {
1247          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1248                   get_alu_src(ctx, instr->src[0]),
1249                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1250       } else {
1251          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1252          nir_print_instr(&instr->instr, stderr);
1253          fprintf(stderr, "\n");
1254       }
1255       break;
1256    }
1257    case nir_op_fmax3: {
1258       if (dst.size() == 1) {
1259          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1260       } else {
1261          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1262          nir_print_instr(&instr->instr, stderr);
1263          fprintf(stderr, "\n");
1264       }
1265       break;
1266    }
1267    case nir_op_fmin3: {
1268       if (dst.size() == 1) {
1269          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1270       } else {
1271          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1272          nir_print_instr(&instr->instr, stderr);
1273          fprintf(stderr, "\n");
1274       }
1275       break;
1276    }
1277    case nir_op_fmed3: {
1278       if (dst.size() == 1) {
1279          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1280       } else {
1281          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1282          nir_print_instr(&instr->instr, stderr);
1283          fprintf(stderr, "\n");
1284       }
1285       break;
1286    }
1287    case nir_op_umax3: {
1288       if (dst.size() == 1) {
1289          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1290       } else {
1291          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1292          nir_print_instr(&instr->instr, stderr);
1293          fprintf(stderr, "\n");
1294       }
1295       break;
1296    }
1297    case nir_op_umin3: {
1298       if (dst.size() == 1) {
1299          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1300       } else {
1301          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1302          nir_print_instr(&instr->instr, stderr);
1303          fprintf(stderr, "\n");
1304       }
1305       break;
1306    }
1307    case nir_op_umed3: {
1308       if (dst.size() == 1) {
1309          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1310       } else {
1311          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1312          nir_print_instr(&instr->instr, stderr);
1313          fprintf(stderr, "\n");
1314       }
1315       break;
1316    }
1317    case nir_op_imax3: {
1318       if (dst.size() == 1) {
1319          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1320       } else {
1321          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1322          nir_print_instr(&instr->instr, stderr);
1323          fprintf(stderr, "\n");
1324       }
1325       break;
1326    }
1327    case nir_op_imin3: {
1328       if (dst.size() == 1) {
1329          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1330       } else {
1331          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1332          nir_print_instr(&instr->instr, stderr);
1333          fprintf(stderr, "\n");
1334       }
1335       break;
1336    }
1337    case nir_op_imed3: {
1338       if (dst.size() == 1) {
1339          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1340       } else {
1341          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1342          nir_print_instr(&instr->instr, stderr);
1343          fprintf(stderr, "\n");
1344       }
1345       break;
1346    }
1347    case nir_op_cube_face_coord: {
1348       Temp in = get_alu_src(ctx, instr->src[0], 3);
1349       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1350                       emit_extract_vector(ctx, in, 1, v1),
1351                       emit_extract_vector(ctx, in, 2, v1) };
1352       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1353       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1354       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1355       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1356       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1357       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1358       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1359       break;
1360    }
1361    case nir_op_cube_face_index: {
1362       Temp in = get_alu_src(ctx, instr->src[0], 3);
1363       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1364                       emit_extract_vector(ctx, in, 1, v1),
1365                       emit_extract_vector(ctx, in, 2, v1) };
1366       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1367       break;
1368    }
1369    case nir_op_bcsel: {
1370       emit_bcsel(ctx, instr, dst);
1371       break;
1372    }
1373    case nir_op_frsq: {
1374       if (dst.size() == 1) {
1375          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1376       } else if (dst.size() == 2) {
1377          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1378       } else {
1379          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1380          nir_print_instr(&instr->instr, stderr);
1381          fprintf(stderr, "\n");
1382       }
1383       break;
1384    }
1385    case nir_op_fneg: {
1386       Temp src = get_alu_src(ctx, instr->src[0]);
1387       if (dst.size() == 1) {
1388          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1389       } else if (dst.size() == 2) {
1390          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1391          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1392          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1393          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1394       } else {
1395          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1396          nir_print_instr(&instr->instr, stderr);
1397          fprintf(stderr, "\n");
1398       }
1399       break;
1400    }
1401    case nir_op_fabs: {
1402       Temp src = get_alu_src(ctx, instr->src[0]);
1403       if (dst.size() == 1) {
1404          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1405       } else if (dst.size() == 2) {
1406          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1407          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1408          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1409          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1410       } else {
1411          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1412          nir_print_instr(&instr->instr, stderr);
1413          fprintf(stderr, "\n");
1414       }
1415       break;
1416    }
1417    case nir_op_fsat: {
1418       Temp src = get_alu_src(ctx, instr->src[0]);
1419       if (dst.size() == 1) {
1420          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1421       } else if (dst.size() == 2) {
1422          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1423          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1424          vop3->clamp = true;
1425       } else {
1426          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1427          nir_print_instr(&instr->instr, stderr);
1428          fprintf(stderr, "\n");
1429       }
1430       break;
1431    }
1432    case nir_op_flog2: {
1433       if (dst.size() == 1) {
1434          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1435       } else {
1436          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1437          nir_print_instr(&instr->instr, stderr);
1438          fprintf(stderr, "\n");
1439       }
1440       break;
1441    }
1442    case nir_op_frcp: {
1443       if (dst.size() == 1) {
1444          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1445       } else if (dst.size() == 2) {
1446          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1447       } else {
1448          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1449          nir_print_instr(&instr->instr, stderr);
1450          fprintf(stderr, "\n");
1451       }
1452       break;
1453    }
1454    case nir_op_fexp2: {
1455       if (dst.size() == 1) {
1456          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1457       } else {
1458          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1459          nir_print_instr(&instr->instr, stderr);
1460          fprintf(stderr, "\n");
1461       }
1462       break;
1463    }
1464    case nir_op_fsqrt: {
1465       if (dst.size() == 1) {
1466          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1467       } else if (dst.size() == 2) {
1468          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1469       } else {
1470          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1471          nir_print_instr(&instr->instr, stderr);
1472          fprintf(stderr, "\n");
1473       }
1474       break;
1475    }
1476    case nir_op_ffract: {
1477       if (dst.size() == 1) {
1478          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1479       } else if (dst.size() == 2) {
1480          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1481       } else {
1482          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1483          nir_print_instr(&instr->instr, stderr);
1484          fprintf(stderr, "\n");
1485       }
1486       break;
1487    }
1488    case nir_op_ffloor: {
1489       if (dst.size() == 1) {
1490          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1491       } else if (dst.size() == 2) {
1492          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1493       } else {
1494          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1495          nir_print_instr(&instr->instr, stderr);
1496          fprintf(stderr, "\n");
1497       }
1498       break;
1499    }
1500    case nir_op_fceil: {
1501       if (dst.size() == 1) {
1502          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1503       } else if (dst.size() == 2) {
1504          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1505       } else {
1506          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1507          nir_print_instr(&instr->instr, stderr);
1508          fprintf(stderr, "\n");
1509       }
1510       break;
1511    }
1512    case nir_op_ftrunc: {
1513       if (dst.size() == 1) {
1514          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1515       } else if (dst.size() == 2) {
1516          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1517       } else {
1518          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1519          nir_print_instr(&instr->instr, stderr);
1520          fprintf(stderr, "\n");
1521       }
1522       break;
1523    }
1524    case nir_op_fround_even: {
1525       if (dst.size() == 1) {
1526          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1527       } else if (dst.size() == 2) {
1528          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1529       } else {
1530          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1531          nir_print_instr(&instr->instr, stderr);
1532          fprintf(stderr, "\n");
1533       }
1534       break;
1535    }
1536    case nir_op_fsin:
1537    case nir_op_fcos: {
1538       Temp src = get_alu_src(ctx, instr->src[0]);
1539       aco_ptr<Instruction> norm;
1540       if (dst.size() == 1) {
1541          Temp tmp;
1542          Operand half_pi(0x3e22f983u);
1543          if (src.type() == RegType::sgpr)
1544             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1545          else
1546             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1547
1548          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1549          if (ctx->options->chip_class < GFX9)
1550             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1551
1552          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1553          bld.vop1(opcode, Definition(dst), tmp);
1554       } else {
1555          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1556          nir_print_instr(&instr->instr, stderr);
1557          fprintf(stderr, "\n");
1558       }
1559       break;
1560    }
1561    case nir_op_ldexp: {
1562       if (dst.size() == 1) {
1563          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1564                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1565                   get_alu_src(ctx, instr->src[1]));
1566       } else if (dst.size() == 2) {
1567          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1568                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1569                   get_alu_src(ctx, instr->src[1]));
1570       } else {
1571          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1572          nir_print_instr(&instr->instr, stderr);
1573          fprintf(stderr, "\n");
1574       }
1575       break;
1576    }
1577    case nir_op_frexp_sig: {
1578       if (dst.size() == 1) {
1579          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1580                   get_alu_src(ctx, instr->src[0]));
1581       } else if (dst.size() == 2) {
1582          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1583                   get_alu_src(ctx, instr->src[0]));
1584       } else {
1585          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1586          nir_print_instr(&instr->instr, stderr);
1587          fprintf(stderr, "\n");
1588       }
1589       break;
1590    }
1591    case nir_op_frexp_exp: {
1592       if (instr->src[0].src.ssa->bit_size == 32) {
1593          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1594                   get_alu_src(ctx, instr->src[0]));
1595       } else if (instr->src[0].src.ssa->bit_size == 64) {
1596          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1597                   get_alu_src(ctx, instr->src[0]));
1598       } else {
1599          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1600          nir_print_instr(&instr->instr, stderr);
1601          fprintf(stderr, "\n");
1602       }
1603       break;
1604    }
1605    case nir_op_fsign: {
1606       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1607       if (dst.size() == 1) {
1608          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1609          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1610          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1611          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1612       } else if (dst.size() == 2) {
1613          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1614          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1615          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, src, cond);
1616
1617          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1618          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1619          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1620
1621          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1622       } else {
1623          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1624          nir_print_instr(&instr->instr, stderr);
1625          fprintf(stderr, "\n");
1626       }
1627       break;
1628    }
1629    case nir_op_f2f32: {
1630       if (instr->src[0].src.ssa->bit_size == 64) {
1631          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1632       } else {
1633          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1634          nir_print_instr(&instr->instr, stderr);
1635          fprintf(stderr, "\n");
1636       }
1637       break;
1638    }
1639    case nir_op_f2f64: {
1640       if (instr->src[0].src.ssa->bit_size == 32) {
1641          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1642       } else {
1643          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1644          nir_print_instr(&instr->instr, stderr);
1645          fprintf(stderr, "\n");
1646       }
1647       break;
1648    }
1649    case nir_op_i2f32: {
1650       assert(dst.size() == 1);
1651       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1652       break;
1653    }
1654    case nir_op_i2f64: {
1655       if (instr->src[0].src.ssa->bit_size == 32) {
1656          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1657       } else if (instr->src[0].src.ssa->bit_size == 64) {
1658          Temp src = get_alu_src(ctx, instr->src[0]);
1659          RegClass rc = RegClass(src.type(), 1);
1660          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1661          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1662          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1663          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1664          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1665          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1666
1667       } else {
1668          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1669          nir_print_instr(&instr->instr, stderr);
1670          fprintf(stderr, "\n");
1671       }
1672       break;
1673    }
1674    case nir_op_u2f32: {
1675       assert(dst.size() == 1);
1676       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1677       break;
1678    }
1679    case nir_op_u2f64: {
1680       if (instr->src[0].src.ssa->bit_size == 32) {
1681          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1682       } else if (instr->src[0].src.ssa->bit_size == 64) {
1683          Temp src = get_alu_src(ctx, instr->src[0]);
1684          RegClass rc = RegClass(src.type(), 1);
1685          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1686          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1687          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1688          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1689          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1690          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1691       } else {
1692          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1693          nir_print_instr(&instr->instr, stderr);
1694          fprintf(stderr, "\n");
1695       }
1696       break;
1697    }
1698    case nir_op_f2i32: {
1699       Temp src = get_alu_src(ctx, instr->src[0]);
1700       if (instr->src[0].src.ssa->bit_size == 32) {
1701          if (dst.type() == RegType::vgpr)
1702             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1703          else
1704             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1705                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1706
1707       } else if (instr->src[0].src.ssa->bit_size == 64) {
1708          if (dst.type() == RegType::vgpr)
1709             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1710          else
1711             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1712                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1713
1714       } else {
1715          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1716          nir_print_instr(&instr->instr, stderr);
1717          fprintf(stderr, "\n");
1718       }
1719       break;
1720    }
1721    case nir_op_f2u32: {
1722       Temp src = get_alu_src(ctx, instr->src[0]);
1723       if (instr->src[0].src.ssa->bit_size == 32) {
1724          if (dst.type() == RegType::vgpr)
1725             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1726          else
1727             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1728                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1729
1730       } else if (instr->src[0].src.ssa->bit_size == 64) {
1731          if (dst.type() == RegType::vgpr)
1732             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1733          else
1734             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1735                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1736
1737       } else {
1738          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1739          nir_print_instr(&instr->instr, stderr);
1740          fprintf(stderr, "\n");
1741       }
1742       break;
1743    }
1744    case nir_op_f2i64: {
1745       Temp src = get_alu_src(ctx, instr->src[0]);
1746       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1747          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1748          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1749          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1750          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1751          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1752          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1753          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1754          Temp new_exponent = bld.tmp(v1);
1755          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1756          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1757          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1758          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1759          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1760          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1761          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1762          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1763          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1764          Temp new_lower = bld.tmp(v1);
1765          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1766          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1767          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1768
1769       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1770          if (src.type() == RegType::vgpr)
1771             src = bld.as_uniform(src);
1772          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1773          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1774          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1775          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1776          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1777          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1778          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1779          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1780          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1781          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1782          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1783          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1784          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1785          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1786          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1787          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1788          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1789          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1790          Temp borrow = bld.tmp(s1);
1791          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1792          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1793          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1794
1795       } else if (instr->src[0].src.ssa->bit_size == 64) {
1796          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1797          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1798          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1799          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1800          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1801          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1802          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1803          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1804          if (dst.type() == RegType::sgpr) {
1805             lower = bld.as_uniform(lower);
1806             upper = bld.as_uniform(upper);
1807          }
1808          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1809
1810       } else {
1811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1812          nir_print_instr(&instr->instr, stderr);
1813          fprintf(stderr, "\n");
1814       }
1815       break;
1816    }
1817    case nir_op_f2u64: {
1818       Temp src = get_alu_src(ctx, instr->src[0]);
1819       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1820          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1821          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1822          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1823          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1824          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1825          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1826          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1827          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1828          Temp new_exponent = bld.tmp(v1);
1829          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1830          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1831          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1832          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1833          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1834          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1835          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1836          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1837          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1838
1839       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1840          if (src.type() == RegType::vgpr)
1841             src = bld.as_uniform(src);
1842          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1843          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1844          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1845          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1846          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1847          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1848          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1849          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1850          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1851          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1852          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1853          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1854          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1855          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1856          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1857          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1858          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1859          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1860
1861       } else if (instr->src[0].src.ssa->bit_size == 64) {
1862          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1863          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1864          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1865          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1866          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1867          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1868          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1869          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1870          if (dst.type() == RegType::sgpr) {
1871             lower = bld.as_uniform(lower);
1872             upper = bld.as_uniform(upper);
1873          }
1874          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1875
1876       } else {
1877          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1878          nir_print_instr(&instr->instr, stderr);
1879          fprintf(stderr, "\n");
1880       }
1881       break;
1882    }
1883    case nir_op_b2f32: {
1884       Temp src = get_alu_src(ctx, instr->src[0]);
1885       if (dst.regClass() == s1) {
1886          src = as_uniform_bool(ctx, src);
1887          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1888       } else if (dst.regClass() == v1) {
1889          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u),
1890                       as_divergent_bool(ctx, src, true));
1891       } else {
1892          unreachable("Wrong destination register class for nir_op_b2f32.");
1893       }
1894       break;
1895    }
1896    case nir_op_b2f64: {
1897       Temp src = get_alu_src(ctx, instr->src[0]);
1898       if (dst.regClass() == s2) {
1899          src = as_uniform_bool(ctx, src);
1900          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1901       } else if (dst.regClass() == v2) {
1902          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1903          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one,
1904                       as_divergent_bool(ctx, src, true));
1905          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1906       } else {
1907          unreachable("Wrong destination register class for nir_op_b2f64.");
1908       }
1909       break;
1910    }
1911    case nir_op_i2i32: {
1912       Temp src = get_alu_src(ctx, instr->src[0]);
1913       if (instr->src[0].src.ssa->bit_size == 64) {
1914          /* we can actually just say dst = src, as it would map the lower register */
1915          emit_extract_vector(ctx, src, 0, dst);
1916       } else {
1917          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1918          nir_print_instr(&instr->instr, stderr);
1919          fprintf(stderr, "\n");
1920       }
1921       break;
1922    }
1923    case nir_op_u2u32: {
1924       Temp src = get_alu_src(ctx, instr->src[0]);
1925       if (instr->src[0].src.ssa->bit_size == 16) {
1926          if (dst.regClass() == s1) {
1927             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1928          } else {
1929             // TODO: do better with SDWA
1930             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1931          }
1932       } else if (instr->src[0].src.ssa->bit_size == 64) {
1933          /* we can actually just say dst = src, as it would map the lower register */
1934          emit_extract_vector(ctx, src, 0, dst);
1935       } else {
1936          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1937          nir_print_instr(&instr->instr, stderr);
1938          fprintf(stderr, "\n");
1939       }
1940       break;
1941    }
1942    case nir_op_i2i64: {
1943       Temp src = get_alu_src(ctx, instr->src[0]);
1944       if (instr->src[0].src.ssa->bit_size == 32) {
1945          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1946       } else {
1947          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1948          nir_print_instr(&instr->instr, stderr);
1949          fprintf(stderr, "\n");
1950       }
1951       break;
1952    }
1953    case nir_op_u2u64: {
1954       Temp src = get_alu_src(ctx, instr->src[0]);
1955       if (instr->src[0].src.ssa->bit_size == 32) {
1956          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1957       } else {
1958          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1959          nir_print_instr(&instr->instr, stderr);
1960          fprintf(stderr, "\n");
1961       }
1962       break;
1963    }
1964    case nir_op_b2i32: {
1965       Temp src = get_alu_src(ctx, instr->src[0]);
1966       if (dst.regClass() == s1) {
1967          if (src.regClass() == s1) {
1968             bld.copy(Definition(dst), src);
1969          } else {
1970             // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
1971             assert(src.regClass() == s2);
1972             bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(dst)), Operand(0u), src);
1973          }
1974       } else {
1975          assert(dst.regClass() == v1 && src.regClass() == s2);
1976          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
1977       }
1978       break;
1979    }
1980    case nir_op_i2b1: {
1981       Temp src = get_alu_src(ctx, instr->src[0]);
1982       if (dst.regClass() == s2) {
1983          assert(src.regClass() == v1 || src.regClass() == v2);
1984          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
1985                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
1986       } else {
1987          assert(src.regClass() == s1 && dst.regClass() == s1);
1988          bld.sopc(aco_opcode::s_cmp_lg_u32, bld.scc(Definition(dst)), Operand(0u), src);
1989       }
1990       break;
1991    }
1992    case nir_op_pack_64_2x32_split: {
1993       Temp src0 = get_alu_src(ctx, instr->src[0]);
1994       Temp src1 = get_alu_src(ctx, instr->src[1]);
1995
1996       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
1997       break;
1998    }
1999    case nir_op_unpack_64_2x32_split_x:
2000       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2001       break;
2002    case nir_op_unpack_64_2x32_split_y:
2003       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2004       break;
2005    case nir_op_pack_half_2x16: {
2006       Temp src = get_alu_src(ctx, instr->src[0], 2);
2007
2008       if (dst.regClass() == v1) {
2009          Temp src0 = bld.tmp(v1);
2010          Temp src1 = bld.tmp(v1);
2011          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2012          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2013
2014       } else {
2015          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2016          nir_print_instr(&instr->instr, stderr);
2017          fprintf(stderr, "\n");
2018       }
2019       break;
2020    }
2021    case nir_op_unpack_half_2x16_split_x: {
2022       if (dst.regClass() == v1) {
2023          Builder bld(ctx->program, ctx->block);
2024          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2025       } else {
2026          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2027          nir_print_instr(&instr->instr, stderr);
2028          fprintf(stderr, "\n");
2029       }
2030       break;
2031    }
2032    case nir_op_unpack_half_2x16_split_y: {
2033       if (dst.regClass() == v1) {
2034          Builder bld(ctx->program, ctx->block);
2035          /* TODO: use SDWA here */
2036          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2037                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2038       } else {
2039          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2040          nir_print_instr(&instr->instr, stderr);
2041          fprintf(stderr, "\n");
2042       }
2043       break;
2044    }
2045    case nir_op_fquantize2f16: {
2046       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2047
2048       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2049
2050       Temp cmp_res = bld.tmp(s2);
2051       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2052
2053       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2054
2055       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2056       break;
2057    }
2058    case nir_op_bfm: {
2059       Temp bits = get_alu_src(ctx, instr->src[0]);
2060       Temp offset = get_alu_src(ctx, instr->src[1]);
2061
2062       if (dst.regClass() == s1) {
2063          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2064       } else if (dst.regClass() == v1) {
2065          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2066       } else {
2067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2068          nir_print_instr(&instr->instr, stderr);
2069          fprintf(stderr, "\n");
2070       }
2071       break;
2072    }
2073    case nir_op_bitfield_select: {
2074       /* (mask & insert) | (~mask & base) */
2075       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2076       Temp insert = get_alu_src(ctx, instr->src[1]);
2077       Temp base = get_alu_src(ctx, instr->src[2]);
2078
2079       /* dst = (insert & bitmask) | (base & ~bitmask) */
2080       if (dst.regClass() == s1) {
2081          aco_ptr<Instruction> sop2;
2082          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2083          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2084          Operand lhs;
2085          if (const_insert && const_bitmask) {
2086             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2087          } else {
2088             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2089             lhs = Operand(insert);
2090          }
2091
2092          Operand rhs;
2093          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2094          if (const_base && const_bitmask) {
2095             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2096          } else {
2097             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2098             rhs = Operand(base);
2099          }
2100
2101          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2102
2103       } else if (dst.regClass() == v1) {
2104          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2105             base = as_vgpr(ctx, base);
2106          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2107             insert = as_vgpr(ctx, insert);
2108
2109          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2110
2111       } else {
2112          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2113          nir_print_instr(&instr->instr, stderr);
2114          fprintf(stderr, "\n");
2115       }
2116       break;
2117    }
2118    case nir_op_ubfe:
2119    case nir_op_ibfe: {
2120       Temp base = get_alu_src(ctx, instr->src[0]);
2121       Temp offset = get_alu_src(ctx, instr->src[1]);
2122       Temp bits = get_alu_src(ctx, instr->src[2]);
2123
2124       if (dst.type() == RegType::sgpr) {
2125          Operand extract;
2126          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2127          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2128          if (const_offset && const_bits) {
2129             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2130             extract = Operand(const_extract);
2131          } else {
2132             Operand width;
2133             if (const_bits) {
2134                width = Operand(const_bits->u32 << 16);
2135             } else {
2136                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2137             }
2138             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2139          }
2140
2141          aco_opcode opcode;
2142          if (dst.regClass() == s1) {
2143             if (instr->op == nir_op_ubfe)
2144                opcode = aco_opcode::s_bfe_u32;
2145             else
2146                opcode = aco_opcode::s_bfe_i32;
2147          } else if (dst.regClass() == s2) {
2148             if (instr->op == nir_op_ubfe)
2149                opcode = aco_opcode::s_bfe_u64;
2150             else
2151                opcode = aco_opcode::s_bfe_i64;
2152          } else {
2153             unreachable("Unsupported BFE bit size");
2154          }
2155
2156          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2157
2158       } else {
2159          aco_opcode opcode;
2160          if (dst.regClass() == v1) {
2161             if (instr->op == nir_op_ubfe)
2162                opcode = aco_opcode::v_bfe_u32;
2163             else
2164                opcode = aco_opcode::v_bfe_i32;
2165          } else {
2166             unreachable("Unsupported BFE bit size");
2167          }
2168
2169          emit_vop3a_instruction(ctx, instr, opcode, dst);
2170       }
2171       break;
2172    }
2173    case nir_op_bit_count: {
2174       Temp src = get_alu_src(ctx, instr->src[0]);
2175       if (src.regClass() == s1) {
2176          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2177       } else if (src.regClass() == v1) {
2178          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2179       } else if (src.regClass() == v2) {
2180          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2181                   emit_extract_vector(ctx, src, 1, v1),
2182                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2183                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2184       } else if (src.regClass() == s2) {
2185          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2186       } else {
2187          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2188          nir_print_instr(&instr->instr, stderr);
2189          fprintf(stderr, "\n");
2190       }
2191       break;
2192    }
2193    case nir_op_flt: {
2194       if (instr->src[0].src.ssa->bit_size == 32)
2195          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f32, dst);
2196       else if (instr->src[0].src.ssa->bit_size == 64)
2197          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f64, dst);
2198       break;
2199    }
2200    case nir_op_fge: {
2201       if (instr->src[0].src.ssa->bit_size == 32)
2202          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f32, dst);
2203       else if (instr->src[0].src.ssa->bit_size == 64)
2204          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f64, dst);
2205       break;
2206    }
2207    case nir_op_feq: {
2208       if (instr->src[0].src.ssa->bit_size == 32)
2209          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f32, dst);
2210       else if (instr->src[0].src.ssa->bit_size == 64)
2211          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f64, dst);
2212       break;
2213    }
2214    case nir_op_fne: {
2215       if (instr->src[0].src.ssa->bit_size == 32)
2216          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f32, dst);
2217       else if (instr->src[0].src.ssa->bit_size == 64)
2218          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f64, dst);
2219       break;
2220    }
2221    case nir_op_ilt: {
2222       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2223          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i32, dst);
2224       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2225          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_i32, dst);
2226       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2227          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i64, dst);
2228       break;
2229    }
2230    case nir_op_ige: {
2231       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2232          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i32, dst);
2233       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2234          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_i32, dst);
2235       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2236          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i64, dst);
2237       break;
2238    }
2239    case nir_op_ieq: {
2240       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2241          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i32, dst);
2242       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2243          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_i32, dst);
2244       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2245          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i64, dst);
2246       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2247          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_u64, dst);
2248       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2249          Temp src0 = get_alu_src(ctx, instr->src[0]);
2250          Temp src1 = get_alu_src(ctx, instr->src[1]);
2251          bld.sopc(aco_opcode::s_cmp_eq_i32, bld.scc(Definition(dst)),
2252                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2253       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2254          Temp src0 = get_alu_src(ctx, instr->src[0]);
2255          Temp src1 = get_alu_src(ctx, instr->src[1]);
2256          bld.sop2(aco_opcode::s_xnor_b64, Definition(dst), bld.def(s1, scc),
2257                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2258       } else {
2259          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2260          nir_print_instr(&instr->instr, stderr);
2261          fprintf(stderr, "\n");
2262       }
2263       break;
2264    }
2265    case nir_op_ine: {
2266       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2267          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i32, dst);
2268       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2269          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i64, dst);
2270       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2271          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_i32, dst);
2272       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2273          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_u64, dst);
2274       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2275          Temp src0 = get_alu_src(ctx, instr->src[0]);
2276          Temp src1 = get_alu_src(ctx, instr->src[1]);
2277          bld.sopc(aco_opcode::s_cmp_lg_i32, bld.scc(Definition(dst)),
2278                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2279       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2280          Temp src0 = get_alu_src(ctx, instr->src[0]);
2281          Temp src1 = get_alu_src(ctx, instr->src[1]);
2282          bld.sop2(aco_opcode::s_xor_b64, Definition(dst), bld.def(s1, scc),
2283                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2284       } else {
2285          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2286          nir_print_instr(&instr->instr, stderr);
2287          fprintf(stderr, "\n");
2288       }
2289       break;
2290    }
2291    case nir_op_ult: {
2292       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2293          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u32, dst);
2294       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2295          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_u32, dst);
2296       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2297          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u64, dst);
2298       break;
2299    }
2300    case nir_op_uge: {
2301       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2302          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u32, dst);
2303       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2304          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_u32, dst);
2305       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2306          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u64, dst);
2307       break;
2308    }
2309    case nir_op_fddx:
2310    case nir_op_fddy:
2311    case nir_op_fddx_fine:
2312    case nir_op_fddy_fine:
2313    case nir_op_fddx_coarse:
2314    case nir_op_fddy_coarse: {
2315       Definition tl = bld.def(v1);
2316       uint16_t dpp_ctrl;
2317       if (instr->op == nir_op_fddx_fine) {
2318          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2319          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2320       } else if (instr->op == nir_op_fddy_fine) {
2321          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2322          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2323       } else {
2324          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2325          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2326             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2327          else
2328             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2329       }
2330
2331       Definition tmp = bld.def(v1);
2332       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2333       emit_wqm(ctx, tmp.getTemp(), dst, true);
2334       break;
2335    }
2336    default:
2337       fprintf(stderr, "Unknown NIR ALU instr: ");
2338       nir_print_instr(&instr->instr, stderr);
2339       fprintf(stderr, "\n");
2340    }
2341 }
2342
2343 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2344 {
2345    Temp dst = get_ssa_temp(ctx, &instr->def);
2346
2347    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2348    // which get truncated the lsb if double and msb if int
2349    // for now, we only use s_mov_b64 with 64bit inline constants
2350    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2351    assert(dst.type() == RegType::sgpr);
2352
2353    if (dst.size() == 1)
2354    {
2355       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(instr->value[0].u32));
2356    } else {
2357       assert(dst.size() != 1);
2358       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2359       if (instr->def.bit_size == 64)
2360          for (unsigned i = 0; i < dst.size(); i++)
2361             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2362       else {
2363          for (unsigned i = 0; i < dst.size(); i++)
2364             vec->operands[i] = Operand{instr->value[i].u32};
2365       }
2366       vec->definitions[0] = Definition(dst);
2367       ctx->block->instructions.emplace_back(std::move(vec));
2368    }
2369 }
2370
2371 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2372 {
2373    uint32_t new_mask = 0;
2374    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2375       if (mask & (1u << i))
2376          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2377    return new_mask;
2378 }
2379
2380 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2381 {
2382    /* This wouldn't work inside control flow or with indirect offsets but
2383     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2384
2385    unsigned write_mask = nir_intrinsic_write_mask(instr);
2386    unsigned component = nir_intrinsic_component(instr);
2387    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2388    unsigned idx = nir_intrinsic_base(instr) + component;
2389
2390    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2391    if (off_instr->type != nir_instr_type_load_const) {
2392       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2393       nir_print_instr(off_instr, stderr);
2394       fprintf(stderr, "\n");
2395    }
2396    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2397
2398    if (instr->src[0].ssa->bit_size == 64)
2399       write_mask = widen_mask(write_mask, 2);
2400
2401    for (unsigned i = 0; i < 8; ++i) {
2402       if (write_mask & (1 << i)) {
2403          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2404          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2405       }
2406       idx++;
2407    }
2408 }
2409
2410 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2411 {
2412    unsigned write_mask = nir_intrinsic_write_mask(instr);
2413    Operand values[4];
2414    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2415    for (unsigned i = 0; i < 4; ++i) {
2416       if (write_mask & (1 << i)) {
2417          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2418          values[i] = Operand(tmp);
2419       } else {
2420          values[i] = Operand(v1);
2421       }
2422    }
2423
2424    unsigned index = nir_intrinsic_base(instr) / 4;
2425    unsigned target, col_format;
2426    unsigned enabled_channels = 0xF;
2427    aco_opcode compr_op = (aco_opcode)0;
2428
2429    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2430    assert(offset && "Non-const offsets on exports not yet supported");
2431    index += offset->u32;
2432
2433    assert(index != FRAG_RESULT_COLOR);
2434
2435    /* Unlike vertex shader exports, it's fine to use multiple exports to
2436     * export separate channels of one target. So shaders which export both
2437     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2438     * TODO: combine the exports in those cases and create better code
2439     */
2440
2441    if (index == FRAG_RESULT_SAMPLE_MASK) {
2442
2443       if (ctx->program->info->ps.writes_z) {
2444          target = V_008DFC_SQ_EXP_MRTZ;
2445          enabled_channels = 0x4;
2446          col_format = (unsigned) -1;
2447
2448          values[2] = values[0];
2449          values[0] = Operand(v1);
2450       } else {
2451          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2452          exp->valid_mask = false;
2453          exp->done = false;
2454          exp->compressed = true;
2455          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2456          exp->enabled_mask = 0xc;
2457          for (int i = 0; i < 4; i++)
2458             exp->operands[i] = Operand(v1);
2459          exp->operands[1] = Operand(values[0]);
2460          ctx->block->instructions.emplace_back(std::move(exp));
2461          return;
2462       }
2463
2464    } else if (index == FRAG_RESULT_DEPTH) {
2465
2466       target = V_008DFC_SQ_EXP_MRTZ;
2467       enabled_channels = 0x1;
2468       col_format = (unsigned) -1;
2469
2470    } else if (index == FRAG_RESULT_STENCIL) {
2471
2472       if (ctx->program->info->ps.writes_z) {
2473          target = V_008DFC_SQ_EXP_MRTZ;
2474          enabled_channels = 0x2;
2475          col_format = (unsigned) -1;
2476
2477          values[1] = values[0];
2478          values[0] = Operand(v1);
2479       } else {
2480          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2481          shift->operands[0] = Operand((uint32_t) 16);
2482          shift->operands[1] = values[0];
2483          Temp tmp = {ctx->program->allocateId(), v1};
2484          shift->definitions[0] = Definition(tmp);
2485          ctx->block->instructions.emplace_back(std::move(shift));
2486
2487          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2488          exp->valid_mask = false;
2489          exp->done = false;
2490          exp->compressed = true;
2491          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2492          exp->enabled_mask = 0x3;
2493          exp->operands[0] = Operand(tmp);
2494          for (int i = 1; i < 4; i++)
2495             exp->operands[i] = Operand(v1);
2496          ctx->block->instructions.emplace_back(std::move(exp));
2497          return;
2498       }
2499
2500    } else {
2501       index -= FRAG_RESULT_DATA0;
2502       target = V_008DFC_SQ_EXP_MRT + index;
2503       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2504    }
2505    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2506    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2507    assert(!is_int8 && !is_int10);
2508
2509    switch (col_format)
2510    {
2511    case V_028714_SPI_SHADER_ZERO:
2512       enabled_channels = 0; /* writemask */
2513       target = V_008DFC_SQ_EXP_NULL;
2514       break;
2515
2516    case V_028714_SPI_SHADER_32_R:
2517       enabled_channels = 1;
2518       break;
2519
2520    case V_028714_SPI_SHADER_32_GR:
2521       enabled_channels = 0x3;
2522       break;
2523
2524    case V_028714_SPI_SHADER_32_AR:
2525       enabled_channels = 0x9;
2526       break;
2527
2528    case V_028714_SPI_SHADER_FP16_ABGR:
2529       enabled_channels = 0x5;
2530       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2531       break;
2532
2533    case V_028714_SPI_SHADER_UNORM16_ABGR:
2534       enabled_channels = 0x5;
2535       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2536       break;
2537
2538    case V_028714_SPI_SHADER_SNORM16_ABGR:
2539       enabled_channels = 0x5;
2540       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2541       break;
2542
2543    case V_028714_SPI_SHADER_UINT16_ABGR:
2544       enabled_channels = 0x5;
2545       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2546       break;
2547
2548    case V_028714_SPI_SHADER_SINT16_ABGR:
2549       enabled_channels = 0x5;
2550       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2551       break;
2552
2553    case V_028714_SPI_SHADER_32_ABGR:
2554       enabled_channels = 0xF;
2555       break;
2556
2557    default:
2558       break;
2559    }
2560
2561    if (target == V_008DFC_SQ_EXP_NULL)
2562       return;
2563
2564    if ((bool)compr_op)
2565    {
2566       for (int i = 0; i < 2; i++)
2567       {
2568          /* check if at least one of the values to be compressed is enabled */
2569          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2570          if (enabled) {
2571             enabled_channels |= enabled << (i*2);
2572             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2573             Temp tmp{ctx->program->allocateId(), v1};
2574             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2575             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2576             compr->definitions[0] = Definition(tmp);
2577             values[i] = Operand(tmp);
2578             ctx->block->instructions.emplace_back(std::move(compr));
2579          } else {
2580             values[i] = Operand(v1);
2581          }
2582       }
2583    }
2584
2585    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2586    exp->valid_mask = false;
2587    exp->done = false;
2588    exp->compressed = (bool) compr_op;
2589    exp->dest = target;
2590    exp->enabled_mask = enabled_channels;
2591    if ((bool) compr_op) {
2592       for (int i = 0; i < 2; i++)
2593          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2594       exp->operands[2] = Operand(v1);
2595       exp->operands[3] = Operand(v1);
2596    } else {
2597       for (int i = 0; i < 4; i++)
2598          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2599    }
2600
2601    ctx->block->instructions.emplace_back(std::move(exp));
2602 }
2603
2604 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2605 {
2606    if (ctx->stage == vertex_vs) {
2607       visit_store_vs_output(ctx, instr);
2608    } else if (ctx->stage == fragment_fs) {
2609       visit_store_fs_output(ctx, instr);
2610    } else {
2611       unreachable("Shader stage not implemented");
2612    }
2613 }
2614
2615 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2616 {
2617    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2618    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2619
2620    Builder bld(ctx->program, ctx->block);
2621    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2622    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2623 }
2624
2625 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2626 {
2627    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2628    for (unsigned i = 0; i < num_components; i++)
2629       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2630
2631    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2632       assert(num_components == 4);
2633       Builder bld(ctx->program, ctx->block);
2634       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2635    }
2636
2637    for (Operand& op : vec->operands)
2638       op = op.isUndefined() ? Operand(0u) : op;
2639
2640    vec->definitions[0] = Definition(dst);
2641    ctx->block->instructions.emplace_back(std::move(vec));
2642    emit_split_vector(ctx, dst, num_components);
2643    return;
2644 }
2645
2646 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2647 {
2648    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2649    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2650    unsigned idx = nir_intrinsic_base(instr);
2651    unsigned component = nir_intrinsic_component(instr);
2652    Temp prim_mask = ctx->prim_mask;
2653
2654    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2655    if (offset) {
2656       assert(offset->u32 == 0);
2657    } else {
2658       /* the lower 15bit of the prim_mask contain the offset into LDS
2659        * while the upper bits contain the number of prims */
2660       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2661       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2662       Builder bld(ctx->program, ctx->block);
2663       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2664       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2665       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2666       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2667       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2668    }
2669
2670    if (instr->dest.ssa.num_components == 1) {
2671       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2672    } else {
2673       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2674       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2675       {
2676          Temp tmp = {ctx->program->allocateId(), v1};
2677          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2678          vec->operands[i] = Operand(tmp);
2679       }
2680       vec->definitions[0] = Definition(dst);
2681       ctx->block->instructions.emplace_back(std::move(vec));
2682    }
2683 }
2684
2685 unsigned get_num_channels_from_data_format(unsigned data_format)
2686 {
2687    switch (data_format) {
2688    case V_008F0C_BUF_DATA_FORMAT_8:
2689    case V_008F0C_BUF_DATA_FORMAT_16:
2690    case V_008F0C_BUF_DATA_FORMAT_32:
2691       return 1;
2692    case V_008F0C_BUF_DATA_FORMAT_8_8:
2693    case V_008F0C_BUF_DATA_FORMAT_16_16:
2694    case V_008F0C_BUF_DATA_FORMAT_32_32:
2695       return 2;
2696    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2697    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2698    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2699       return 3;
2700    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2701    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2702    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2703    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2704    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2705       return 4;
2706    default:
2707       break;
2708    }
2709
2710    return 4;
2711 }
2712
2713 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2714  * so we may need to fix it up. */
2715 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
2716 {
2717    Builder bld(ctx->program, ctx->block);
2718
2719    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
2720       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
2721
2722    /* For the integer-like cases, do a natural sign extension.
2723     *
2724     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
2725     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
2726     * exponent.
2727     */
2728    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
2729    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
2730
2731    /* Convert back to the right type. */
2732    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
2733       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2734       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
2735       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
2736    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
2737       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2738    }
2739
2740    return alpha;
2741 }
2742
2743 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
2744 {
2745    Builder bld(ctx->program, ctx->block);
2746    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2747    if (ctx->stage & sw_vs) {
2748
2749       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2750       if (off_instr->type != nir_instr_type_load_const) {
2751          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2752          nir_print_instr(off_instr, stderr);
2753          fprintf(stderr, "\n");
2754       }
2755       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
2756
2757       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
2758
2759       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
2760       unsigned component = nir_intrinsic_component(instr);
2761       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
2762       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
2763       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
2764       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
2765
2766       unsigned dfmt = attrib_format & 0xf;
2767
2768       unsigned nfmt = (attrib_format >> 4) & 0x7;
2769       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
2770       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
2771       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
2772       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
2773       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
2774       if (post_shuffle)
2775          num_channels = MAX2(num_channels, 3);
2776
2777       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
2778
2779       Temp index;
2780       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
2781          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
2782          if (divisor) {
2783             ctx->needs_instance_id = true;
2784
2785             if (divisor != 1) {
2786                Temp divided = bld.tmp(v1);
2787                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
2788                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
2789             } else {
2790                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
2791             }
2792          } else {
2793             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
2794          }
2795       } else {
2796          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
2797       }
2798
2799       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
2800          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
2801          attrib_offset = attrib_offset % attrib_stride;
2802       }
2803
2804       Operand soffset(0u);
2805       if (attrib_offset >= 4096) {
2806          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
2807          attrib_offset = 0;
2808       }
2809
2810       aco_opcode opcode;
2811       switch (num_channels) {
2812       case 1:
2813          opcode = aco_opcode::tbuffer_load_format_x;
2814          break;
2815       case 2:
2816          opcode = aco_opcode::tbuffer_load_format_xy;
2817          break;
2818       case 3:
2819          opcode = aco_opcode::tbuffer_load_format_xyz;
2820          break;
2821       case 4:
2822          opcode = aco_opcode::tbuffer_load_format_xyzw;
2823          break;
2824       default:
2825          unreachable("Unimplemented load_input vector size");
2826       }
2827
2828       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
2829
2830       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
2831       mubuf->operands[0] = Operand(index);
2832       mubuf->operands[1] = Operand(list);
2833       mubuf->operands[2] = soffset;
2834       mubuf->definitions[0] = Definition(tmp);
2835       mubuf->idxen = true;
2836       mubuf->can_reorder = true;
2837       mubuf->dfmt = dfmt;
2838       mubuf->nfmt = nfmt;
2839       assert(attrib_offset < 4096);
2840       mubuf->offset = attrib_offset;
2841       ctx->block->instructions.emplace_back(std::move(mubuf));
2842
2843       emit_split_vector(ctx, tmp, tmp.size());
2844
2845       if (tmp.id() != dst.id()) {
2846          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
2847                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
2848
2849          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
2850          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
2851          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
2852
2853          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2854          for (unsigned i = 0; i < dst.size(); i++) {
2855             unsigned idx = i + component;
2856             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
2857                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
2858                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
2859             } else if (idx < num_channels) {
2860                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
2861             } else if (is_float && idx == 3) {
2862                vec->operands[i] = Operand(0x3f800000u);
2863             } else if (!is_float && idx == 3) {
2864                vec->operands[i] = Operand(1u);
2865             } else {
2866                vec->operands[i] = Operand(0u);
2867             }
2868          }
2869          vec->definitions[0] = Definition(dst);
2870          ctx->block->instructions.emplace_back(std::move(vec));
2871          emit_split_vector(ctx, dst, dst.size());
2872       }
2873
2874    } else if (ctx->stage == fragment_fs) {
2875       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2876       if (off_instr->type != nir_instr_type_load_const ||
2877           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
2878          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2879          nir_print_instr(off_instr, stderr);
2880          fprintf(stderr, "\n");
2881       }
2882
2883       Temp prim_mask = ctx->prim_mask;
2884       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
2885       if (offset) {
2886          assert(offset->u32 == 0);
2887       } else {
2888          /* the lower 15bit of the prim_mask contain the offset into LDS
2889           * while the upper bits contain the number of prims */
2890          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
2891          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2892          Builder bld(ctx->program, ctx->block);
2893          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2894          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2895          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2896          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2897          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2898       }
2899
2900       unsigned idx = nir_intrinsic_base(instr);
2901       unsigned component = nir_intrinsic_component(instr);
2902
2903       if (dst.size() == 1) {
2904          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
2905       } else {
2906          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2907          for (unsigned i = 0; i < dst.size(); i++)
2908             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
2909          vec->definitions[0] = Definition(dst);
2910          bld.insert(std::move(vec));
2911       }
2912
2913    } else {
2914       unreachable("Shader stage not implemented");
2915    }
2916 }
2917
2918 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
2919 {
2920    if (ctx->program->info->need_indirect_descriptor_sets) {
2921       Builder bld(ctx->program, ctx->block);
2922       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
2923       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
2924    }
2925
2926    return ctx->descriptor_sets[desc_set];
2927 }
2928
2929
2930 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
2931 {
2932    Builder bld(ctx->program, ctx->block);
2933    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
2934    unsigned desc_set = nir_intrinsic_desc_set(instr);
2935    unsigned binding = nir_intrinsic_binding(instr);
2936
2937    Temp desc_ptr;
2938    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
2939    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
2940    unsigned offset = layout->binding[binding].offset;
2941    unsigned stride;
2942    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
2943        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
2944       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
2945       desc_ptr = ctx->push_constants;
2946       offset = pipeline_layout->push_constant_size + 16 * idx;
2947       stride = 16;
2948    } else {
2949       desc_ptr = load_desc_ptr(ctx, desc_set);
2950       stride = layout->binding[binding].size;
2951    }
2952
2953    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
2954    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
2955    if (stride != 1) {
2956       if (nir_const_index) {
2957          const_index = const_index * stride;
2958       } else {
2959          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
2960       }
2961    }
2962    if (offset) {
2963       if (nir_const_index) {
2964          const_index = const_index + offset;
2965       } else {
2966          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
2967       }
2968    }
2969
2970    if (nir_const_index && const_index == 0) {
2971       index = desc_ptr;
2972    } else {
2973       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
2974                        nir_const_index ? Operand(const_index) : Operand(index),
2975                        Operand(desc_ptr));
2976    }
2977
2978    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2979    bld.sop1(aco_opcode::s_mov_b32, Definition(dst), index);
2980 }
2981
2982 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst, Temp rsrc, Temp offset, bool glc=false)
2983 {
2984    Builder bld(ctx->program, ctx->block);
2985
2986    unsigned num_bytes = dst.size() * 4;
2987    bool dlc = glc && ctx->options->chip_class >= GFX10;
2988
2989    aco_opcode op;
2990    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
2991       if (ctx->options->chip_class < GFX8)
2992          offset = as_vgpr(ctx, offset);
2993
2994       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
2995       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
2996       unsigned const_offset = 0;
2997
2998       Temp lower = Temp();
2999       if (num_bytes > 16) {
3000          assert(num_components == 3 || num_components == 4);
3001          op = aco_opcode::buffer_load_dwordx4;
3002          lower = bld.tmp(v4);
3003          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3004          mubuf->definitions[0] = Definition(lower);
3005          mubuf->operands[0] = vaddr;
3006          mubuf->operands[1] = Operand(rsrc);
3007          mubuf->operands[2] = soffset;
3008          mubuf->offen = (offset.type() == RegType::vgpr);
3009          mubuf->glc = glc;
3010          mubuf->dlc = dlc;
3011          mubuf->barrier = barrier_buffer;
3012          bld.insert(std::move(mubuf));
3013          emit_split_vector(ctx, lower, 2);
3014          num_bytes -= 16;
3015          const_offset = 16;
3016       }
3017
3018       switch (num_bytes) {
3019          case 4:
3020             op = aco_opcode::buffer_load_dword;
3021             break;
3022          case 8:
3023             op = aco_opcode::buffer_load_dwordx2;
3024             break;
3025          case 12:
3026             op = aco_opcode::buffer_load_dwordx3;
3027             break;
3028          case 16:
3029             op = aco_opcode::buffer_load_dwordx4;
3030             break;
3031          default:
3032             unreachable("Load SSBO not implemented for this size.");
3033       }
3034       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3035       mubuf->operands[0] = vaddr;
3036       mubuf->operands[1] = Operand(rsrc);
3037       mubuf->operands[2] = soffset;
3038       mubuf->offen = (offset.type() == RegType::vgpr);
3039       mubuf->glc = glc;
3040       mubuf->dlc = dlc;
3041       mubuf->barrier = barrier_buffer;
3042       mubuf->offset = const_offset;
3043       aco_ptr<Instruction> instr = std::move(mubuf);
3044
3045       if (dst.size() > 4) {
3046          assert(lower != Temp());
3047          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3048          instr->definitions[0] = Definition(upper);
3049          bld.insert(std::move(instr));
3050          if (dst.size() == 8)
3051             emit_split_vector(ctx, upper, 2);
3052          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3053          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3054          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3055          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3056          if (dst.size() == 8)
3057             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3058       }
3059
3060       if (dst.type() == RegType::sgpr) {
3061          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3062          instr->definitions[0] = Definition(vec);
3063          bld.insert(std::move(instr));
3064          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3065       } else {
3066          instr->definitions[0] = Definition(dst);
3067          bld.insert(std::move(instr));
3068       }
3069    } else {
3070       switch (num_bytes) {
3071          case 4:
3072             op = aco_opcode::s_buffer_load_dword;
3073             break;
3074          case 8:
3075             op = aco_opcode::s_buffer_load_dwordx2;
3076             break;
3077          case 12:
3078          case 16:
3079             op = aco_opcode::s_buffer_load_dwordx4;
3080             break;
3081          case 24:
3082          case 32:
3083             op = aco_opcode::s_buffer_load_dwordx8;
3084             break;
3085          default:
3086             unreachable("Load SSBO not implemented for this size.");
3087       }
3088       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3089       load->operands[0] = Operand(rsrc);
3090       load->operands[1] = Operand(bld.as_uniform(offset));
3091       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3092       load->definitions[0] = Definition(dst);
3093       load->glc = glc;
3094       load->dlc = dlc;
3095       load->barrier = barrier_buffer;
3096       assert(ctx->options->chip_class >= GFX8 || !glc);
3097
3098       /* trim vector */
3099       if (dst.size() == 3) {
3100          Temp vec = bld.tmp(s4);
3101          load->definitions[0] = Definition(vec);
3102          bld.insert(std::move(load));
3103          emit_split_vector(ctx, vec, 4);
3104
3105          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3106                     emit_extract_vector(ctx, vec, 0, s1),
3107                     emit_extract_vector(ctx, vec, 1, s1),
3108                     emit_extract_vector(ctx, vec, 2, s1));
3109       } else if (dst.size() == 6) {
3110          Temp vec = bld.tmp(s8);
3111          load->definitions[0] = Definition(vec);
3112          bld.insert(std::move(load));
3113          emit_split_vector(ctx, vec, 4);
3114
3115          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3116                     emit_extract_vector(ctx, vec, 0, s2),
3117                     emit_extract_vector(ctx, vec, 1, s2),
3118                     emit_extract_vector(ctx, vec, 2, s2));
3119       } else {
3120          bld.insert(std::move(load));
3121       }
3122
3123    }
3124    emit_split_vector(ctx, dst, num_components);
3125 }
3126
3127 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3128 {
3129    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3130    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3131
3132    Builder bld(ctx->program, ctx->block);
3133
3134    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3135    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3136    unsigned binding = nir_intrinsic_binding(idx_instr);
3137    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3138
3139    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3140       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3141                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3142                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3143                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3144       if (ctx->options->chip_class >= GFX10) {
3145          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3146                       S_008F0C_OOB_SELECT(3) |
3147                       S_008F0C_RESOURCE_LEVEL(1);
3148       } else {
3149          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3150                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3151       }
3152       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3153                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3154                                      Operand(0xFFFFFFFFu),
3155                                      Operand(desc_type));
3156       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3157                         rsrc, upper_dwords);
3158    } else {
3159       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3160       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3161    }
3162
3163    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3164 }
3165
3166 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3167 {
3168    Builder bld(ctx->program, ctx->block);
3169    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3170
3171    unsigned offset = nir_intrinsic_base(instr);
3172    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3173    if (index_cv && instr->dest.ssa.bit_size == 32) {
3174
3175       unsigned count = instr->dest.ssa.num_components;
3176       unsigned start = (offset + index_cv->u32) / 4u;
3177       start -= ctx->base_inline_push_consts;
3178       if (start + count <= ctx->num_inline_push_consts) {
3179          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3180          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3181          for (unsigned i = 0; i < count; ++i) {
3182             elems[i] = ctx->inline_push_consts[start + i];
3183             vec->operands[i] = Operand{elems[i]};
3184          }
3185          vec->definitions[0] = Definition(dst);
3186          ctx->block->instructions.emplace_back(std::move(vec));
3187          ctx->allocated_vec.emplace(dst.id(), elems);
3188          return;
3189       }
3190    }
3191
3192    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3193    if (offset != 0) // TODO check if index != 0 as well
3194       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3195    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3196    Temp vec = dst;
3197    bool trim = false;
3198    aco_opcode op;
3199
3200    switch (dst.size()) {
3201    case 1:
3202       op = aco_opcode::s_load_dword;
3203       break;
3204    case 2:
3205       op = aco_opcode::s_load_dwordx2;
3206       break;
3207    case 3:
3208       vec = bld.tmp(s4);
3209       trim = true;
3210    case 4:
3211       op = aco_opcode::s_load_dwordx4;
3212       break;
3213    case 6:
3214       vec = bld.tmp(s8);
3215       trim = true;
3216    case 8:
3217       op = aco_opcode::s_load_dwordx8;
3218       break;
3219    default:
3220       unreachable("unimplemented or forbidden load_push_constant.");
3221    }
3222
3223    bld.smem(op, Definition(vec), ptr, index);
3224
3225    if (trim) {
3226       emit_split_vector(ctx, vec, 4);
3227       RegClass rc = dst.size() == 3 ? s1 : s2;
3228       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3229                  emit_extract_vector(ctx, vec, 0, rc),
3230                  emit_extract_vector(ctx, vec, 1, rc),
3231                  emit_extract_vector(ctx, vec, 2, rc));
3232
3233    }
3234    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3235 }
3236
3237 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3238 {
3239    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3240
3241    Builder bld(ctx->program, ctx->block);
3242
3243    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3244                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3245                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3246                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3247    if (ctx->options->chip_class >= GFX10) {
3248       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3249                    S_008F0C_OOB_SELECT(3) |
3250                    S_008F0C_RESOURCE_LEVEL(1);
3251    } else {
3252       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3253                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3254    }
3255
3256    unsigned base = nir_intrinsic_base(instr);
3257    unsigned range = nir_intrinsic_range(instr);
3258
3259    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3260    if (base && offset.type() == RegType::sgpr)
3261       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3262    else if (base && offset.type() == RegType::vgpr)
3263       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3264
3265    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3266                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3267                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3268                           Operand(desc_type));
3269
3270    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3271 }
3272
3273 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3274 {
3275    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3276       ctx->cf_info.exec_potentially_empty = true;
3277
3278    ctx->program->needs_exact = true;
3279
3280    // TODO: optimize uniform conditions
3281    Builder bld(ctx->program, ctx->block);
3282    Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
3283    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3284    bld.pseudo(aco_opcode::p_discard_if, src);
3285    ctx->block->kind |= block_kind_uses_discard_if;
3286    return;
3287 }
3288
3289 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3290 {
3291    Builder bld(ctx->program, ctx->block);
3292
3293    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3294       ctx->cf_info.exec_potentially_empty = true;
3295
3296    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3297                     ctx->cf_info.parent_loop.has_divergent_continue;
3298
3299    if (ctx->block->loop_nest_depth &&
3300        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3301       /* we handle discards the same way as jump instructions */
3302       append_logical_end(ctx->block);
3303
3304       /* in loops, discard behaves like break */
3305       Block *linear_target = ctx->cf_info.parent_loop.exit;
3306       ctx->block->kind |= block_kind_discard;
3307
3308       if (!divergent) {
3309          /* uniform discard - loop ends here */
3310          assert(nir_instr_is_last(&instr->instr));
3311          ctx->block->kind |= block_kind_uniform;
3312          ctx->cf_info.has_branch = true;
3313          bld.branch(aco_opcode::p_branch);
3314          add_linear_edge(ctx->block->index, linear_target);
3315          return;
3316       }
3317
3318       /* we add a break right behind the discard() instructions */
3319       ctx->block->kind |= block_kind_break;
3320       unsigned idx = ctx->block->index;
3321
3322       /* remove critical edges from linear CFG */
3323       bld.branch(aco_opcode::p_branch);
3324       Block* break_block = ctx->program->create_and_insert_block();
3325       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3326       break_block->kind |= block_kind_uniform;
3327       add_linear_edge(idx, break_block);
3328       add_linear_edge(break_block->index, linear_target);
3329       bld.reset(break_block);
3330       bld.branch(aco_opcode::p_branch);
3331
3332       Block* continue_block = ctx->program->create_and_insert_block();
3333       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3334       add_linear_edge(idx, continue_block);
3335       append_logical_start(continue_block);
3336       ctx->block = continue_block;
3337
3338       return;
3339    }
3340
3341    /* it can currently happen that NIR doesn't remove the unreachable code */
3342    if (!nir_instr_is_last(&instr->instr)) {
3343       ctx->program->needs_exact = true;
3344       /* save exec somewhere temporarily so that it doesn't get
3345        * overwritten before the discard from outer exec masks */
3346       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3347       bld.pseudo(aco_opcode::p_discard_if, cond);
3348       ctx->block->kind |= block_kind_uses_discard_if;
3349       return;
3350    }
3351
3352    /* This condition is incorrect for uniformly branched discards in a loop
3353     * predicated by a divergent condition, but the above code catches that case
3354     * and the discard would end up turning into a discard_if.
3355     * For example:
3356     * if (divergent) {
3357     *    while (...) {
3358     *       if (uniform) {
3359     *          discard;
3360     *       }
3361     *    }
3362     * }
3363     */
3364    if (!ctx->cf_info.parent_if.is_divergent) {
3365       /* program just ends here */
3366       ctx->block->kind |= block_kind_uniform;
3367       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3368               0 /* enabled mask */, 9 /* dest */,
3369               false /* compressed */, true/* done */, true /* valid mask */);
3370       bld.sopp(aco_opcode::s_endpgm);
3371       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3372    } else {
3373       ctx->block->kind |= block_kind_discard;
3374       /* branch and linear edge is added by visit_if() */
3375    }
3376 }
3377
3378 enum aco_descriptor_type {
3379    ACO_DESC_IMAGE,
3380    ACO_DESC_FMASK,
3381    ACO_DESC_SAMPLER,
3382    ACO_DESC_BUFFER,
3383    ACO_DESC_PLANE_0,
3384    ACO_DESC_PLANE_1,
3385    ACO_DESC_PLANE_2,
3386 };
3387
3388 static bool
3389 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3390    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3391       return false;
3392    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3393    return dim == ac_image_cube ||
3394           dim == ac_image_1darray ||
3395           dim == ac_image_2darray ||
3396           dim == ac_image_2darraymsaa;
3397 }
3398
3399 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3400                       enum aco_descriptor_type desc_type,
3401                       const nir_tex_instr *tex_instr, bool image, bool write)
3402 {
3403 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3404    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3405    if (it != ctx->tex_desc.end())
3406       return it->second;
3407 */
3408    Temp index = Temp();
3409    bool index_set = false;
3410    unsigned constant_index = 0;
3411    unsigned descriptor_set;
3412    unsigned base_index;
3413    Builder bld(ctx->program, ctx->block);
3414
3415    if (!deref_instr) {
3416       assert(tex_instr && !image);
3417       descriptor_set = 0;
3418       base_index = tex_instr->sampler_index;
3419    } else {
3420       while(deref_instr->deref_type != nir_deref_type_var) {
3421          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3422          if (!array_size)
3423             array_size = 1;
3424
3425          assert(deref_instr->deref_type == nir_deref_type_array);
3426          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3427          if (const_value) {
3428             constant_index += array_size * const_value->u32;
3429          } else {
3430             Temp indirect = bld.as_uniform(get_ssa_temp(ctx, deref_instr->arr.index.ssa));
3431
3432             if (array_size != 1)
3433                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3434
3435             if (!index_set) {
3436                index = indirect;
3437                index_set = true;
3438             } else {
3439                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3440             }
3441          }
3442
3443          deref_instr = nir_src_as_deref(deref_instr->parent);
3444       }
3445       descriptor_set = deref_instr->var->data.descriptor_set;
3446       base_index = deref_instr->var->data.binding;
3447    }
3448
3449    Temp list = load_desc_ptr(ctx, descriptor_set);
3450    list = convert_pointer_to_64_bit(ctx, list);
3451
3452    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3453    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3454    unsigned offset = binding->offset;
3455    unsigned stride = binding->size;
3456    aco_opcode opcode;
3457    RegClass type;
3458
3459    assert(base_index < layout->binding_count);
3460
3461    switch (desc_type) {
3462    case ACO_DESC_IMAGE:
3463       type = s8;
3464       opcode = aco_opcode::s_load_dwordx8;
3465       break;
3466    case ACO_DESC_FMASK:
3467       type = s8;
3468       opcode = aco_opcode::s_load_dwordx8;
3469       offset += 32;
3470       break;
3471    case ACO_DESC_SAMPLER:
3472       type = s4;
3473       opcode = aco_opcode::s_load_dwordx4;
3474       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3475          offset += radv_combined_image_descriptor_sampler_offset(binding);
3476       break;
3477    case ACO_DESC_BUFFER:
3478       type = s4;
3479       opcode = aco_opcode::s_load_dwordx4;
3480       break;
3481    case ACO_DESC_PLANE_0:
3482    case ACO_DESC_PLANE_1:
3483       type = s8;
3484       opcode = aco_opcode::s_load_dwordx8;
3485       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3486       break;
3487    case ACO_DESC_PLANE_2:
3488       type = s4;
3489       opcode = aco_opcode::s_load_dwordx4;
3490       offset += 64;
3491       break;
3492    default:
3493       unreachable("invalid desc_type\n");
3494    }
3495
3496    offset += constant_index * stride;
3497
3498    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3499       (!index_set || binding->immutable_samplers_equal)) {
3500       if (binding->immutable_samplers_equal)
3501          constant_index = 0;
3502
3503       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3504       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3505                         Operand(samplers[constant_index * 4 + 0]),
3506                         Operand(samplers[constant_index * 4 + 1]),
3507                         Operand(samplers[constant_index * 4 + 2]),
3508                         Operand(samplers[constant_index * 4 + 3]));
3509    }
3510
3511    Operand off;
3512    if (!index_set) {
3513       off = Operand(offset);
3514    } else {
3515       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3516                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3517    }
3518
3519    Temp res = bld.smem(opcode, bld.def(type), list, off);
3520
3521    if (desc_type == ACO_DESC_PLANE_2) {
3522       Temp components[8];
3523       for (unsigned i = 0; i < 8; i++)
3524          components[i] = bld.tmp(s1);
3525       bld.pseudo(aco_opcode::p_split_vector,
3526                  Definition(components[0]),
3527                  Definition(components[1]),
3528                  Definition(components[2]),
3529                  Definition(components[3]),
3530                  res);
3531
3532       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3533       bld.pseudo(aco_opcode::p_split_vector,
3534                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3535                  Definition(components[4]),
3536                  Definition(components[5]),
3537                  Definition(components[6]),
3538                  Definition(components[7]),
3539                  desc2);
3540
3541       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3542                        components[0], components[1], components[2], components[3],
3543                        components[4], components[5], components[6], components[7]);
3544    }
3545
3546    return res;
3547 }
3548
3549 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3550 {
3551    switch (dim) {
3552    case GLSL_SAMPLER_DIM_BUF:
3553       return 1;
3554    case GLSL_SAMPLER_DIM_1D:
3555       return array ? 2 : 1;
3556    case GLSL_SAMPLER_DIM_2D:
3557       return array ? 3 : 2;
3558    case GLSL_SAMPLER_DIM_MS:
3559       return array ? 4 : 3;
3560    case GLSL_SAMPLER_DIM_3D:
3561    case GLSL_SAMPLER_DIM_CUBE:
3562       return 3;
3563    case GLSL_SAMPLER_DIM_RECT:
3564    case GLSL_SAMPLER_DIM_SUBPASS:
3565       return 2;
3566    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3567       return 3;
3568    default:
3569       break;
3570    }
3571    return 0;
3572 }
3573
3574
3575 /* Adjust the sample index according to FMASK.
3576  *
3577  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3578  * which is the identity mapping. Each nibble says which physical sample
3579  * should be fetched to get that sample.
3580  *
3581  * For example, 0x11111100 means there are only 2 samples stored and
3582  * the second sample covers 3/4 of the pixel. When reading samples 0
3583  * and 1, return physical sample 0 (determined by the first two 0s
3584  * in FMASK), otherwise return physical sample 1.
3585  *
3586  * The sample index should be adjusted as follows:
3587  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3588  */
3589 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3590 {
3591    Builder bld(ctx->program, ctx->block);
3592    Temp fmask = bld.tmp(v1);
3593    unsigned dim = ctx->options->chip_class >= GFX10
3594                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3595                   : 0;
3596
3597    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3598    load->operands[0] = Operand(coords);
3599    load->operands[1] = Operand(fmask_desc_ptr);
3600    load->definitions[0] = Definition(fmask);
3601    load->glc = false;
3602    load->dlc = false;
3603    load->dmask = 0x1;
3604    load->unrm = true;
3605    load->da = da;
3606    load->dim = dim;
3607    load->can_reorder = true; /* fmask images shouldn't be modified */
3608    ctx->block->instructions.emplace_back(std::move(load));
3609
3610    Operand sample_index4;
3611    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3612       sample_index4 = Operand(sample_index.constantValue() << 2);
3613    } else if (sample_index.regClass() == s1) {
3614       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3615    } else {
3616       assert(sample_index.regClass() == v1);
3617       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3618    }
3619
3620    Temp final_sample;
3621    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3622       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3623    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3624       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3625    else
3626       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3627
3628    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3629     * resource descriptor is 0 (invalid),
3630     */
3631    Temp compare = bld.tmp(s2);
3632    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3633                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3634
3635    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3636
3637    /* Replace the MSAA sample index. */
3638    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3639 }
3640
3641 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3642 {
3643
3644    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3645    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3646    bool is_array = glsl_sampler_type_is_array(type);
3647    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3648    assert(!add_frag_pos && "Input attachments should be lowered.");
3649    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3650    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3651    int count = image_type_to_components_count(dim, is_array);
3652    std::vector<Operand> coords(count);
3653
3654    if (is_ms) {
3655       Operand sample_index;
3656       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3657       if (sample_cv)
3658          sample_index = Operand(sample_cv->u32);
3659       else
3660          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3661
3662       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3663          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3664          for (unsigned i = 0; i < vec->operands.size(); i++)
3665             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3666          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3667          vec->definitions[0] = Definition(fmask_load_address);
3668          ctx->block->instructions.emplace_back(std::move(vec));
3669
3670          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3671          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3672       }
3673       count--;
3674       coords[count] = sample_index;
3675    }
3676
3677    if (count == 1 && !gfx9_1d)
3678       return emit_extract_vector(ctx, src0, 0, v1);
3679
3680    if (gfx9_1d) {
3681       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3682       coords.resize(coords.size() + 1);
3683       coords[1] = Operand((uint32_t) 0);
3684       if (is_array)
3685          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3686    } else {
3687       for (int i = 0; i < count; i++)
3688          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3689    }
3690
3691    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3692    for (unsigned i = 0; i < coords.size(); i++)
3693       vec->operands[i] = coords[i];
3694    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3695    vec->definitions[0] = Definition(res);
3696    ctx->block->instructions.emplace_back(std::move(vec));
3697    return res;
3698 }
3699
3700
3701 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
3702 {
3703    Builder bld(ctx->program, ctx->block);
3704    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3705    const struct glsl_type *type = glsl_without_array(var->type);
3706    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3707    bool is_array = glsl_sampler_type_is_array(type);
3708    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3709
3710    if (dim == GLSL_SAMPLER_DIM_BUF) {
3711       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
3712       unsigned num_channels = util_last_bit(mask);
3713       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3714       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3715
3716       aco_opcode opcode;
3717       switch (num_channels) {
3718       case 1:
3719          opcode = aco_opcode::buffer_load_format_x;
3720          break;
3721       case 2:
3722          opcode = aco_opcode::buffer_load_format_xy;
3723          break;
3724       case 3:
3725          opcode = aco_opcode::buffer_load_format_xyz;
3726          break;
3727       case 4:
3728          opcode = aco_opcode::buffer_load_format_xyzw;
3729          break;
3730       default:
3731          unreachable(">4 channel buffer image load");
3732       }
3733       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
3734       load->operands[0] = Operand(vindex);
3735       load->operands[1] = Operand(rsrc);
3736       load->operands[2] = Operand((uint32_t) 0);
3737       Temp tmp;
3738       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3739          tmp = dst;
3740       else
3741          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
3742       load->definitions[0] = Definition(tmp);
3743       load->idxen = true;
3744       load->barrier = barrier_image;
3745       ctx->block->instructions.emplace_back(std::move(load));
3746
3747       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
3748       return;
3749    }
3750
3751    Temp coords = get_image_coords(ctx, instr, type);
3752    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3753
3754    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
3755    unsigned num_components = util_bitcount(dmask);
3756    Temp tmp;
3757    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3758       tmp = dst;
3759    else
3760       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
3761
3762    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3763    load->operands[0] = Operand(coords);
3764    load->operands[1] = Operand(resource);
3765    load->definitions[0] = Definition(tmp);
3766    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
3767    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
3768    load->dmask = dmask;
3769    load->unrm = true;
3770    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3771    load->barrier = barrier_image;
3772    ctx->block->instructions.emplace_back(std::move(load));
3773
3774    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
3775    return;
3776 }
3777
3778 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
3779 {
3780    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3781    const struct glsl_type *type = glsl_without_array(var->type);
3782    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3783    bool is_array = glsl_sampler_type_is_array(type);
3784    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3785
3786    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
3787
3788    if (dim == GLSL_SAMPLER_DIM_BUF) {
3789       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3790       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3791       aco_opcode opcode;
3792       switch (data.size()) {
3793       case 1:
3794          opcode = aco_opcode::buffer_store_format_x;
3795          break;
3796       case 2:
3797          opcode = aco_opcode::buffer_store_format_xy;
3798          break;
3799       case 3:
3800          opcode = aco_opcode::buffer_store_format_xyz;
3801          break;
3802       case 4:
3803          opcode = aco_opcode::buffer_store_format_xyzw;
3804          break;
3805       default:
3806          unreachable(">4 channel buffer image store");
3807       }
3808       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
3809       store->operands[0] = Operand(vindex);
3810       store->operands[1] = Operand(rsrc);
3811       store->operands[2] = Operand((uint32_t) 0);
3812       store->operands[3] = Operand(data);
3813       store->idxen = true;
3814       store->glc = glc;
3815       store->dlc = false;
3816       store->disable_wqm = true;
3817       store->barrier = barrier_image;
3818       ctx->program->needs_exact = true;
3819       ctx->block->instructions.emplace_back(std::move(store));
3820       return;
3821    }
3822
3823    assert(data.type() == RegType::vgpr);
3824    Temp coords = get_image_coords(ctx, instr, type);
3825    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3826
3827    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
3828    store->operands[0] = Operand(coords);
3829    store->operands[1] = Operand(resource);
3830    store->operands[2] = Operand(s4);
3831    store->operands[3] = Operand(data);
3832    store->glc = glc;
3833    store->dlc = false;
3834    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
3835    store->dmask = (1 << data.size()) - 1;
3836    store->unrm = true;
3837    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3838    store->disable_wqm = true;
3839    store->barrier = barrier_image;
3840    ctx->program->needs_exact = true;
3841    ctx->block->instructions.emplace_back(std::move(store));
3842    return;
3843 }
3844
3845 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
3846 {
3847    /* return the previous value if dest is ever used */
3848    bool return_previous = false;
3849    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
3850       return_previous = true;
3851       break;
3852    }
3853    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
3854       return_previous = true;
3855       break;
3856    }
3857
3858    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3859    const struct glsl_type *type = glsl_without_array(var->type);
3860    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3861    bool is_array = glsl_sampler_type_is_array(type);
3862    Builder bld(ctx->program, ctx->block);
3863
3864    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3865    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
3866
3867    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
3868       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
3869
3870    aco_opcode buf_op, image_op;
3871    switch (instr->intrinsic) {
3872       case nir_intrinsic_image_deref_atomic_add:
3873          buf_op = aco_opcode::buffer_atomic_add;
3874          image_op = aco_opcode::image_atomic_add;
3875          break;
3876       case nir_intrinsic_image_deref_atomic_umin:
3877          buf_op = aco_opcode::buffer_atomic_umin;
3878          image_op = aco_opcode::image_atomic_umin;
3879          break;
3880       case nir_intrinsic_image_deref_atomic_imin:
3881          buf_op = aco_opcode::buffer_atomic_smin;
3882          image_op = aco_opcode::image_atomic_smin;
3883          break;
3884       case nir_intrinsic_image_deref_atomic_umax:
3885          buf_op = aco_opcode::buffer_atomic_umax;
3886          image_op = aco_opcode::image_atomic_umax;
3887          break;
3888       case nir_intrinsic_image_deref_atomic_imax:
3889          buf_op = aco_opcode::buffer_atomic_smax;
3890          image_op = aco_opcode::image_atomic_smax;
3891          break;
3892       case nir_intrinsic_image_deref_atomic_and:
3893          buf_op = aco_opcode::buffer_atomic_and;
3894          image_op = aco_opcode::image_atomic_and;
3895          break;
3896       case nir_intrinsic_image_deref_atomic_or:
3897          buf_op = aco_opcode::buffer_atomic_or;
3898          image_op = aco_opcode::image_atomic_or;
3899          break;
3900       case nir_intrinsic_image_deref_atomic_xor:
3901          buf_op = aco_opcode::buffer_atomic_xor;
3902          image_op = aco_opcode::image_atomic_xor;
3903          break;
3904       case nir_intrinsic_image_deref_atomic_exchange:
3905          buf_op = aco_opcode::buffer_atomic_swap;
3906          image_op = aco_opcode::image_atomic_swap;
3907          break;
3908       case nir_intrinsic_image_deref_atomic_comp_swap:
3909          buf_op = aco_opcode::buffer_atomic_cmpswap;
3910          image_op = aco_opcode::image_atomic_cmpswap;
3911          break;
3912       default:
3913          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
3914    }
3915
3916    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3917
3918    if (dim == GLSL_SAMPLER_DIM_BUF) {
3919       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3920       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3921       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
3922       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
3923       mubuf->operands[0] = Operand(vindex);
3924       mubuf->operands[1] = Operand(resource);
3925       mubuf->operands[2] = Operand((uint32_t)0);
3926       mubuf->operands[3] = Operand(data);
3927       if (return_previous)
3928          mubuf->definitions[0] = Definition(dst);
3929       mubuf->offset = 0;
3930       mubuf->idxen = true;
3931       mubuf->glc = return_previous;
3932       mubuf->dlc = false; /* Not needed for atomics */
3933       mubuf->disable_wqm = true;
3934       mubuf->barrier = barrier_image;
3935       ctx->program->needs_exact = true;
3936       ctx->block->instructions.emplace_back(std::move(mubuf));
3937       return;
3938    }
3939
3940    Temp coords = get_image_coords(ctx, instr, type);
3941    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3942    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
3943    mimg->operands[0] = Operand(coords);
3944    mimg->operands[1] = Operand(resource);
3945    mimg->operands[2] = Operand(s4); /* no sampler */
3946    mimg->operands[3] = Operand(data);
3947    if (return_previous)
3948       mimg->definitions[0] = Definition(dst);
3949    mimg->glc = return_previous;
3950    mimg->dlc = false; /* Not needed for atomics */
3951    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
3952    mimg->dmask = (1 << data.size()) - 1;
3953    mimg->unrm = true;
3954    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3955    mimg->disable_wqm = true;
3956    mimg->barrier = barrier_image;
3957    ctx->program->needs_exact = true;
3958    ctx->block->instructions.emplace_back(std::move(mimg));
3959    return;
3960 }
3961
3962 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
3963 {
3964    if (in_elements && ctx->options->chip_class == GFX8) {
3965       Builder bld(ctx->program, ctx->block);
3966
3967       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
3968       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
3969       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
3970       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
3971
3972       Temp size = emit_extract_vector(ctx, desc, 2, s1);
3973       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
3974
3975       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
3976       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
3977       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
3978
3979       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
3980       /* idea
3981        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
3982        * in case 12 (or 3?), we have to divide by 3:
3983        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
3984        * use v_mul_hi_u32 with magic number to divide
3985        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
3986        * disable v_skip
3987        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
3988        */
3989
3990    } else {
3991       emit_extract_vector(ctx, desc, 2, dst);
3992    }
3993 }
3994
3995 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
3996 {
3997    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3998    const struct glsl_type *type = glsl_without_array(var->type);
3999    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4000    bool is_array = glsl_sampler_type_is_array(type);
4001    Builder bld(ctx->program, ctx->block);
4002
4003    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4004       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4005       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4006    }
4007
4008    /* LOD */
4009    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4010
4011    /* Resource */
4012    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4013
4014    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4015
4016    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4017    mimg->operands[0] = Operand(lod);
4018    mimg->operands[1] = Operand(resource);
4019    unsigned& dmask = mimg->dmask;
4020    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4021    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4022    mimg->da = glsl_sampler_type_is_array(type);
4023    mimg->can_reorder = true;
4024    Definition& def = mimg->definitions[0];
4025    ctx->block->instructions.emplace_back(std::move(mimg));
4026
4027    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4028        glsl_sampler_type_is_array(type)) {
4029
4030       assert(instr->dest.ssa.num_components == 3);
4031       Temp tmp = {ctx->program->allocateId(), v3};
4032       def = Definition(tmp);
4033       emit_split_vector(ctx, tmp, 3);
4034
4035       /* divide 3rd value by 6 by multiplying with magic number */
4036       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4037       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4038
4039       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4040                  emit_extract_vector(ctx, tmp, 0, v1),
4041                  emit_extract_vector(ctx, tmp, 1, v1),
4042                  by_6);
4043
4044    } else if (ctx->options->chip_class == GFX9 &&
4045               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4046               glsl_sampler_type_is_array(type)) {
4047       assert(instr->dest.ssa.num_components == 2);
4048       def = Definition(dst);
4049       dmask = 0x5;
4050    } else {
4051       def = Definition(dst);
4052    }
4053
4054    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4055 }
4056
4057 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4058 {
4059    Builder bld(ctx->program, ctx->block);
4060    unsigned num_components = instr->num_components;
4061
4062    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4063    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4064    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4065
4066    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4067    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc);
4068 }
4069
4070 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4071 {
4072    Builder bld(ctx->program, ctx->block);
4073    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4074    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4075    unsigned writemask = nir_intrinsic_write_mask(instr);
4076
4077    Temp offset;
4078    if (ctx->options->chip_class < GFX8)
4079       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4080    else
4081       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4082
4083    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4084    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4085
4086    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4087                ctx->options->chip_class >= GFX8;
4088    if (smem)
4089       offset = bld.as_uniform(offset);
4090    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4091
4092    while (writemask) {
4093       int start, count;
4094       u_bit_scan_consecutive_range(&writemask, &start, &count);
4095       if (count == 3 && smem) {
4096          writemask |= 1u << (start + 2);
4097          count = 2;
4098       }
4099       int num_bytes = count * elem_size_bytes;
4100
4101       if (num_bytes > 16) {
4102          assert(elem_size_bytes == 8);
4103          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4104          count = 2;
4105          num_bytes = 16;
4106       }
4107
4108       // TODO: check alignment of sub-dword stores
4109       // TODO: split 3 bytes. there is no store instruction for that
4110
4111       Temp write_data;
4112       if (count != instr->num_components) {
4113          emit_split_vector(ctx, data, instr->num_components);
4114          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4115          for (int i = 0; i < count; i++) {
4116             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4117             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4118          }
4119          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4120          vec->definitions[0] = Definition(write_data);
4121          ctx->block->instructions.emplace_back(std::move(vec));
4122       } else if (!smem && data.type() != RegType::vgpr) {
4123          assert(num_bytes % 4 == 0);
4124          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4125       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4126          assert(num_bytes % 4 == 0);
4127          write_data = bld.as_uniform(data);
4128       } else {
4129          write_data = data;
4130       }
4131
4132       aco_opcode vmem_op, smem_op;
4133       switch (num_bytes) {
4134          case 4:
4135             vmem_op = aco_opcode::buffer_store_dword;
4136             smem_op = aco_opcode::s_buffer_store_dword;
4137             break;
4138          case 8:
4139             vmem_op = aco_opcode::buffer_store_dwordx2;
4140             smem_op = aco_opcode::s_buffer_store_dwordx2;
4141             break;
4142          case 12:
4143             vmem_op = aco_opcode::buffer_store_dwordx3;
4144             smem_op = aco_opcode::last_opcode;
4145             assert(!smem);
4146             break;
4147          case 16:
4148             vmem_op = aco_opcode::buffer_store_dwordx4;
4149             smem_op = aco_opcode::s_buffer_store_dwordx4;
4150             break;
4151          default:
4152             unreachable("Store SSBO not implemented for this size.");
4153       }
4154       if (ctx->stage == fragment_fs)
4155          smem_op = aco_opcode::p_fs_buffer_store_smem;
4156
4157       if (smem) {
4158          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4159          store->operands[0] = Operand(rsrc);
4160          if (start) {
4161             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4162                                 offset, Operand(start * elem_size_bytes));
4163             store->operands[1] = Operand(off);
4164          } else {
4165             store->operands[1] = Operand(offset);
4166          }
4167          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4168             store->operands[1].setFixed(m0);
4169          store->operands[2] = Operand(write_data);
4170          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4171          store->dlc = false;
4172          store->disable_wqm = true;
4173          store->barrier = barrier_buffer;
4174          ctx->block->instructions.emplace_back(std::move(store));
4175          ctx->program->wb_smem_l1_on_end = true;
4176          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4177             ctx->block->kind |= block_kind_needs_lowering;
4178             ctx->program->needs_exact = true;
4179          }
4180       } else {
4181          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4182          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4183          store->operands[1] = Operand(rsrc);
4184          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4185          store->operands[3] = Operand(write_data);
4186          store->offset = start * elem_size_bytes;
4187          store->offen = (offset.type() == RegType::vgpr);
4188          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4189          store->dlc = false;
4190          store->disable_wqm = true;
4191          store->barrier = barrier_buffer;
4192          ctx->program->needs_exact = true;
4193          ctx->block->instructions.emplace_back(std::move(store));
4194       }
4195    }
4196 }
4197
4198 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4199 {
4200    /* return the previous value if dest is ever used */
4201    bool return_previous = false;
4202    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4203       return_previous = true;
4204       break;
4205    }
4206    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4207       return_previous = true;
4208       break;
4209    }
4210
4211    Builder bld(ctx->program, ctx->block);
4212    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4213
4214    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4215       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4216                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4217
4218    Temp offset;
4219    if (ctx->options->chip_class < GFX8)
4220       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4221    else
4222       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4223
4224    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4225    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4226
4227    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4228
4229    aco_opcode op32, op64;
4230    switch (instr->intrinsic) {
4231       case nir_intrinsic_ssbo_atomic_add:
4232          op32 = aco_opcode::buffer_atomic_add;
4233          op64 = aco_opcode::buffer_atomic_add_x2;
4234          break;
4235       case nir_intrinsic_ssbo_atomic_imin:
4236          op32 = aco_opcode::buffer_atomic_smin;
4237          op64 = aco_opcode::buffer_atomic_smin_x2;
4238          break;
4239       case nir_intrinsic_ssbo_atomic_umin:
4240          op32 = aco_opcode::buffer_atomic_umin;
4241          op64 = aco_opcode::buffer_atomic_umin_x2;
4242          break;
4243       case nir_intrinsic_ssbo_atomic_imax:
4244          op32 = aco_opcode::buffer_atomic_smax;
4245          op64 = aco_opcode::buffer_atomic_smax_x2;
4246          break;
4247       case nir_intrinsic_ssbo_atomic_umax:
4248          op32 = aco_opcode::buffer_atomic_umax;
4249          op64 = aco_opcode::buffer_atomic_umax_x2;
4250          break;
4251       case nir_intrinsic_ssbo_atomic_and:
4252          op32 = aco_opcode::buffer_atomic_and;
4253          op64 = aco_opcode::buffer_atomic_and_x2;
4254          break;
4255       case nir_intrinsic_ssbo_atomic_or:
4256          op32 = aco_opcode::buffer_atomic_or;
4257          op64 = aco_opcode::buffer_atomic_or_x2;
4258          break;
4259       case nir_intrinsic_ssbo_atomic_xor:
4260          op32 = aco_opcode::buffer_atomic_xor;
4261          op64 = aco_opcode::buffer_atomic_xor_x2;
4262          break;
4263       case nir_intrinsic_ssbo_atomic_exchange:
4264          op32 = aco_opcode::buffer_atomic_swap;
4265          op64 = aco_opcode::buffer_atomic_swap_x2;
4266          break;
4267       case nir_intrinsic_ssbo_atomic_comp_swap:
4268          op32 = aco_opcode::buffer_atomic_cmpswap;
4269          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4270          break;
4271       default:
4272          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4273    }
4274    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4275    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4276    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4277    mubuf->operands[1] = Operand(rsrc);
4278    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4279    mubuf->operands[3] = Operand(data);
4280    if (return_previous)
4281       mubuf->definitions[0] = Definition(dst);
4282    mubuf->offset = 0;
4283    mubuf->offen = (offset.type() == RegType::vgpr);
4284    mubuf->glc = return_previous;
4285    mubuf->dlc = false; /* Not needed for atomics */
4286    mubuf->disable_wqm = true;
4287    mubuf->barrier = barrier_buffer;
4288    ctx->program->needs_exact = true;
4289    ctx->block->instructions.emplace_back(std::move(mubuf));
4290 }
4291
4292 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4293
4294    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4295    Builder bld(ctx->program, ctx->block);
4296    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4297    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4298 }
4299
4300 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4301 {
4302    Builder bld(ctx->program, ctx->block);
4303    unsigned num_components = instr->num_components;
4304    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4305
4306    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4307    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4308
4309    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4310    bool dlc = glc && ctx->options->chip_class >= GFX10;
4311    aco_opcode op;
4312    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4313       bool global = ctx->options->chip_class >= GFX9;
4314       aco_opcode op;
4315       switch (num_bytes) {
4316       case 4:
4317          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4318          break;
4319       case 8:
4320          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4321          break;
4322       case 12:
4323          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4324          break;
4325       case 16:
4326          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4327          break;
4328       default:
4329          unreachable("load_global not implemented for this size.");
4330       }
4331       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4332       flat->operands[0] = Operand(addr);
4333       flat->operands[1] = Operand(s1);
4334       flat->glc = glc;
4335       flat->dlc = dlc;
4336
4337       if (dst.type() == RegType::sgpr) {
4338          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4339          flat->definitions[0] = Definition(vec);
4340          ctx->block->instructions.emplace_back(std::move(flat));
4341          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4342       } else {
4343          flat->definitions[0] = Definition(dst);
4344          ctx->block->instructions.emplace_back(std::move(flat));
4345       }
4346       emit_split_vector(ctx, dst, num_components);
4347    } else {
4348       switch (num_bytes) {
4349          case 4:
4350             op = aco_opcode::s_load_dword;
4351             break;
4352          case 8:
4353             op = aco_opcode::s_load_dwordx2;
4354             break;
4355          case 12:
4356          case 16:
4357             op = aco_opcode::s_load_dwordx4;
4358             break;
4359          default:
4360             unreachable("load_global not implemented for this size.");
4361       }
4362       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4363       load->operands[0] = Operand(addr);
4364       load->operands[1] = Operand(0u);
4365       load->definitions[0] = Definition(dst);
4366       load->glc = glc;
4367       load->dlc = dlc;
4368       load->barrier = barrier_buffer;
4369       assert(ctx->options->chip_class >= GFX8 || !glc);
4370
4371       if (dst.size() == 3) {
4372          /* trim vector */
4373          Temp vec = bld.tmp(s4);
4374          load->definitions[0] = Definition(vec);
4375          ctx->block->instructions.emplace_back(std::move(load));
4376          emit_split_vector(ctx, vec, 4);
4377
4378          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4379                     emit_extract_vector(ctx, vec, 0, s1),
4380                     emit_extract_vector(ctx, vec, 1, s1),
4381                     emit_extract_vector(ctx, vec, 2, s1));
4382       } else {
4383          ctx->block->instructions.emplace_back(std::move(load));
4384       }
4385    }
4386 }
4387
4388 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4389 {
4390    Builder bld(ctx->program, ctx->block);
4391    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4392
4393    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4394    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4395
4396    unsigned writemask = nir_intrinsic_write_mask(instr);
4397    while (writemask) {
4398       int start, count;
4399       u_bit_scan_consecutive_range(&writemask, &start, &count);
4400       unsigned num_bytes = count * elem_size_bytes;
4401
4402       Temp write_data = data;
4403       if (count != instr->num_components) {
4404          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4405          for (int i = 0; i < count; i++)
4406             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4407          write_data = bld.tmp(RegType::vgpr, count);
4408          vec->definitions[0] = Definition(write_data);
4409          ctx->block->instructions.emplace_back(std::move(vec));
4410       }
4411
4412       unsigned offset = start * elem_size_bytes;
4413       if (offset > 0 && ctx->options->chip_class < GFX9) {
4414          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4415          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4416          Temp carry = bld.tmp(s2);
4417          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4418
4419          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4420                   Operand(offset), addr0);
4421          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4422                   Operand(0u), addr1,
4423                   carry).def(1).setHint(vcc);
4424
4425          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4426
4427          offset = 0;
4428       }
4429
4430       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4431       bool global = ctx->options->chip_class >= GFX9;
4432       aco_opcode op;
4433       switch (num_bytes) {
4434       case 4:
4435          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4436          break;
4437       case 8:
4438          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4439          break;
4440       case 12:
4441          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4442          break;
4443       case 16:
4444          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4445          break;
4446       default:
4447          unreachable("store_global not implemented for this size.");
4448       }
4449       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4450       flat->operands[0] = Operand(addr);
4451       flat->operands[1] = Operand(s1);
4452       flat->operands[2] = Operand(data);
4453       flat->glc = glc;
4454       flat->dlc = false;
4455       flat->offset = offset;
4456       ctx->block->instructions.emplace_back(std::move(flat));
4457    }
4458 }
4459
4460 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4461    Builder bld(ctx->program, ctx->block);
4462    switch(instr->intrinsic) {
4463       case nir_intrinsic_group_memory_barrier:
4464       case nir_intrinsic_memory_barrier:
4465          bld.barrier(aco_opcode::p_memory_barrier_all);
4466          break;
4467       case nir_intrinsic_memory_barrier_atomic_counter:
4468          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4469          break;
4470       case nir_intrinsic_memory_barrier_buffer:
4471          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4472          break;
4473       case nir_intrinsic_memory_barrier_image:
4474          bld.barrier(aco_opcode::p_memory_barrier_image);
4475          break;
4476       case nir_intrinsic_memory_barrier_shared:
4477          bld.barrier(aco_opcode::p_memory_barrier_shared);
4478          break;
4479       default:
4480          unreachable("Unimplemented memory barrier intrinsic");
4481          break;
4482    }
4483 }
4484
4485 Operand load_lds_size_m0(isel_context *ctx)
4486 {
4487    /* TODO: m0 does not need to be initialized on GFX9+ */
4488    Builder bld(ctx->program, ctx->block);
4489    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
4490 }
4491
4492
4493 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4494 {
4495    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4496    Operand m = load_lds_size_m0(ctx);
4497    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4498    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4499    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4500    Builder bld(ctx->program, ctx->block);
4501
4502    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4503    unsigned bytes_read = 0;
4504    unsigned result_size = 0;
4505    unsigned total_bytes = instr->num_components * elem_size_bytes;
4506    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : instr->dest.ssa.bit_size / 8;
4507    std::array<Temp, 4> result;
4508
4509    while (bytes_read < total_bytes) {
4510       unsigned todo = total_bytes - bytes_read;
4511       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
4512       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
4513
4514       aco_opcode op = aco_opcode::last_opcode;
4515       if (todo >= 16 && aligned16) {
4516          op = aco_opcode::ds_read_b128;
4517          todo = 16;
4518       } else if (todo >= 12 && aligned16) {
4519          op = aco_opcode::ds_read_b96;
4520          todo = 12;
4521       } else if (todo >= 8) {
4522          op = aligned8 ? aco_opcode::ds_read_b64 : aco_opcode::ds_read2_b32;
4523          todo = 8;
4524       } else if (todo >= 4) {
4525          op = aco_opcode::ds_read_b32;
4526          todo = 4;
4527       } else {
4528          assert(false);
4529       }
4530       assert(todo % elem_size_bytes == 0);
4531       unsigned num_elements = todo / elem_size_bytes;
4532       unsigned offset = nir_intrinsic_base(instr) + bytes_read;
4533       unsigned max_offset = op == aco_opcode::ds_read2_b32 ? 1019 : 65535;
4534
4535       Temp address_offset = address;
4536       if (offset > max_offset) {
4537          address_offset = bld.vadd32(bld.def(v1), Operand((uint32_t)nir_intrinsic_base(instr)), address_offset);
4538          offset = bytes_read;
4539       }
4540       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
4541
4542       Temp res;
4543       if (instr->num_components == 1 && dst.type() == RegType::vgpr)
4544          res = dst;
4545       else
4546          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
4547
4548       if (op == aco_opcode::ds_read2_b32)
4549          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
4550       else
4551          res = bld.ds(op, Definition(res), address_offset, m, offset);
4552
4553       if (instr->num_components == 1) {
4554          assert(todo == total_bytes);
4555          if (dst.type() == RegType::sgpr)
4556             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4557          return;
4558       }
4559
4560       if (dst.type() == RegType::sgpr)
4561          res = bld.as_uniform(res);
4562
4563       if (num_elements == 1) {
4564          result[result_size++] = res;
4565       } else {
4566          assert(res != dst && res.size() % num_elements == 0);
4567          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
4568          split->operands[0] = Operand(res);
4569          for (unsigned i = 0; i < num_elements; i++)
4570             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
4571          ctx->block->instructions.emplace_back(std::move(split));
4572       }
4573
4574       bytes_read += todo;
4575    }
4576
4577    assert(result_size == instr->num_components && result_size > 1);
4578    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
4579    for (unsigned i = 0; i < result_size; i++)
4580       vec->operands[i] = Operand(result[i]);
4581    vec->definitions[0] = Definition(dst);
4582    ctx->block->instructions.emplace_back(std::move(vec));
4583    ctx->allocated_vec.emplace(dst.id(), result);
4584 }
4585
4586 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned offset0, unsigned offset1, unsigned align)
4587 {
4588    Builder bld(ctx->program, ctx->block);
4589    unsigned bytes_written = 0;
4590    while (bytes_written < data.size() * 4) {
4591       unsigned todo = data.size() * 4 - bytes_written;
4592       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
4593       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
4594
4595       aco_opcode op = aco_opcode::last_opcode;
4596       unsigned size = 0;
4597       if (todo >= 16 && aligned16) {
4598          op = aco_opcode::ds_write_b128;
4599          size = 4;
4600       } else if (todo >= 12 && aligned16) {
4601          op = aco_opcode::ds_write_b96;
4602          size = 3;
4603       } else if (todo >= 8) {
4604          op = aligned8 ? aco_opcode::ds_write_b64 : aco_opcode::ds_write2_b32;
4605          size = 2;
4606       } else if (todo >= 4) {
4607          op = aco_opcode::ds_write_b32;
4608          size = 1;
4609       } else {
4610          assert(false);
4611       }
4612
4613       bool write2 = op == aco_opcode::ds_write2_b32;
4614       unsigned offset = offset0 + offset1 + bytes_written;
4615       unsigned max_offset = write2 ? 1020 : 65535;
4616       Temp address_offset = address;
4617       if (offset > max_offset) {
4618          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
4619          offset = offset1 + bytes_written;
4620       }
4621       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
4622
4623       if (write2) {
4624          Temp val0 = emit_extract_vector(ctx, data, bytes_written >> 2, v1);
4625          Temp val1 = emit_extract_vector(ctx, data, (bytes_written >> 2) + 1, v1);
4626          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
4627       } else {
4628          Temp val = emit_extract_vector(ctx, data, bytes_written >> 2, RegClass(RegType::vgpr, size));
4629          bld.ds(op, address_offset, val, m, offset);
4630       }
4631
4632       bytes_written += size * 4;
4633    }
4634 }
4635
4636 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4637 {
4638    unsigned offset = nir_intrinsic_base(instr);
4639    unsigned writemask = nir_intrinsic_write_mask(instr);
4640    Operand m = load_lds_size_m0(ctx);
4641    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4642    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4643    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4644    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4645
4646    /* we need at most two stores for 32bit variables */
4647    int start[2], count[2];
4648    u_bit_scan_consecutive_range(&writemask, &start[0], &count[0]);
4649    u_bit_scan_consecutive_range(&writemask, &start[1], &count[1]);
4650    assert(writemask == 0);
4651
4652    /* one combined store is sufficient */
4653    if (count[0] == count[1]) {
4654       Builder bld(ctx->program, ctx->block);
4655
4656       Temp address_offset = address;
4657       if ((offset >> 2) + start[1] > 255) {
4658          address_offset = bld.vadd32(bld.def(v1), Operand(offset), address_offset);
4659          offset = 0;
4660       }
4661
4662       assert(count[0] == 1);
4663       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
4664       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
4665       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
4666       offset = offset / elem_size_bytes;
4667       bld.ds(op, address_offset, val0, val1, m,
4668              offset + start[0], offset + start[1]);
4669       return;
4670    }
4671
4672    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4673    for (unsigned i = 0; i < 2; i++) {
4674       if (count[i] == 0)
4675          continue;
4676
4677       Temp write_data = emit_extract_vector(ctx, data, start[i], RegClass(RegType::vgpr, count[i] * elem_size_bytes / 4));
4678       ds_write_helper(ctx, m, address, write_data, offset, start[i] * elem_size_bytes, align);
4679    }
4680    return;
4681 }
4682
4683 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4684 {
4685    unsigned offset = nir_intrinsic_base(instr);
4686    Operand m = load_lds_size_m0(ctx);
4687    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4688    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4689
4690    unsigned num_operands = 3;
4691    aco_opcode op32, op64, op32_rtn, op64_rtn;
4692    switch(instr->intrinsic) {
4693       case nir_intrinsic_shared_atomic_add:
4694          op32 = aco_opcode::ds_add_u32;
4695          op64 = aco_opcode::ds_add_u64;
4696          op32_rtn = aco_opcode::ds_add_rtn_u32;
4697          op64_rtn = aco_opcode::ds_add_rtn_u64;
4698          break;
4699       case nir_intrinsic_shared_atomic_imin:
4700          op32 = aco_opcode::ds_min_i32;
4701          op64 = aco_opcode::ds_min_i64;
4702          op32_rtn = aco_opcode::ds_min_rtn_i32;
4703          op64_rtn = aco_opcode::ds_min_rtn_i64;
4704          break;
4705       case nir_intrinsic_shared_atomic_umin:
4706          op32 = aco_opcode::ds_min_u32;
4707          op64 = aco_opcode::ds_min_u64;
4708          op32_rtn = aco_opcode::ds_min_rtn_u32;
4709          op64_rtn = aco_opcode::ds_min_rtn_u64;
4710          break;
4711       case nir_intrinsic_shared_atomic_imax:
4712          op32 = aco_opcode::ds_max_i32;
4713          op64 = aco_opcode::ds_max_i64;
4714          op32_rtn = aco_opcode::ds_max_rtn_i32;
4715          op64_rtn = aco_opcode::ds_max_rtn_i64;
4716          break;
4717       case nir_intrinsic_shared_atomic_umax:
4718          op32 = aco_opcode::ds_max_u32;
4719          op64 = aco_opcode::ds_max_u64;
4720          op32_rtn = aco_opcode::ds_max_rtn_u32;
4721          op64_rtn = aco_opcode::ds_max_rtn_u64;
4722          break;
4723       case nir_intrinsic_shared_atomic_and:
4724          op32 = aco_opcode::ds_and_b32;
4725          op64 = aco_opcode::ds_and_b64;
4726          op32_rtn = aco_opcode::ds_and_rtn_b32;
4727          op64_rtn = aco_opcode::ds_and_rtn_b64;
4728          break;
4729       case nir_intrinsic_shared_atomic_or:
4730          op32 = aco_opcode::ds_or_b32;
4731          op64 = aco_opcode::ds_or_b64;
4732          op32_rtn = aco_opcode::ds_or_rtn_b32;
4733          op64_rtn = aco_opcode::ds_or_rtn_b64;
4734          break;
4735       case nir_intrinsic_shared_atomic_xor:
4736          op32 = aco_opcode::ds_xor_b32;
4737          op64 = aco_opcode::ds_xor_b64;
4738          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4739          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4740          break;
4741       case nir_intrinsic_shared_atomic_exchange:
4742          op32 = aco_opcode::ds_write_b32;
4743          op64 = aco_opcode::ds_write_b64;
4744          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4745          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4746          break;
4747       case nir_intrinsic_shared_atomic_comp_swap:
4748          op32 = aco_opcode::ds_cmpst_b32;
4749          op64 = aco_opcode::ds_cmpst_b64;
4750          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4751          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4752          num_operands = 4;
4753          break;
4754       default:
4755          unreachable("Unhandled shared atomic intrinsic");
4756    }
4757
4758    /* return the previous value if dest is ever used */
4759    bool return_previous = false;
4760    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4761       return_previous = true;
4762       break;
4763    }
4764    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4765       return_previous = true;
4766       break;
4767    }
4768
4769    aco_opcode op;
4770    if (data.size() == 1) {
4771       assert(instr->dest.ssa.bit_size == 32);
4772       op = return_previous ? op32_rtn : op32;
4773    } else {
4774       assert(instr->dest.ssa.bit_size == 64);
4775       op = return_previous ? op64_rtn : op64;
4776    }
4777
4778    if (offset > 65535) {
4779       Builder bld(ctx->program, ctx->block);
4780       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4781       offset = 0;
4782    }
4783
4784    aco_ptr<DS_instruction> ds;
4785    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4786    ds->operands[0] = Operand(address);
4787    ds->operands[1] = Operand(data);
4788    if (num_operands == 4)
4789       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4790    ds->operands[num_operands - 1] = m;
4791    ds->offset0 = offset;
4792    if (return_previous)
4793       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4794    ctx->block->instructions.emplace_back(std::move(ds));
4795 }
4796
4797 Temp get_scratch_resource(isel_context *ctx)
4798 {
4799    Builder bld(ctx->program, ctx->block);
4800    Temp scratch_addr = ctx->private_segment_buffer;
4801    if (ctx->stage != compute_cs)
4802       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), ctx->private_segment_buffer, Operand(0u));
4803
4804    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
4805                         S_008F0C_INDEX_STRIDE(ctx->options->wave_size == 64 ? 3 : 2);;
4806
4807    if (ctx->program->chip_class >= GFX10) {
4808       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4809                    S_008F0C_OOB_SELECT(3) |
4810                    S_008F0C_RESOURCE_LEVEL(1);
4811    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
4812       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4813                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4814    }
4815
4816    /* older generations need element size = 16 bytes. element size removed in GFX9 */
4817    if (ctx->program->chip_class <= GFX8)
4818       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
4819
4820    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4821 }
4822
4823 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4824    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4825    Builder bld(ctx->program, ctx->block);
4826    Temp rsrc = get_scratch_resource(ctx);
4827    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4828    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4829
4830    aco_opcode op;
4831    switch (dst.size()) {
4832       case 1:
4833          op = aco_opcode::buffer_load_dword;
4834          break;
4835       case 2:
4836          op = aco_opcode::buffer_load_dwordx2;
4837          break;
4838       case 3:
4839          op = aco_opcode::buffer_load_dwordx3;
4840          break;
4841       case 4:
4842          op = aco_opcode::buffer_load_dwordx4;
4843          break;
4844       case 6:
4845       case 8: {
4846          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4847          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4848                                 bld.def(v4), offset, rsrc,
4849                                 ctx->scratch_offset, 0, true);
4850          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4851                                                   aco_opcode::buffer_load_dwordx4,
4852                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4853                                 offset, rsrc, ctx->scratch_offset, 16, true);
4854          emit_split_vector(ctx, lower, 2);
4855          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4856          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4857          if (dst.size() == 8) {
4858             emit_split_vector(ctx, upper, 2);
4859             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4860             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4861          } else {
4862             elems[2] = upper;
4863          }
4864
4865          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4866                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4867          for (unsigned i = 0; i < dst.size() / 2; i++)
4868             vec->operands[i] = Operand(elems[i]);
4869          vec->definitions[0] = Definition(dst);
4870          bld.insert(std::move(vec));
4871          ctx->allocated_vec.emplace(dst.id(), elems);
4872          return;
4873       }
4874       default:
4875          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
4876    }
4877
4878    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->scratch_offset, 0, true);
4879    emit_split_vector(ctx, dst, instr->num_components);
4880 }
4881
4882 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4883    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
4884    Builder bld(ctx->program, ctx->block);
4885    Temp rsrc = get_scratch_resource(ctx);
4886    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4887    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4888
4889    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4890    unsigned writemask = nir_intrinsic_write_mask(instr);
4891
4892    while (writemask) {
4893       int start, count;
4894       u_bit_scan_consecutive_range(&writemask, &start, &count);
4895       int num_bytes = count * elem_size_bytes;
4896
4897       if (num_bytes > 16) {
4898          assert(elem_size_bytes == 8);
4899          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4900          count = 2;
4901          num_bytes = 16;
4902       }
4903
4904       // TODO: check alignment of sub-dword stores
4905       // TODO: split 3 bytes. there is no store instruction for that
4906
4907       Temp write_data;
4908       if (count != instr->num_components) {
4909          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4910          for (int i = 0; i < count; i++) {
4911             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
4912             vec->operands[i] = Operand(elem);
4913          }
4914          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
4915          vec->definitions[0] = Definition(write_data);
4916          ctx->block->instructions.emplace_back(std::move(vec));
4917       } else {
4918          write_data = data;
4919       }
4920
4921       aco_opcode op;
4922       switch (num_bytes) {
4923          case 4:
4924             op = aco_opcode::buffer_store_dword;
4925             break;
4926          case 8:
4927             op = aco_opcode::buffer_store_dwordx2;
4928             break;
4929          case 12:
4930             op = aco_opcode::buffer_store_dwordx3;
4931             break;
4932          case 16:
4933             op = aco_opcode::buffer_store_dwordx4;
4934             break;
4935          default:
4936             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
4937       }
4938
4939       bld.mubuf(op, offset, rsrc, ctx->scratch_offset, write_data, start * elem_size_bytes, true);
4940    }
4941 }
4942
4943 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
4944    uint8_t log2_ps_iter_samples;
4945    if (ctx->program->info->ps.force_persample) {
4946       log2_ps_iter_samples =
4947          util_logbase2(ctx->options->key.fs.num_samples);
4948    } else {
4949       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
4950    }
4951
4952    /* The bit pattern matches that used by fixed function fragment
4953     * processing. */
4954    static const unsigned ps_iter_masks[] = {
4955       0xffff, /* not used */
4956       0x5555,
4957       0x1111,
4958       0x0101,
4959       0x0001,
4960    };
4961    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
4962
4963    Builder bld(ctx->program, ctx->block);
4964
4965    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
4966    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
4967    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
4968    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4969    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
4970 }
4971
4972 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
4973 {
4974    Builder bld(ctx->program, ctx->block);
4975
4976    if (cluster_size == 1) {
4977       return src;
4978    } if (op == nir_op_iand && cluster_size == 4) {
4979       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
4980       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
4981       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
4982                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
4983    } else if (op == nir_op_ior && cluster_size == 4) {
4984       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
4985       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
4986                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
4987    } else if (op == nir_op_iand && cluster_size == 64) {
4988       //subgroupAnd(val) -> (exec & ~val) == 0
4989       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
4990       return bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), tmp, Operand(0u));
4991    } else if (op == nir_op_ior && cluster_size == 64) {
4992       //subgroupOr(val) -> (val & exec) != 0
4993       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
4994    } else if (op == nir_op_ixor && cluster_size == 64) {
4995       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
4996       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
4997       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
4998       return bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
4999    } else {
5000       //subgroupClustered{And,Or,Xor}(val, n) ->
5001       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5002       //cluster_offset = ~(n - 1) & lane_id
5003       //cluster_mask = ((1 << n) - 1)
5004       //subgroupClusteredAnd():
5005       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5006       //subgroupClusteredOr():
5007       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5008       //subgroupClusteredXor():
5009       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5010       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5011                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5012       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5013
5014       Temp tmp;
5015       if (op == nir_op_iand)
5016          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5017       else
5018          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5019
5020       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5021       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5022       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5023       if (cluster_mask != 0xffffffff)
5024          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5025
5026       Definition cmp_def = Definition();
5027       if (op == nir_op_iand) {
5028          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5029       } else if (op == nir_op_ior) {
5030          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5031       } else if (op == nir_op_ixor) {
5032          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5033                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5034          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5035       }
5036       cmp_def.setHint(vcc);
5037       return cmp_def.getTemp();
5038    }
5039 }
5040
5041 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5042 {
5043    Builder bld(ctx->program, ctx->block);
5044
5045    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5046    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5047    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5048    Temp tmp;
5049    if (op == nir_op_iand)
5050       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5051    else
5052       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5053
5054    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5055    Temp lo = lohi.def(0).getTemp();
5056    Temp hi = lohi.def(1).getTemp();
5057    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5058                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5059
5060    Definition cmp_def = Definition();
5061    if (op == nir_op_iand)
5062       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5063    else if (op == nir_op_ior)
5064       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5065    else if (op == nir_op_ixor)
5066       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5067                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5068    cmp_def.setHint(vcc);
5069    return cmp_def.getTemp();
5070 }
5071
5072 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5073 {
5074    Builder bld(ctx->program, ctx->block);
5075
5076    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5077    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5078    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5079    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5080    if (op == nir_op_iand)
5081       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5082    else if (op == nir_op_ior)
5083       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5084    else if (op == nir_op_ixor)
5085       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5086
5087    assert(false);
5088    return Temp();
5089 }
5090
5091 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5092 {
5093    Builder bld(ctx->program, ctx->block);
5094    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5095    if (src.regClass().type() == RegType::vgpr) {
5096       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5097    } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5098       bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(dst), Operand(0u), Operand(src));
5099    } else if (src.regClass() == s1) {
5100       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5101    } else if (src.regClass() == s2) {
5102       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5103    } else {
5104       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5105       nir_print_instr(&instr->instr, stderr);
5106       fprintf(stderr, "\n");
5107    }
5108 }
5109
5110 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5111 {
5112    Builder bld(ctx->program, ctx->block);
5113    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5114    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5115
5116    /* Build DD X/Y */
5117    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5118    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5119    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5120    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5121    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5122    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5123
5124    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5125    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5126    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5127    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5128    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5129    Temp wqm1 = bld.tmp(v1);
5130    emit_wqm(ctx, tmp1, wqm1, true);
5131    Temp wqm2 = bld.tmp(v1);
5132    emit_wqm(ctx, tmp2, wqm2, true);
5133    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5134    return;
5135 }
5136
5137 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5138 {
5139    Builder bld(ctx->program, ctx->block);
5140    switch(instr->intrinsic) {
5141    case nir_intrinsic_load_barycentric_sample:
5142    case nir_intrinsic_load_barycentric_pixel:
5143    case nir_intrinsic_load_barycentric_centroid: {
5144       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5145       fs_input input = get_interp_input(instr->intrinsic, mode);
5146
5147       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5148       if (input == fs_input::max_inputs) {
5149          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5150                     Operand(0u), Operand(0u));
5151       } else {
5152          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5153                     ctx->fs_inputs[input],
5154                     ctx->fs_inputs[input + 1]);
5155       }
5156       emit_split_vector(ctx, dst, 2);
5157       break;
5158    }
5159    case nir_intrinsic_load_barycentric_at_sample: {
5160       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5161       switch (ctx->options->key.fs.num_samples) {
5162          case 2: sample_pos_offset += 1 << 3; break;
5163          case 4: sample_pos_offset += 3 << 3; break;
5164          case 8: sample_pos_offset += 7 << 3; break;
5165          default: break;
5166       }
5167       Temp sample_pos;
5168       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5169       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5170       if (addr.type() == RegType::sgpr) {
5171          Operand offset;
5172          if (const_addr) {
5173             sample_pos_offset += const_addr->u32 << 3;
5174             offset = Operand(sample_pos_offset);
5175          } else if (ctx->options->chip_class >= GFX9) {
5176             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5177          } else {
5178             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5179             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5180          }
5181          addr = ctx->private_segment_buffer;
5182          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), addr, Operand(offset));
5183
5184       } else if (ctx->options->chip_class >= GFX9) {
5185          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5186          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, ctx->private_segment_buffer, sample_pos_offset);
5187       } else {
5188          /* addr += ctx->private_segment_buffer + sample_pos_offset */
5189          Temp tmp0 = bld.tmp(s1);
5190          Temp tmp1 = bld.tmp(s1);
5191          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), ctx->private_segment_buffer);
5192          Definition scc_tmp = bld.def(s1, scc);
5193          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5194          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), scc_tmp.getTemp());
5195          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5196          Temp pck0 = bld.tmp(v1);
5197          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5198          tmp1 = as_vgpr(ctx, tmp1);
5199          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5200          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5201
5202          /* sample_pos = flat_load_dwordx2 addr */
5203          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5204       }
5205
5206       /* sample_pos -= 0.5 */
5207       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5208       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5209       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5210       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5211       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5212
5213       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5214       break;
5215    }
5216    case nir_intrinsic_load_barycentric_at_offset: {
5217       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5218       RegClass rc = RegClass(offset.type(), 1);
5219       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5220       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5221       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5222       break;
5223    }
5224    case nir_intrinsic_load_front_face: {
5225       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5226                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5227       break;
5228    }
5229    case nir_intrinsic_load_view_index:
5230    case nir_intrinsic_load_layer_id: {
5231       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5232          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5233          bld.copy(Definition(dst), Operand(ctx->view_index));
5234          break;
5235       }
5236
5237       unsigned idx = nir_intrinsic_base(instr);
5238       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5239                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5240       break;
5241    }
5242    case nir_intrinsic_load_frag_coord: {
5243       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5244       break;
5245    }
5246    case nir_intrinsic_load_sample_pos: {
5247       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5248       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5249       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5250                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5251                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5252       break;
5253    }
5254    case nir_intrinsic_load_interpolated_input:
5255       visit_load_interpolated_input(ctx, instr);
5256       break;
5257    case nir_intrinsic_store_output:
5258       visit_store_output(ctx, instr);
5259       break;
5260    case nir_intrinsic_load_input:
5261       visit_load_input(ctx, instr);
5262       break;
5263    case nir_intrinsic_load_ubo:
5264       visit_load_ubo(ctx, instr);
5265       break;
5266    case nir_intrinsic_load_push_constant:
5267       visit_load_push_constant(ctx, instr);
5268       break;
5269    case nir_intrinsic_load_constant:
5270       visit_load_constant(ctx, instr);
5271       break;
5272    case nir_intrinsic_vulkan_resource_index:
5273       visit_load_resource(ctx, instr);
5274       break;
5275    case nir_intrinsic_discard:
5276       visit_discard(ctx, instr);
5277       break;
5278    case nir_intrinsic_discard_if:
5279       visit_discard_if(ctx, instr);
5280       break;
5281    case nir_intrinsic_load_shared:
5282       visit_load_shared(ctx, instr);
5283       break;
5284    case nir_intrinsic_store_shared:
5285       visit_store_shared(ctx, instr);
5286       break;
5287    case nir_intrinsic_shared_atomic_add:
5288    case nir_intrinsic_shared_atomic_imin:
5289    case nir_intrinsic_shared_atomic_umin:
5290    case nir_intrinsic_shared_atomic_imax:
5291    case nir_intrinsic_shared_atomic_umax:
5292    case nir_intrinsic_shared_atomic_and:
5293    case nir_intrinsic_shared_atomic_or:
5294    case nir_intrinsic_shared_atomic_xor:
5295    case nir_intrinsic_shared_atomic_exchange:
5296    case nir_intrinsic_shared_atomic_comp_swap:
5297       visit_shared_atomic(ctx, instr);
5298       break;
5299    case nir_intrinsic_image_deref_load:
5300       visit_image_load(ctx, instr);
5301       break;
5302    case nir_intrinsic_image_deref_store:
5303       visit_image_store(ctx, instr);
5304       break;
5305    case nir_intrinsic_image_deref_atomic_add:
5306    case nir_intrinsic_image_deref_atomic_umin:
5307    case nir_intrinsic_image_deref_atomic_imin:
5308    case nir_intrinsic_image_deref_atomic_umax:
5309    case nir_intrinsic_image_deref_atomic_imax:
5310    case nir_intrinsic_image_deref_atomic_and:
5311    case nir_intrinsic_image_deref_atomic_or:
5312    case nir_intrinsic_image_deref_atomic_xor:
5313    case nir_intrinsic_image_deref_atomic_exchange:
5314    case nir_intrinsic_image_deref_atomic_comp_swap:
5315       visit_image_atomic(ctx, instr);
5316       break;
5317    case nir_intrinsic_image_deref_size:
5318       visit_image_size(ctx, instr);
5319       break;
5320    case nir_intrinsic_load_ssbo:
5321       visit_load_ssbo(ctx, instr);
5322       break;
5323    case nir_intrinsic_store_ssbo:
5324       visit_store_ssbo(ctx, instr);
5325       break;
5326    case nir_intrinsic_load_global:
5327       visit_load_global(ctx, instr);
5328       break;
5329    case nir_intrinsic_store_global:
5330       visit_store_global(ctx, instr);
5331       break;
5332    case nir_intrinsic_ssbo_atomic_add:
5333    case nir_intrinsic_ssbo_atomic_imin:
5334    case nir_intrinsic_ssbo_atomic_umin:
5335    case nir_intrinsic_ssbo_atomic_imax:
5336    case nir_intrinsic_ssbo_atomic_umax:
5337    case nir_intrinsic_ssbo_atomic_and:
5338    case nir_intrinsic_ssbo_atomic_or:
5339    case nir_intrinsic_ssbo_atomic_xor:
5340    case nir_intrinsic_ssbo_atomic_exchange:
5341    case nir_intrinsic_ssbo_atomic_comp_swap:
5342       visit_atomic_ssbo(ctx, instr);
5343       break;
5344    case nir_intrinsic_load_scratch:
5345       visit_load_scratch(ctx, instr);
5346       break;
5347    case nir_intrinsic_store_scratch:
5348       visit_store_scratch(ctx, instr);
5349       break;
5350    case nir_intrinsic_get_buffer_size:
5351       visit_get_buffer_size(ctx, instr);
5352       break;
5353    case nir_intrinsic_barrier: {
5354       unsigned* bsize = ctx->program->info->cs.block_size;
5355       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5356       if (workgroup_size > 64)
5357          bld.sopp(aco_opcode::s_barrier);
5358       break;
5359    }
5360    case nir_intrinsic_group_memory_barrier:
5361    case nir_intrinsic_memory_barrier:
5362    case nir_intrinsic_memory_barrier_atomic_counter:
5363    case nir_intrinsic_memory_barrier_buffer:
5364    case nir_intrinsic_memory_barrier_image:
5365    case nir_intrinsic_memory_barrier_shared:
5366       emit_memory_barrier(ctx, instr);
5367       break;
5368    case nir_intrinsic_load_num_work_groups:
5369    case nir_intrinsic_load_work_group_id:
5370    case nir_intrinsic_load_local_invocation_id: {
5371       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5372       Temp* ids;
5373       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5374          ids = ctx->num_workgroups;
5375       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5376          ids = ctx->workgroup_ids;
5377       else
5378          ids = ctx->local_invocation_ids;
5379       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5380                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5381                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5382                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5383       emit_split_vector(ctx, dst, 3);
5384       break;
5385    }
5386    case nir_intrinsic_load_local_invocation_index: {
5387       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5388                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5389       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5390       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5391       break;
5392    }
5393    case nir_intrinsic_load_subgroup_id: {
5394       if (ctx->stage == compute_cs) {
5395          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5396          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5397       } else {
5398          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5399       }
5400       break;
5401    }
5402    case nir_intrinsic_load_subgroup_invocation: {
5403       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5404                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5405       break;
5406    }
5407    case nir_intrinsic_load_num_subgroups: {
5408       if (ctx->stage == compute_cs)
5409          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5410       else
5411          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5412       break;
5413    }
5414    case nir_intrinsic_ballot: {
5415       Definition tmp = bld.def(s2);
5416       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5417       if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s2) {
5418          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5419       } else if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s1) {
5420          bld.sop2(aco_opcode::s_cselect_b64, tmp, Operand(exec, s2), Operand(0u), bld.scc(src));
5421       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5422          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5423       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5424          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5425       } else {
5426          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5427          nir_print_instr(&instr->instr, stderr);
5428          fprintf(stderr, "\n");
5429       }
5430       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5431       break;
5432    }
5433    case nir_intrinsic_shuffle: {
5434       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5435       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5436          emit_uniform_subgroup(ctx, instr, src);
5437       } else {
5438          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5439          assert(tid.regClass() == v1);
5440          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5441          if (src.regClass() == v1) {
5442             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5443             emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, src), dst);
5444          } else if (src.regClass() == v2) {
5445             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5446
5447             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5448             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5449             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, lo));
5450             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, hi));
5451             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5452             emit_split_vector(ctx, dst, 2);
5453          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5454             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5455             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5456             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5457             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5458          } else {
5459             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5460             nir_print_instr(&instr->instr, stderr);
5461             fprintf(stderr, "\n");
5462          }
5463       }
5464       break;
5465    }
5466    case nir_intrinsic_load_sample_id: {
5467       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5468                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5469       break;
5470    }
5471    case nir_intrinsic_load_sample_mask_in: {
5472       visit_load_sample_mask_in(ctx, instr);
5473       break;
5474    }
5475    case nir_intrinsic_read_first_invocation: {
5476       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5477       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5478       if (src.regClass() == v1) {
5479          emit_wqm(ctx,
5480                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5481                   dst);
5482       } else if (src.regClass() == v2) {
5483          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5484          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5485          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5486          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5487          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5488          emit_split_vector(ctx, dst, 2);
5489       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5490          emit_wqm(ctx,
5491                   bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5492                            bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2))),
5493                   dst);
5494       } else if (src.regClass() == s1) {
5495          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5496       } else if (src.regClass() == s2) {
5497          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5498       } else {
5499          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5500          nir_print_instr(&instr->instr, stderr);
5501          fprintf(stderr, "\n");
5502       }
5503       break;
5504    }
5505    case nir_intrinsic_read_invocation: {
5506       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5507       Temp lane = get_ssa_temp(ctx, instr->src[1].ssa);
5508       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5509       assert(lane.regClass() == s1);
5510       if (src.regClass() == v1) {
5511          emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), src, lane), dst);
5512       } else if (src.regClass() == v2) {
5513          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5514          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5515          lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), lo, lane));
5516          hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), hi, lane));
5517          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5518          emit_split_vector(ctx, dst, 2);
5519       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5520          emit_wqm(ctx, bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, lane), dst);
5521       } else if (src.regClass() == s1) {
5522          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5523       } else if (src.regClass() == s2) {
5524          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5525       } else {
5526          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5527          nir_print_instr(&instr->instr, stderr);
5528          fprintf(stderr, "\n");
5529       }
5530       break;
5531    }
5532    case nir_intrinsic_vote_all: {
5533       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5534       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5535       assert(src.regClass() == s2);
5536       assert(dst.regClass() == s1);
5537
5538       Definition tmp = bld.def(s1);
5539       bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(tmp),
5540                bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)),
5541                Operand(exec, s2));
5542       emit_wqm(ctx, tmp.getTemp(), dst);
5543       break;
5544    }
5545    case nir_intrinsic_vote_any: {
5546       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5547       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5548       assert(src.regClass() == s2);
5549       assert(dst.regClass() == s1);
5550
5551       Definition tmp = bld.def(s1);
5552       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(tmp), src, Operand(exec, s2));
5553       emit_wqm(ctx, tmp.getTemp(), dst);
5554       break;
5555    }
5556    case nir_intrinsic_reduce:
5557    case nir_intrinsic_inclusive_scan:
5558    case nir_intrinsic_exclusive_scan: {
5559       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5560       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5561       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5562       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5563          nir_intrinsic_cluster_size(instr) : 0;
5564       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5565
5566       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5567          emit_uniform_subgroup(ctx, instr, src);
5568       } else if (instr->dest.ssa.bit_size == 1) {
5569          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5570             op = nir_op_iand;
5571          else if (op == nir_op_iadd)
5572             op = nir_op_ixor;
5573          else if (op == nir_op_umax || op == nir_op_imax)
5574             op = nir_op_ior;
5575          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5576
5577          switch (instr->intrinsic) {
5578          case nir_intrinsic_reduce:
5579             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5580             break;
5581          case nir_intrinsic_exclusive_scan:
5582             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5583             break;
5584          case nir_intrinsic_inclusive_scan:
5585             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5586             break;
5587          default:
5588             assert(false);
5589          }
5590       } else if (cluster_size == 1) {
5591          bld.copy(Definition(dst), src);
5592       } else {
5593          src = as_vgpr(ctx, src);
5594
5595          ReduceOp reduce_op;
5596          switch (op) {
5597          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5598             CASE(iadd)
5599             CASE(imul)
5600             CASE(fadd)
5601             CASE(fmul)
5602             CASE(imin)
5603             CASE(umin)
5604             CASE(fmin)
5605             CASE(imax)
5606             CASE(umax)
5607             CASE(fmax)
5608             CASE(iand)
5609             CASE(ior)
5610             CASE(ixor)
5611             default:
5612                unreachable("unknown reduction op");
5613          #undef CASE
5614          }
5615
5616          aco_opcode aco_op;
5617          switch (instr->intrinsic) {
5618             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5619             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5620             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5621             default:
5622                unreachable("unknown reduce intrinsic");
5623          }
5624
5625          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5626          reduce->operands[0] = Operand(src);
5627          // filled in by aco_reduce_assign.cpp, used internally as part of the
5628          // reduce sequence
5629          assert(dst.size() == 1 || dst.size() == 2);
5630          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5631          reduce->operands[2] = Operand(v1.as_linear());
5632
5633          Temp tmp_dst = bld.tmp(dst.regClass());
5634          reduce->definitions[0] = Definition(tmp_dst);
5635          reduce->definitions[1] = bld.def(s2); // used internally
5636          reduce->definitions[2] = Definition();
5637          reduce->definitions[3] = Definition(scc, s1);
5638          reduce->definitions[4] = Definition();
5639          reduce->reduce_op = reduce_op;
5640          reduce->cluster_size = cluster_size;
5641          ctx->block->instructions.emplace_back(std::move(reduce));
5642
5643          emit_wqm(ctx, tmp_dst, dst);
5644       }
5645       break;
5646    }
5647    case nir_intrinsic_quad_broadcast: {
5648       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5649       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5650          emit_uniform_subgroup(ctx, instr, src);
5651       } else {
5652          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5653          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5654          if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5655             uint32_t half_mask = 0x11111111u << lane;
5656             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5657             Temp tmp = bld.tmp(s2);
5658             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5659                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5660                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5661             emit_wqm(ctx, tmp, dst);
5662          } else if (instr->dest.ssa.bit_size == 32) {
5663             emit_wqm(ctx,
5664                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5665                                   dpp_quad_perm(lane, lane, lane, lane)),
5666                      dst);
5667          } else if (instr->dest.ssa.bit_size == 64) {
5668             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5669             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5670             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5671             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5672             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5673             emit_split_vector(ctx, dst, 2);
5674          } else {
5675             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5676             nir_print_instr(&instr->instr, stderr);
5677             fprintf(stderr, "\n");
5678          }
5679       }
5680       break;
5681    }
5682    case nir_intrinsic_quad_swap_horizontal:
5683    case nir_intrinsic_quad_swap_vertical:
5684    case nir_intrinsic_quad_swap_diagonal:
5685    case nir_intrinsic_quad_swizzle_amd: {
5686       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5687       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5688          emit_uniform_subgroup(ctx, instr, src);
5689          break;
5690       }
5691       uint16_t dpp_ctrl = 0;
5692       switch (instr->intrinsic) {
5693       case nir_intrinsic_quad_swap_horizontal:
5694          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5695          break;
5696       case nir_intrinsic_quad_swap_vertical:
5697          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5698          break;
5699       case nir_intrinsic_quad_swap_diagonal:
5700          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5701          break;
5702       case nir_intrinsic_quad_swizzle_amd: {
5703          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5704          break;
5705       }
5706       default:
5707          break;
5708       }
5709
5710       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5711       if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5712          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5713          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5714          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5715          emit_wqm(ctx, tmp, dst);
5716       } else if (instr->dest.ssa.bit_size == 32) {
5717          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5718          emit_wqm(ctx, tmp, dst);
5719       } else if (instr->dest.ssa.bit_size == 64) {
5720          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5721          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5722          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5723          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5724          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5725          emit_split_vector(ctx, dst, 2);
5726       } else {
5727          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5728          nir_print_instr(&instr->instr, stderr);
5729          fprintf(stderr, "\n");
5730       }
5731       break;
5732    }
5733    case nir_intrinsic_masked_swizzle_amd: {
5734       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5735       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5736          emit_uniform_subgroup(ctx, instr, src);
5737          break;
5738       }
5739       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5740       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5741       if (dst.regClass() == v1) {
5742          emit_wqm(ctx,
5743                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5744                   dst);
5745       } else if (dst.regClass() == v2) {
5746          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5747          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5748          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5749          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5750          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5751          emit_split_vector(ctx, dst, 2);
5752       } else {
5753          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5754          nir_print_instr(&instr->instr, stderr);
5755          fprintf(stderr, "\n");
5756       }
5757       break;
5758    }
5759    case nir_intrinsic_write_invocation_amd: {
5760       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5761       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5762       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5763       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5764       if (dst.regClass() == v1) {
5765          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5766          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5767       } else if (dst.regClass() == v2) {
5768          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5769          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5770          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5771          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5772          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5773          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5774          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5775          emit_split_vector(ctx, dst, 2);
5776       } else {
5777          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5778          nir_print_instr(&instr->instr, stderr);
5779          fprintf(stderr, "\n");
5780       }
5781       break;
5782    }
5783    case nir_intrinsic_mbcnt_amd: {
5784       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5785       RegClass rc = RegClass(src.type(), 1);
5786       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5787       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5788       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5789       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5790       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5791       emit_wqm(ctx, wqm_tmp, dst);
5792       break;
5793    }
5794    case nir_intrinsic_load_helper_invocation: {
5795       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5796       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5797       ctx->block->kind |= block_kind_needs_lowering;
5798       ctx->program->needs_exact = true;
5799       break;
5800    }
5801    case nir_intrinsic_is_helper_invocation: {
5802       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5803       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5804       ctx->block->kind |= block_kind_needs_lowering;
5805       ctx->program->needs_exact = true;
5806       break;
5807    }
5808    case nir_intrinsic_demote:
5809       bld.pseudo(aco_opcode::p_demote_to_helper);
5810       ctx->block->kind |= block_kind_uses_demote;
5811       ctx->program->needs_exact = true;
5812       break;
5813    case nir_intrinsic_demote_if: {
5814       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc),
5815                            as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false),
5816                            Operand(exec, s2));
5817       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5818       ctx->block->kind |= block_kind_uses_demote;
5819       ctx->program->needs_exact = true;
5820       break;
5821    }
5822    case nir_intrinsic_first_invocation: {
5823       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5824                get_ssa_temp(ctx, &instr->dest.ssa));
5825       break;
5826    }
5827    case nir_intrinsic_shader_clock:
5828       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5829       break;
5830    case nir_intrinsic_load_vertex_id_zero_base: {
5831       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5832       bld.copy(Definition(dst), ctx->vertex_id);
5833       break;
5834    }
5835    case nir_intrinsic_load_first_vertex: {
5836       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5837       bld.copy(Definition(dst), ctx->base_vertex);
5838       break;
5839    }
5840    case nir_intrinsic_load_base_instance: {
5841       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5842       bld.copy(Definition(dst), ctx->start_instance);
5843       break;
5844    }
5845    case nir_intrinsic_load_instance_id: {
5846       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5847       bld.copy(Definition(dst), ctx->instance_id);
5848       break;
5849    }
5850    case nir_intrinsic_load_draw_id: {
5851       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5852       bld.copy(Definition(dst), ctx->draw_id);
5853       break;
5854    }
5855    default:
5856       fprintf(stderr, "Unimplemented intrinsic instr: ");
5857       nir_print_instr(&instr->instr, stderr);
5858       fprintf(stderr, "\n");
5859       abort();
5860
5861       break;
5862    }
5863 }
5864
5865
5866 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5867                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5868                     enum glsl_base_type *stype)
5869 {
5870    nir_deref_instr *texture_deref_instr = NULL;
5871    nir_deref_instr *sampler_deref_instr = NULL;
5872    int plane = -1;
5873
5874    for (unsigned i = 0; i < instr->num_srcs; i++) {
5875       switch (instr->src[i].src_type) {
5876       case nir_tex_src_texture_deref:
5877          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
5878          break;
5879       case nir_tex_src_sampler_deref:
5880          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
5881          break;
5882       case nir_tex_src_plane:
5883          plane = nir_src_as_int(instr->src[i].src);
5884          break;
5885       default:
5886          break;
5887       }
5888    }
5889
5890    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
5891
5892    if (!sampler_deref_instr)
5893       sampler_deref_instr = texture_deref_instr;
5894
5895    if (plane >= 0) {
5896       assert(instr->op != nir_texop_txf_ms &&
5897              instr->op != nir_texop_samples_identical);
5898       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
5899       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
5900    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
5901       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
5902    } else {
5903       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
5904    }
5905    if (samp_ptr) {
5906       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
5907       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
5908          fprintf(stderr, "Unimplemented sampler descriptor: ");
5909          nir_print_instr(&instr->instr, stderr);
5910          fprintf(stderr, "\n");
5911          abort();
5912          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
5913       }
5914    }
5915    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
5916                      instr->op == nir_texop_samples_identical))
5917       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
5918 }
5919
5920 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
5921                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
5922 {
5923    Builder bld(ctx->program, ctx->block);
5924
5925    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
5926    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
5927    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
5928
5929    Operand neg_one(0xbf800000u);
5930    Operand one(0x3f800000u);
5931    Operand two(0x40000000u);
5932    Operand four(0x40800000u);
5933
5934    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
5935    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
5936    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
5937
5938    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
5939    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
5940    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
5941    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
5942
5943    // select sc
5944    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
5945    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
5946                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
5947                        one, is_ma_y);
5948    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5949
5950    // select tc
5951    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
5952    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
5953    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5954
5955    // select ma
5956    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
5957                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
5958                   deriv_z, is_ma_z);
5959    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
5960    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
5961 }
5962
5963 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
5964 {
5965    Builder bld(ctx->program, ctx->block);
5966    Temp coord_args[4], ma, tc, sc, id;
5967    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
5968       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
5969
5970    if (is_array) {
5971       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
5972
5973       // see comment in ac_prepare_cube_coords()
5974       if (ctx->options->chip_class <= GFX8)
5975          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
5976    }
5977
5978    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
5979
5980    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
5981    vop3a->operands[0] = Operand(ma);
5982    vop3a->abs[0] = true;
5983    Temp invma = bld.tmp(v1);
5984    vop3a->definitions[0] = Definition(invma);
5985    ctx->block->instructions.emplace_back(std::move(vop3a));
5986
5987    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
5988    if (!is_deriv)
5989       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
5990
5991    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
5992    if (!is_deriv)
5993       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
5994
5995    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
5996
5997    if (is_deriv) {
5998       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
5999       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6000
6001       for (unsigned i = 0; i < 2; i++) {
6002          // see comment in ac_prepare_cube_coords()
6003          Temp deriv_ma;
6004          Temp deriv_sc, deriv_tc;
6005          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6006                            &deriv_ma, &deriv_sc, &deriv_tc);
6007
6008          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6009
6010          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6011                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6012                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6013          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6014                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6015                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6016          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6017       }
6018
6019       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6020       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6021    }
6022
6023    if (is_array)
6024       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6025    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6026
6027 }
6028
6029 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6030 {
6031    Temp coord_vec[3];
6032    for (unsigned i = 0; i < coords.size(); i++)
6033       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6034
6035    Builder bld(ctx->program, ctx->block);
6036    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6037
6038    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6039    for (unsigned i = 0; i < coords.size(); i++)
6040       vec->operands[i] = Operand(coord_vec[i]);
6041    Temp res = bld.tmp(RegType::vgpr, coords.size());
6042    vec->definitions[0] = Definition(res);
6043    ctx->block->instructions.emplace_back(std::move(vec));
6044    return res;
6045 }
6046
6047 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6048 {
6049    if (vec->parent_instr->type != nir_instr_type_alu)
6050       return;
6051    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6052    if (vec_instr->op != nir_op_vec(vec->num_components))
6053       return;
6054
6055    for (unsigned i = 0; i < vec->num_components; i++) {
6056       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6057               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6058    }
6059 }
6060
6061 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6062 {
6063    Builder bld(ctx->program, ctx->block);
6064    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6065         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6066    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6067         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6068    nir_const_value *sample_index_cv = NULL;
6069    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6070    enum glsl_base_type stype;
6071    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6072
6073    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6074                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6075    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6076                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6077
6078    for (unsigned i = 0; i < instr->num_srcs; i++) {
6079       switch (instr->src[i].src_type) {
6080       case nir_tex_src_coord:
6081          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6082          break;
6083       case nir_tex_src_bias:
6084          if (instr->op == nir_texop_txb) {
6085             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6086             has_bias = true;
6087          }
6088          break;
6089       case nir_tex_src_lod: {
6090          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6091
6092          if (val && val->f32 <= 0.0) {
6093             level_zero = true;
6094          } else {
6095             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6096             has_lod = true;
6097          }
6098          break;
6099       }
6100       case nir_tex_src_comparator:
6101          if (instr->is_shadow) {
6102             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6103             has_compare = true;
6104          }
6105          break;
6106       case nir_tex_src_offset:
6107          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6108          get_const_vec(instr->src[i].src.ssa, const_offset);
6109          has_offset = true;
6110          break;
6111       case nir_tex_src_ddx:
6112          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6113          has_ddx = true;
6114          break;
6115       case nir_tex_src_ddy:
6116          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6117          has_ddy = true;
6118          break;
6119       case nir_tex_src_ms_index:
6120          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6121          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6122          has_sample_index = true;
6123          break;
6124       case nir_tex_src_texture_offset:
6125       case nir_tex_src_sampler_offset:
6126       default:
6127          break;
6128       }
6129    }
6130 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6131    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6132       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6133
6134    if (instr->op == nir_texop_texture_samples) {
6135       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6136
6137       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6138       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6139       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6140       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6141
6142       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6143                samples, Operand(1u), bld.scc(is_msaa));
6144       return;
6145    }
6146
6147    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6148       aco_ptr<Instruction> tmp_instr;
6149       Temp acc, pack = Temp();
6150
6151       uint32_t pack_const = 0;
6152       for (unsigned i = 0; i < offset.size(); i++) {
6153          if (!const_offset[i])
6154             continue;
6155          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6156       }
6157
6158       if (offset.type() == RegType::sgpr) {
6159          for (unsigned i = 0; i < offset.size(); i++) {
6160             if (const_offset[i])
6161                continue;
6162
6163             acc = emit_extract_vector(ctx, offset, i, s1);
6164             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6165
6166             if (i) {
6167                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6168             }
6169
6170             if (pack == Temp()) {
6171                pack = acc;
6172             } else {
6173                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6174             }
6175          }
6176
6177          if (pack_const && pack != Temp())
6178             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6179       } else {
6180          for (unsigned i = 0; i < offset.size(); i++) {
6181             if (const_offset[i])
6182                continue;
6183
6184             acc = emit_extract_vector(ctx, offset, i, v1);
6185             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6186
6187             if (i) {
6188                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6189             }
6190
6191             if (pack == Temp()) {
6192                pack = acc;
6193             } else {
6194                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6195             }
6196          }
6197
6198          if (pack_const && pack != Temp())
6199             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6200       }
6201       if (pack_const && pack == Temp())
6202          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6203       else if (pack == Temp())
6204          has_offset = false;
6205       else
6206          offset = pack;
6207    }
6208
6209    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6210       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6211
6212    /* pack derivatives */
6213    if (has_ddx || has_ddy) {
6214       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6215          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6216                              ddx, Operand(0u), ddy, Operand(0u));
6217       } else {
6218          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6219       }
6220       has_derivs = true;
6221    }
6222
6223    if (instr->coord_components > 1 &&
6224        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6225        instr->is_array &&
6226        instr->op != nir_texop_txf)
6227       coords = apply_round_slice(ctx, coords, 1);
6228
6229    if (instr->coord_components > 2 &&
6230       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6231        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6232        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6233        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6234        instr->is_array &&
6235        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6236       coords = apply_round_slice(ctx, coords, 2);
6237
6238    if (ctx->options->chip_class == GFX9 &&
6239        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6240        instr->op != nir_texop_lod && instr->coord_components) {
6241       assert(coords.size() > 0 && coords.size() < 3);
6242
6243       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6244       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6245       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6246       if (coords.size() > 1)
6247          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6248       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6249       vec->definitions[0] = Definition(coords);
6250       ctx->block->instructions.emplace_back(std::move(vec));
6251    }
6252
6253    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6254
6255    if (instr->op == nir_texop_samples_identical)
6256       resource = fmask_ptr;
6257
6258    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6259              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6260             instr->op != nir_texop_txs) {
6261       assert(has_sample_index);
6262       Operand op(sample_index);
6263       if (sample_index_cv)
6264          op = Operand(sample_index_cv->u32);
6265       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6266    }
6267
6268    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6269       Temp split_coords[coords.size()];
6270       emit_split_vector(ctx, coords, coords.size());
6271       for (unsigned i = 0; i < coords.size(); i++)
6272          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6273
6274       unsigned i = 0;
6275       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6276          Temp off = emit_extract_vector(ctx, offset, i, v1);
6277          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6278       }
6279
6280       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6281       for (unsigned i = 0; i < coords.size(); i++)
6282          vec->operands[i] = Operand(split_coords[i]);
6283       coords = bld.tmp(coords.regClass());
6284       vec->definitions[0] = Definition(coords);
6285       ctx->block->instructions.emplace_back(std::move(vec));
6286
6287       has_offset = false;
6288    }
6289
6290    /* Build tex instruction */
6291    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6292    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6293                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6294                   : 0;
6295    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6296    Temp tmp_dst = dst;
6297
6298    /* gather4 selects the component by dmask and always returns vec4 */
6299    if (instr->op == nir_texop_tg4) {
6300       assert(instr->dest.ssa.num_components == 4);
6301       if (instr->is_shadow)
6302          dmask = 1;
6303       else
6304          dmask = 1 << instr->component;
6305       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6306          tmp_dst = bld.tmp(v4);
6307    } else if (instr->op == nir_texop_samples_identical) {
6308       tmp_dst = bld.tmp(v1);
6309    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6310       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6311    }
6312
6313    aco_ptr<MIMG_instruction> tex;
6314    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6315       if (!has_lod)
6316          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6317
6318       bool div_by_6 = instr->op == nir_texop_txs &&
6319                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6320                       instr->is_array &&
6321                       (dmask & (1 << 2));
6322       if (tmp_dst.id() == dst.id() && div_by_6)
6323          tmp_dst = bld.tmp(tmp_dst.regClass());
6324
6325       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6326       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6327       tex->operands[1] = Operand(resource);
6328       if (ctx->options->chip_class == GFX9 &&
6329           instr->op == nir_texop_txs &&
6330           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6331           instr->is_array) {
6332          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6333       } else if (instr->op == nir_texop_query_levels) {
6334          tex->dmask = 1 << 3;
6335       } else {
6336          tex->dmask = dmask;
6337       }
6338       tex->da = da;
6339       tex->definitions[0] = Definition(tmp_dst);
6340       tex->dim = dim;
6341       tex->can_reorder = true;
6342       ctx->block->instructions.emplace_back(std::move(tex));
6343
6344       if (div_by_6) {
6345          /* divide 3rd value by 6 by multiplying with magic number */
6346          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6347          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6348          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6349          assert(instr->dest.ssa.num_components == 3);
6350          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6351          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6352                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6353                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6354                               by_6);
6355
6356       }
6357
6358       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6359       return;
6360    }
6361
6362    Temp tg4_compare_cube_wa64 = Temp();
6363
6364    if (tg4_integer_workarounds) {
6365       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6366       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6367       tex->operands[1] = Operand(resource);
6368       tex->dim = dim;
6369       tex->dmask = 0x3;
6370       tex->da = da;
6371       Temp size = bld.tmp(v2);
6372       tex->definitions[0] = Definition(size);
6373       tex->can_reorder = true;
6374       ctx->block->instructions.emplace_back(std::move(tex));
6375       emit_split_vector(ctx, size, size.size());
6376
6377       Temp half_texel[2];
6378       for (unsigned i = 0; i < 2; i++) {
6379          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6380          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6381          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6382          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6383       }
6384
6385       Temp orig_coords[2] = {
6386          emit_extract_vector(ctx, coords, 0, v1),
6387          emit_extract_vector(ctx, coords, 1, v1)};
6388       Temp new_coords[2] = {
6389          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6390          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6391       };
6392
6393       if (tg4_integer_cube_workaround) {
6394          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6395          Temp desc[resource.size()];
6396          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6397                                                                            Format::PSEUDO, 1, resource.size())};
6398          split->operands[0] = Operand(resource);
6399          for (unsigned i = 0; i < resource.size(); i++) {
6400             desc[i] = bld.tmp(s1);
6401             split->definitions[i] = Definition(desc[i]);
6402          }
6403          ctx->block->instructions.emplace_back(std::move(split));
6404
6405          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6406          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6407                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6408
6409          Temp nfmt;
6410          if (stype == GLSL_TYPE_UINT) {
6411             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6412                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6413                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6414                             bld.scc(compare_cube_wa));
6415          } else {
6416             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6417                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6418                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6419                             bld.scc(compare_cube_wa));
6420          }
6421          tg4_compare_cube_wa64 = as_divergent_bool(ctx, compare_cube_wa, true);
6422          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6423
6424          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6425                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6426          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6427
6428          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6429                                                                          Format::PSEUDO, resource.size(), 1)};
6430          for (unsigned i = 0; i < resource.size(); i++)
6431             vec->operands[i] = Operand(desc[i]);
6432          resource = bld.tmp(resource.regClass());
6433          vec->definitions[0] = Definition(resource);
6434          ctx->block->instructions.emplace_back(std::move(vec));
6435
6436          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6437                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6438          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6439                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6440       }
6441
6442       if (coords.size() == 3) {
6443          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6444                              new_coords[0], new_coords[1],
6445                              emit_extract_vector(ctx, coords, 2, v1));
6446       } else {
6447          assert(coords.size() == 2);
6448          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6449                              new_coords[0], new_coords[1]);
6450       }
6451    }
6452
6453    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6454        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6455        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6456       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6457
6458    std::vector<Operand> args;
6459    if (has_offset)
6460       args.emplace_back(Operand(offset));
6461    if (has_bias)
6462       args.emplace_back(Operand(bias));
6463    if (has_compare)
6464       args.emplace_back(Operand(compare));
6465    if (has_derivs)
6466       args.emplace_back(Operand(derivs));
6467    args.emplace_back(Operand(coords));
6468    if (has_sample_index)
6469       args.emplace_back(Operand(sample_index));
6470    if (has_lod)
6471       args.emplace_back(lod);
6472
6473    Operand arg;
6474    if (args.size() > 1) {
6475       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6476       unsigned size = 0;
6477       for (unsigned i = 0; i < args.size(); i++) {
6478          size += args[i].size();
6479          vec->operands[i] = args[i];
6480       }
6481       RegClass rc = RegClass(RegType::vgpr, size);
6482       Temp tmp = bld.tmp(rc);
6483       vec->definitions[0] = Definition(tmp);
6484       ctx->block->instructions.emplace_back(std::move(vec));
6485       arg = Operand(tmp);
6486    } else {
6487       assert(args[0].isTemp());
6488       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6489    }
6490
6491    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6492       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6493
6494       assert(coords.size() == 1);
6495       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6496       aco_opcode op;
6497       switch (last_bit) {
6498       case 1:
6499          op = aco_opcode::buffer_load_format_x; break;
6500       case 2:
6501          op = aco_opcode::buffer_load_format_xy; break;
6502       case 3:
6503          op = aco_opcode::buffer_load_format_xyz; break;
6504       case 4:
6505          op = aco_opcode::buffer_load_format_xyzw; break;
6506       default:
6507          unreachable("Tex instruction loads more than 4 components.");
6508       }
6509
6510       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6511       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6512          tmp_dst = dst;
6513       else
6514          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6515
6516       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6517       mubuf->operands[0] = Operand(coords);
6518       mubuf->operands[1] = Operand(resource);
6519       mubuf->operands[2] = Operand((uint32_t) 0);
6520       mubuf->definitions[0] = Definition(tmp_dst);
6521       mubuf->idxen = true;
6522       mubuf->can_reorder = true;
6523       ctx->block->instructions.emplace_back(std::move(mubuf));
6524
6525       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6526       return;
6527    }
6528
6529
6530    if (instr->op == nir_texop_txf ||
6531        instr->op == nir_texop_txf_ms ||
6532        instr->op == nir_texop_samples_identical) {
6533       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6534       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6535       tex->operands[0] = Operand(arg);
6536       tex->operands[1] = Operand(resource);
6537       tex->dim = dim;
6538       tex->dmask = dmask;
6539       tex->unrm = true;
6540       tex->da = da;
6541       tex->definitions[0] = Definition(tmp_dst);
6542       tex->can_reorder = true;
6543       ctx->block->instructions.emplace_back(std::move(tex));
6544
6545       if (instr->op == nir_texop_samples_identical) {
6546          assert(dmask == 1 && dst.regClass() == v1);
6547          assert(dst.id() != tmp_dst.id());
6548
6549          Temp tmp = bld.tmp(s2);
6550          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6551          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6552
6553       } else {
6554          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6555       }
6556       return;
6557    }
6558
6559    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6560    aco_opcode opcode = aco_opcode::image_sample;
6561    if (has_offset) { /* image_sample_*_o */
6562       if (has_compare) {
6563          opcode = aco_opcode::image_sample_c_o;
6564          if (has_derivs)
6565             opcode = aco_opcode::image_sample_c_d_o;
6566          if (has_bias)
6567             opcode = aco_opcode::image_sample_c_b_o;
6568          if (level_zero)
6569             opcode = aco_opcode::image_sample_c_lz_o;
6570          if (has_lod)
6571             opcode = aco_opcode::image_sample_c_l_o;
6572       } else {
6573          opcode = aco_opcode::image_sample_o;
6574          if (has_derivs)
6575             opcode = aco_opcode::image_sample_d_o;
6576          if (has_bias)
6577             opcode = aco_opcode::image_sample_b_o;
6578          if (level_zero)
6579             opcode = aco_opcode::image_sample_lz_o;
6580          if (has_lod)
6581             opcode = aco_opcode::image_sample_l_o;
6582       }
6583    } else { /* no offset */
6584       if (has_compare) {
6585          opcode = aco_opcode::image_sample_c;
6586          if (has_derivs)
6587             opcode = aco_opcode::image_sample_c_d;
6588          if (has_bias)
6589             opcode = aco_opcode::image_sample_c_b;
6590          if (level_zero)
6591             opcode = aco_opcode::image_sample_c_lz;
6592          if (has_lod)
6593             opcode = aco_opcode::image_sample_c_l;
6594       } else {
6595          opcode = aco_opcode::image_sample;
6596          if (has_derivs)
6597             opcode = aco_opcode::image_sample_d;
6598          if (has_bias)
6599             opcode = aco_opcode::image_sample_b;
6600          if (level_zero)
6601             opcode = aco_opcode::image_sample_lz;
6602          if (has_lod)
6603             opcode = aco_opcode::image_sample_l;
6604       }
6605    }
6606
6607    if (instr->op == nir_texop_tg4) {
6608       if (has_offset) {
6609          opcode = aco_opcode::image_gather4_lz_o;
6610          if (has_compare)
6611             opcode = aco_opcode::image_gather4_c_lz_o;
6612       } else {
6613          opcode = aco_opcode::image_gather4_lz;
6614          if (has_compare)
6615             opcode = aco_opcode::image_gather4_c_lz;
6616       }
6617    } else if (instr->op == nir_texop_lod) {
6618       opcode = aco_opcode::image_get_lod;
6619    }
6620
6621    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6622    tex->operands[0] = arg;
6623    tex->operands[1] = Operand(resource);
6624    tex->operands[2] = Operand(sampler);
6625    tex->dim = dim;
6626    tex->dmask = dmask;
6627    tex->da = da;
6628    tex->definitions[0] = Definition(tmp_dst);
6629    tex->can_reorder = true;
6630    ctx->block->instructions.emplace_back(std::move(tex));
6631
6632    if (tg4_integer_cube_workaround) {
6633       assert(tmp_dst.id() != dst.id());
6634       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6635
6636       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6637       Temp val[4];
6638       for (unsigned i = 0; i < dst.size(); i++) {
6639          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6640          Temp cvt_val;
6641          if (stype == GLSL_TYPE_UINT)
6642             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6643          else
6644             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6645          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6646       }
6647       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6648       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6649                            val[0], val[1], val[2], val[3]);
6650    }
6651    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6652    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6653
6654 }
6655
6656
6657 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6658 {
6659    Temp tmp = get_ssa_temp(ctx, ssa);
6660    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6661       return Operand(tmp.regClass());
6662    else
6663       return Operand(tmp);
6664 }
6665
6666 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6667 {
6668    aco_ptr<Pseudo_instruction> phi;
6669    unsigned num_src = exec_list_length(&instr->srcs);
6670    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6671
6672    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6673
6674    std::map<unsigned, nir_ssa_def*> phi_src;
6675    bool all_undef = true;
6676    nir_foreach_phi_src(src, instr) {
6677       phi_src[src->pred->index] = src->src.ssa;
6678       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6679          all_undef = false;
6680    }
6681    if (all_undef) {
6682       Builder bld(ctx->program, ctx->block);
6683       if (dst.regClass() == s1) {
6684          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6685       } else if (dst.regClass() == v1) {
6686          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6687       } else {
6688          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6689          for (unsigned i = 0; i < dst.size(); i++)
6690             vec->operands[i] = Operand(0u);
6691          vec->definitions[0] = Definition(dst);
6692          ctx->block->instructions.emplace_back(std::move(vec));
6693       }
6694       return;
6695    }
6696
6697    /* try to scalarize vector phis */
6698    if (dst.size() > 1) {
6699       // TODO: scalarize linear phis on divergent ifs
6700       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6701       std::array<Temp, 4> new_vec;
6702       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6703          Operand src = get_phi_operand(ctx, pair.second);
6704          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6705             can_scalarize = false;
6706             break;
6707          }
6708       }
6709       if (can_scalarize) {
6710          unsigned num_components = instr->dest.ssa.num_components;
6711          assert(dst.size() % num_components == 0);
6712          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6713
6714          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6715          for (unsigned k = 0; k < num_components; k++) {
6716             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6717             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6718             for (unsigned i = 0; i < num_src; i++) {
6719                Operand src = get_phi_operand(ctx, it->second);
6720                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6721                ++it;
6722             }
6723             Temp phi_dst = {ctx->program->allocateId(), rc};
6724             phi->definitions[0] = Definition(phi_dst);
6725             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6726             new_vec[k] = phi_dst;
6727             vec->operands[k] = Operand(phi_dst);
6728          }
6729          vec->definitions[0] = Definition(dst);
6730          ctx->block->instructions.emplace_back(std::move(vec));
6731          ctx->allocated_vec.emplace(dst.id(), new_vec);
6732          return;
6733       }
6734    }
6735
6736    unsigned extra_src = 0;
6737    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6738        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6739       extra_src++;
6740    }
6741
6742    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6743
6744    /* if we have a linear phi on a divergent if, we know that one src is undef */
6745    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6746       assert(extra_src == 0);
6747       Block* block;
6748       /* we place the phi either in the invert-block or in the current block */
6749       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6750          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6751          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6752          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6753          assert(block->kind & block_kind_invert);
6754          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6755       } else {
6756          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6757          block = ctx->block;
6758          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6759       }
6760       phi->operands[1] = Operand(dst.regClass());
6761       phi->definitions[0] = Definition(dst);
6762       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6763       return;
6764    }
6765
6766    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6767    for (unsigned i = 0; i < num_src; i++) {
6768       phi->operands[i] = get_phi_operand(ctx, it->second);
6769       ++it;
6770    }
6771    for (unsigned i = 0; i < extra_src; i++)
6772       phi->operands[num_src + i] = Operand(dst.regClass());
6773    phi->definitions[0] = Definition(dst);
6774    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6775 }
6776
6777
6778 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6779 {
6780    Temp dst = get_ssa_temp(ctx, &instr->def);
6781
6782    assert(dst.type() == RegType::sgpr);
6783
6784    if (dst.size() == 1) {
6785       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6786    } else {
6787       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6788       for (unsigned i = 0; i < dst.size(); i++)
6789          vec->operands[i] = Operand(0u);
6790       vec->definitions[0] = Definition(dst);
6791       ctx->block->instructions.emplace_back(std::move(vec));
6792    }
6793 }
6794
6795 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6796 {
6797    Builder bld(ctx->program, ctx->block);
6798    Block *logical_target;
6799    append_logical_end(ctx->block);
6800    unsigned idx = ctx->block->index;
6801
6802    switch (instr->type) {
6803    case nir_jump_break:
6804       logical_target = ctx->cf_info.parent_loop.exit;
6805       add_logical_edge(idx, logical_target);
6806       ctx->block->kind |= block_kind_break;
6807
6808       if (!ctx->cf_info.parent_if.is_divergent &&
6809           !ctx->cf_info.parent_loop.has_divergent_continue) {
6810          /* uniform break - directly jump out of the loop */
6811          ctx->block->kind |= block_kind_uniform;
6812          ctx->cf_info.has_branch = true;
6813          bld.branch(aco_opcode::p_branch);
6814          add_linear_edge(idx, logical_target);
6815          return;
6816       }
6817       ctx->cf_info.parent_loop.has_divergent_branch = true;
6818       break;
6819    case nir_jump_continue:
6820       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6821       add_logical_edge(idx, logical_target);
6822       ctx->block->kind |= block_kind_continue;
6823
6824       if (ctx->cf_info.parent_if.is_divergent) {
6825          /* for potential uniform breaks after this continue,
6826             we must ensure that they are handled correctly */
6827          ctx->cf_info.parent_loop.has_divergent_continue = true;
6828          ctx->cf_info.parent_loop.has_divergent_branch = true;
6829       } else {
6830          /* uniform continue - directly jump to the loop header */
6831          ctx->block->kind |= block_kind_uniform;
6832          ctx->cf_info.has_branch = true;
6833          bld.branch(aco_opcode::p_branch);
6834          add_linear_edge(idx, logical_target);
6835          return;
6836       }
6837       break;
6838    default:
6839       fprintf(stderr, "Unknown NIR jump instr: ");
6840       nir_print_instr(&instr->instr, stderr);
6841       fprintf(stderr, "\n");
6842       abort();
6843    }
6844
6845    /* remove critical edges from linear CFG */
6846    bld.branch(aco_opcode::p_branch);
6847    Block* break_block = ctx->program->create_and_insert_block();
6848    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6849    break_block->kind |= block_kind_uniform;
6850    add_linear_edge(idx, break_block);
6851    /* the loop_header pointer might be invalidated by this point */
6852    if (instr->type == nir_jump_continue)
6853       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6854    add_linear_edge(break_block->index, logical_target);
6855    bld.reset(break_block);
6856    bld.branch(aco_opcode::p_branch);
6857
6858    Block* continue_block = ctx->program->create_and_insert_block();
6859    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6860    add_linear_edge(idx, continue_block);
6861    append_logical_start(continue_block);
6862    ctx->block = continue_block;
6863    return;
6864 }
6865
6866 void visit_block(isel_context *ctx, nir_block *block)
6867 {
6868    nir_foreach_instr(instr, block) {
6869       switch (instr->type) {
6870       case nir_instr_type_alu:
6871          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6872          break;
6873       case nir_instr_type_load_const:
6874          visit_load_const(ctx, nir_instr_as_load_const(instr));
6875          break;
6876       case nir_instr_type_intrinsic:
6877          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
6878          break;
6879       case nir_instr_type_tex:
6880          visit_tex(ctx, nir_instr_as_tex(instr));
6881          break;
6882       case nir_instr_type_phi:
6883          visit_phi(ctx, nir_instr_as_phi(instr));
6884          break;
6885       case nir_instr_type_ssa_undef:
6886          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
6887          break;
6888       case nir_instr_type_deref:
6889          break;
6890       case nir_instr_type_jump:
6891          visit_jump(ctx, nir_instr_as_jump(instr));
6892          break;
6893       default:
6894          fprintf(stderr, "Unknown NIR instr type: ");
6895          nir_print_instr(instr, stderr);
6896          fprintf(stderr, "\n");
6897          //abort();
6898       }
6899    }
6900 }
6901
6902
6903
6904 static void visit_loop(isel_context *ctx, nir_loop *loop)
6905 {
6906    append_logical_end(ctx->block);
6907    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
6908    Builder bld(ctx->program, ctx->block);
6909    bld.branch(aco_opcode::p_branch);
6910    unsigned loop_preheader_idx = ctx->block->index;
6911
6912    Block loop_exit = Block();
6913    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
6914    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
6915
6916    Block* loop_header = ctx->program->create_and_insert_block();
6917    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
6918    loop_header->kind |= block_kind_loop_header;
6919    add_edge(loop_preheader_idx, loop_header);
6920    ctx->block = loop_header;
6921
6922    /* emit loop body */
6923    unsigned loop_header_idx = loop_header->index;
6924    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
6925    append_logical_start(ctx->block);
6926    visit_cf_list(ctx, &loop->body);
6927
6928    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
6929    if (!ctx->cf_info.has_branch) {
6930       append_logical_end(ctx->block);
6931       if (ctx->cf_info.exec_potentially_empty) {
6932          /* Discards can result in code running with an empty exec mask.
6933           * This would result in divergent breaks not ever being taken. As a
6934           * workaround, break the loop when the loop mask is empty instead of
6935           * always continuing. */
6936          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
6937
6938          /* create "loop_almost_exit" to avoid critical edges */
6939          unsigned block_idx = ctx->block->index;
6940          Block *loop_almost_exit = ctx->program->create_and_insert_block();
6941          loop_almost_exit->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6942          loop_almost_exit->kind = block_kind_uniform;
6943          bld.reset(loop_almost_exit);
6944          bld.branch(aco_opcode::p_branch);
6945
6946          add_linear_edge(block_idx, loop_almost_exit);
6947          add_linear_edge(loop_almost_exit->index, &loop_exit);
6948
6949          ctx->block = &ctx->program->blocks[block_idx];
6950       } else {
6951          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
6952       }
6953       if (!ctx->cf_info.parent_loop.has_divergent_branch)
6954          add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6955       else
6956          add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6957       bld.reset(ctx->block);
6958       bld.branch(aco_opcode::p_branch);
6959    }
6960
6961    /* fixup phis in loop header from unreachable blocks */
6962    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
6963       bool linear = ctx->cf_info.has_branch;
6964       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
6965       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
6966          if ((logical && instr->opcode == aco_opcode::p_phi) ||
6967              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
6968             /* the last operand should be the one that needs to be removed */
6969             instr->operands.pop_back();
6970          } else if (!is_phi(instr)) {
6971             break;
6972          }
6973       }
6974    }
6975
6976    ctx->cf_info.has_branch = false;
6977
6978    // TODO: if the loop has not a single exit, we must add one °°
6979    /* emit loop successor block */
6980    ctx->block = ctx->program->insert_block(std::move(loop_exit));
6981    append_logical_start(ctx->block);
6982
6983    #if 0
6984    // TODO: check if it is beneficial to not branch on continues
6985    /* trim linear phis in loop header */
6986    for (auto&& instr : loop_entry->instructions) {
6987       if (instr->opcode == aco_opcode::p_linear_phi) {
6988          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
6989          new_phi->definitions[0] = instr->definitions[0];
6990          for (unsigned i = 0; i < new_phi->operands.size(); i++)
6991             new_phi->operands[i] = instr->operands[i];
6992          /* check that the remaining operands are all the same */
6993          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
6994             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
6995          instr.swap(new_phi);
6996       } else if (instr->opcode == aco_opcode::p_phi) {
6997          continue;
6998       } else {
6999          break;
7000       }
7001    }
7002    #endif
7003 }
7004
7005 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7006 {
7007    ic->cond = cond;
7008
7009    append_logical_end(ctx->block);
7010    ctx->block->kind |= block_kind_branch;
7011
7012    /* branch to linear then block */
7013    assert(cond.regClass() == s2);
7014    aco_ptr<Pseudo_branch_instruction> branch;
7015    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7016    branch->operands[0] = Operand(cond);
7017    ctx->block->instructions.push_back(std::move(branch));
7018
7019    ic->BB_if_idx = ctx->block->index;
7020    ic->BB_invert = Block();
7021    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7022    /* Invert blocks are intentionally not marked as top level because they
7023     * are not part of the logical cfg. */
7024    ic->BB_invert.kind |= block_kind_invert;
7025    ic->BB_endif = Block();
7026    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7027    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7028
7029    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7030    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7031    ctx->cf_info.parent_if.is_divergent = true;
7032    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7033
7034    /** emit logical then block */
7035    Block* BB_then_logical = ctx->program->create_and_insert_block();
7036    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7037    add_edge(ic->BB_if_idx, BB_then_logical);
7038    ctx->block = BB_then_logical;
7039    append_logical_start(BB_then_logical);
7040 }
7041
7042 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7043 {
7044    Block *BB_then_logical = ctx->block;
7045    append_logical_end(BB_then_logical);
7046     /* branch from logical then block to invert block */
7047    aco_ptr<Pseudo_branch_instruction> branch;
7048    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7049    BB_then_logical->instructions.emplace_back(std::move(branch));
7050    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7051    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7052       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7053    BB_then_logical->kind |= block_kind_uniform;
7054    assert(!ctx->cf_info.has_branch);
7055    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7056    ctx->cf_info.parent_loop.has_divergent_branch = false;
7057
7058    /** emit linear then block */
7059    Block* BB_then_linear = ctx->program->create_and_insert_block();
7060    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7061    BB_then_linear->kind |= block_kind_uniform;
7062    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7063    /* branch from linear then block to invert block */
7064    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7065    BB_then_linear->instructions.emplace_back(std::move(branch));
7066    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7067
7068    /** emit invert merge block */
7069    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7070    ic->invert_idx = ctx->block->index;
7071
7072    /* branch to linear else block (skip else) */
7073    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7074    branch->operands[0] = Operand(ic->cond);
7075    ctx->block->instructions.push_back(std::move(branch));
7076
7077    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7078    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7079
7080    /** emit logical else block */
7081    Block* BB_else_logical = ctx->program->create_and_insert_block();
7082    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7083    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7084    add_linear_edge(ic->invert_idx, BB_else_logical);
7085    ctx->block = BB_else_logical;
7086    append_logical_start(BB_else_logical);
7087 }
7088
7089 static void end_divergent_if(isel_context *ctx, if_context *ic)
7090 {
7091    Block *BB_else_logical = ctx->block;
7092    append_logical_end(BB_else_logical);
7093
7094    /* branch from logical else block to endif block */
7095    aco_ptr<Pseudo_branch_instruction> branch;
7096    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7097    BB_else_logical->instructions.emplace_back(std::move(branch));
7098    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7099    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7100       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7101    BB_else_logical->kind |= block_kind_uniform;
7102
7103    assert(!ctx->cf_info.has_branch);
7104    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7105
7106
7107    /** emit linear else block */
7108    Block* BB_else_linear = ctx->program->create_and_insert_block();
7109    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7110    BB_else_linear->kind |= block_kind_uniform;
7111    add_linear_edge(ic->invert_idx, BB_else_linear);
7112
7113    /* branch from linear else block to endif block */
7114    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7115    BB_else_linear->instructions.emplace_back(std::move(branch));
7116    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7117
7118
7119    /** emit endif merge block */
7120    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7121    append_logical_start(ctx->block);
7122
7123
7124    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7125    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7126    /* uniform control flow never has an empty exec-mask */
7127    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7128       ctx->cf_info.exec_potentially_empty = false;
7129 }
7130
7131 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7132 {
7133    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7134    Builder bld(ctx->program, ctx->block);
7135    aco_ptr<Pseudo_branch_instruction> branch;
7136
7137    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7138       /**
7139        * Uniform conditionals are represented in the following way*) :
7140        *
7141        * The linear and logical CFG:
7142        *                        BB_IF
7143        *                        /    \
7144        *       BB_THEN (logical)      BB_ELSE (logical)
7145        *                        \    /
7146        *                        BB_ENDIF
7147        *
7148        * *) Exceptions may be due to break and continue statements within loops
7149        *    If a break/continue happens within uniform control flow, it branches
7150        *    to the loop exit/entry block. Otherwise, it branches to the next
7151        *    merge block.
7152        **/
7153       append_logical_end(ctx->block);
7154       ctx->block->kind |= block_kind_uniform;
7155
7156       /* emit branch */
7157       if (cond.regClass() == s2) {
7158          // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7159          cond = as_uniform_bool(ctx, cond);
7160       }
7161       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7162       branch->operands[0] = Operand(cond);
7163       branch->operands[0].setFixed(scc);
7164       ctx->block->instructions.emplace_back(std::move(branch));
7165
7166       unsigned BB_if_idx = ctx->block->index;
7167       Block BB_endif = Block();
7168       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7169       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7170
7171       /** emit then block */
7172       Block* BB_then = ctx->program->create_and_insert_block();
7173       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7174       add_edge(BB_if_idx, BB_then);
7175       append_logical_start(BB_then);
7176       ctx->block = BB_then;
7177       visit_cf_list(ctx, &if_stmt->then_list);
7178       BB_then = ctx->block;
7179       bool then_branch = ctx->cf_info.has_branch;
7180       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7181
7182       if (!then_branch) {
7183          append_logical_end(BB_then);
7184          /* branch from then block to endif block */
7185          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7186          BB_then->instructions.emplace_back(std::move(branch));
7187          add_linear_edge(BB_then->index, &BB_endif);
7188          if (!then_branch_divergent)
7189             add_logical_edge(BB_then->index, &BB_endif);
7190          BB_then->kind |= block_kind_uniform;
7191       }
7192
7193       ctx->cf_info.has_branch = false;
7194       ctx->cf_info.parent_loop.has_divergent_branch = false;
7195
7196       /** emit else block */
7197       Block* BB_else = ctx->program->create_and_insert_block();
7198       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7199       add_edge(BB_if_idx, BB_else);
7200       append_logical_start(BB_else);
7201       ctx->block = BB_else;
7202       visit_cf_list(ctx, &if_stmt->else_list);
7203       BB_else = ctx->block;
7204
7205       if (!ctx->cf_info.has_branch) {
7206          append_logical_end(BB_else);
7207          /* branch from then block to endif block */
7208          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7209          BB_else->instructions.emplace_back(std::move(branch));
7210          add_linear_edge(BB_else->index, &BB_endif);
7211          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7212             add_logical_edge(BB_else->index, &BB_endif);
7213          BB_else->kind |= block_kind_uniform;
7214       }
7215
7216       ctx->cf_info.has_branch &= then_branch;
7217       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7218
7219       /** emit endif merge block */
7220       if (!ctx->cf_info.has_branch) {
7221          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7222          append_logical_start(ctx->block);
7223       }
7224    } else { /* non-uniform condition */
7225       /**
7226        * To maintain a logical and linear CFG without critical edges,
7227        * non-uniform conditionals are represented in the following way*) :
7228        *
7229        * The linear CFG:
7230        *                        BB_IF
7231        *                        /    \
7232        *       BB_THEN (logical)      BB_THEN (linear)
7233        *                        \    /
7234        *                        BB_INVERT (linear)
7235        *                        /    \
7236        *       BB_ELSE (logical)      BB_ELSE (linear)
7237        *                        \    /
7238        *                        BB_ENDIF
7239        *
7240        * The logical CFG:
7241        *                        BB_IF
7242        *                        /    \
7243        *       BB_THEN (logical)      BB_ELSE (logical)
7244        *                        \    /
7245        *                        BB_ENDIF
7246        *
7247        * *) Exceptions may be due to break and continue statements within loops
7248        **/
7249
7250       if_context ic;
7251
7252       begin_divergent_if_then(ctx, &ic, cond);
7253       visit_cf_list(ctx, &if_stmt->then_list);
7254
7255       begin_divergent_if_else(ctx, &ic);
7256       visit_cf_list(ctx, &if_stmt->else_list);
7257
7258       end_divergent_if(ctx, &ic);
7259    }
7260 }
7261
7262 static void visit_cf_list(isel_context *ctx,
7263                           struct exec_list *list)
7264 {
7265    foreach_list_typed(nir_cf_node, node, node, list) {
7266       switch (node->type) {
7267       case nir_cf_node_block:
7268          visit_block(ctx, nir_cf_node_as_block(node));
7269          break;
7270       case nir_cf_node_if:
7271          visit_if(ctx, nir_cf_node_as_if(node));
7272          break;
7273       case nir_cf_node_loop:
7274          visit_loop(ctx, nir_cf_node_as_loop(node));
7275          break;
7276       default:
7277          unreachable("unimplemented cf list type");
7278       }
7279    }
7280 }
7281
7282 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7283 {
7284    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7285    uint64_t mask = ctx->vs_output.mask[slot];
7286    if (!is_pos && !mask)
7287       return;
7288    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7289       return;
7290    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7291    exp->enabled_mask = mask;
7292    for (unsigned i = 0; i < 4; ++i) {
7293       if (mask & (1 << i))
7294          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7295       else
7296          exp->operands[i] = Operand(v1);
7297    }
7298    exp->valid_mask = false;
7299    exp->done = false;
7300    exp->compressed = false;
7301    if (is_pos)
7302       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7303    else
7304       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7305    ctx->block->instructions.emplace_back(std::move(exp));
7306 }
7307
7308 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7309 {
7310    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7311    exp->enabled_mask = 0;
7312    for (unsigned i = 0; i < 4; ++i)
7313       exp->operands[i] = Operand(v1);
7314    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7315       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7316       exp->enabled_mask |= 0x1;
7317    }
7318    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7319       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7320       exp->enabled_mask |= 0x4;
7321    }
7322    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7323       if (ctx->options->chip_class < GFX9) {
7324          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7325          exp->enabled_mask |= 0x8;
7326       } else {
7327          Builder bld(ctx->program, ctx->block);
7328
7329          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7330                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7331          if (exp->operands[2].isTemp())
7332             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7333
7334          exp->operands[2] = Operand(out);
7335          exp->enabled_mask |= 0x4;
7336       }
7337    }
7338    exp->valid_mask = false;
7339    exp->done = false;
7340    exp->compressed = false;
7341    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7342    ctx->block->instructions.emplace_back(std::move(exp));
7343 }
7344
7345 static void create_vs_exports(isel_context *ctx)
7346 {
7347    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7348
7349    if (outinfo->export_prim_id) {
7350       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7351       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7352    }
7353
7354    if (ctx->options->key.has_multiview_view_index) {
7355       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7356       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7357    }
7358
7359    /* the order these position exports are created is important */
7360    int next_pos = 0;
7361    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7362    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7363       export_vs_psiz_layer_viewport(ctx, &next_pos);
7364    }
7365    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7366       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7367    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7368       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7369
7370    if (ctx->options->key.vs_common_out.export_clip_dists) {
7371       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7372          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7373       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7374          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7375    }
7376
7377    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7378       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7379           i != VARYING_SLOT_PRIMITIVE_ID)
7380          continue;
7381
7382       export_vs_varying(ctx, i, false, NULL);
7383    }
7384 }
7385
7386 static void emit_stream_output(isel_context *ctx,
7387                                Temp const *so_buffers,
7388                                Temp const *so_write_offset,
7389                                const struct radv_stream_output *output)
7390 {
7391    unsigned num_comps = util_bitcount(output->component_mask);
7392    unsigned loc = output->location;
7393    unsigned buf = output->buffer;
7394    unsigned offset = output->offset;
7395
7396    assert(num_comps && num_comps <= 4);
7397    if (!num_comps || num_comps > 4)
7398       return;
7399
7400    unsigned start = ffs(output->component_mask) - 1;
7401
7402    Temp out[4];
7403    bool all_undef = true;
7404    assert(ctx->stage == vertex_vs);
7405    for (unsigned i = 0; i < num_comps; i++) {
7406       out[i] = ctx->vs_output.outputs[loc][start + i];
7407       all_undef = all_undef && !out[i].id();
7408    }
7409    if (all_undef)
7410       return;
7411
7412    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7413    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7414    for (unsigned i = 0; i < num_comps; ++i)
7415       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7416    vec->definitions[0] = Definition(write_data);
7417    ctx->block->instructions.emplace_back(std::move(vec));
7418
7419    aco_opcode opcode;
7420    switch (num_comps) {
7421    case 1:
7422       opcode = aco_opcode::buffer_store_dword;
7423       break;
7424    case 2:
7425       opcode = aco_opcode::buffer_store_dwordx2;
7426       break;
7427    case 3:
7428       opcode = aco_opcode::buffer_store_dwordx3;
7429       break;
7430    case 4:
7431       opcode = aco_opcode::buffer_store_dwordx4;
7432       break;
7433    }
7434
7435    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7436    store->operands[0] = Operand(so_write_offset[buf]);
7437    store->operands[1] = Operand(so_buffers[buf]);
7438    store->operands[2] = Operand((uint32_t) 0);
7439    store->operands[3] = Operand(write_data);
7440    if (offset > 4095) {
7441       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7442       Builder bld(ctx->program, ctx->block);
7443       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7444    } else {
7445       store->offset = offset;
7446    }
7447    store->offen = true;
7448    store->glc = true;
7449    store->dlc = false;
7450    store->slc = true;
7451    store->can_reorder = true;
7452    ctx->block->instructions.emplace_back(std::move(store));
7453 }
7454
7455 static void emit_streamout(isel_context *ctx, unsigned stream)
7456 {
7457    Builder bld(ctx->program, ctx->block);
7458
7459    Temp so_buffers[4];
7460    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7461    for (unsigned i = 0; i < 4; i++) {
7462       unsigned stride = ctx->program->info->so.strides[i];
7463       if (!stride)
7464          continue;
7465
7466       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7467    }
7468
7469    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7470                                 ctx->streamout_config, Operand(0x70010u));
7471
7472    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7473                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7474
7475    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7476
7477    if_context ic;
7478    begin_divergent_if_then(ctx, &ic, can_emit);
7479
7480    bld.reset(ctx->block);
7481
7482    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7483
7484    Temp so_write_offset[4];
7485
7486    for (unsigned i = 0; i < 4; i++) {
7487       unsigned stride = ctx->program->info->so.strides[i];
7488       if (!stride)
7489          continue;
7490
7491       if (stride == 1) {
7492          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7493                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7494          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7495
7496          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7497       } else {
7498          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7499          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7500          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7501       }
7502    }
7503
7504    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7505       struct radv_stream_output *output =
7506          &ctx->program->info->so.outputs[i];
7507       if (stream != output->stream)
7508          continue;
7509
7510       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7511    }
7512
7513    begin_divergent_if_else(ctx, &ic);
7514    end_divergent_if(ctx, &ic);
7515 }
7516
7517 } /* end namespace */
7518
7519 void handle_bc_optimize(isel_context *ctx)
7520 {
7521    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7522    Builder bld(ctx->program, ctx->block);
7523    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7524    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7525    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7526    if (uses_center && uses_centroid) {
7527       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7528
7529       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7530          for (unsigned i = 0; i < 2; i++) {
7531             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7532                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7533                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7534                                       sel);
7535             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7536          }
7537       }
7538
7539       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7540          for (unsigned i = 0; i < 2; i++) {
7541             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7542                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7543                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7544                                       sel);
7545             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7546          }
7547       }
7548    }
7549 }
7550
7551 void select_program(Program *program,
7552                     unsigned shader_count,
7553                     struct nir_shader *const *shaders,
7554                     ac_shader_config* config,
7555                     struct radv_shader_info *info,
7556                     struct radv_nir_compiler_options *options)
7557 {
7558    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7559
7560    for (unsigned i = 0; i < shader_count; i++) {
7561       nir_shader *nir = shaders[i];
7562       init_context(&ctx, nir);
7563
7564       if (!i) {
7565          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7566          append_logical_start(ctx.block);
7567       }
7568
7569       if_context ic;
7570       if (shader_count >= 2) {
7571          Builder bld(ctx.program, ctx.block);
7572          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7573          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7574                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7575          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7576
7577          begin_divergent_if_then(&ctx, &ic, cond);
7578       }
7579
7580       if (i) {
7581          Builder bld(ctx.program, ctx.block);
7582          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7583          bld.sopp(aco_opcode::s_barrier);
7584       }
7585
7586       if (ctx.stage == fragment_fs)
7587          handle_bc_optimize(&ctx);
7588
7589       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7590       visit_cf_list(&ctx, &func->body);
7591
7592       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7593          emit_streamout(&ctx, 0);
7594
7595       if (ctx.stage == vertex_vs)
7596          create_vs_exports(&ctx);
7597
7598       if (shader_count >= 2) {
7599          begin_divergent_if_else(&ctx, &ic);
7600          end_divergent_if(&ctx, &ic);
7601       }
7602
7603       ralloc_free(ctx.divergent_vals);
7604    }
7605
7606    append_logical_end(ctx.block);
7607    ctx.block->kind |= block_kind_uniform;
7608    Builder bld(ctx.program, ctx.block);
7609    if (ctx.program->wb_smem_l1_on_end)
7610       bld.smem(aco_opcode::s_dcache_wb, false);
7611    bld.sopp(aco_opcode::s_endpgm);
7612
7613    /* cleanup CFG */
7614    for (Block& BB : program->blocks) {
7615       for (unsigned idx : BB.linear_preds)
7616          program->blocks[idx].linear_succs.emplace_back(BB.index);
7617       for (unsigned idx : BB.logical_preds)
7618          program->blocks[idx].logical_succs.emplace_back(BB.index);
7619    }
7620 }
7621 }