src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 127 {
 128    Builder bld(ctx->program, ctx->block);
 129
 130    if (!dst.id())
 131       dst = bld.tmp(src.regClass());
 132
 133    if (ctx->stage != fragment_fs) {
 134       if (!dst.id())
 135          return src;
 136
 137       bld.copy(Definition(dst), src);
 138       return dst;
 139    }
 140
 141    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 142    ctx->program->needs_wqm |= program_needs_wqm;
 143    return dst;
 144 }
 145
 146 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 147 {
 148    if (index.regClass() == s1)
 149       return bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), data, index);
 150
 151    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 152
 153    /* Currently not implemented on GFX6-7 */
 154    assert(ctx->options->chip_class >= GFX8);
 155
 156    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 157       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 158    }
 159
 160    /* GFX10, wave64 mode:
 161     * The bpermute instruction is limited to half-wave operation, which means that it can't
 162     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 163     * emulate it here.
 164     */
 165    if (!ctx->has_gfx10_wave64_bpermute) {
 166       ctx->has_gfx10_wave64_bpermute = true;
 167       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 168       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 169    }
 170
 171    Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u));
 172    lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1), lane_id);
 173    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 174    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 175    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 176
 177    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 178                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 179 }
 180
 181 Temp as_vgpr(isel_context *ctx, Temp val)
 182 {
 183    if (val.type() == RegType::sgpr) {
 184       Builder bld(ctx->program, ctx->block);
 185       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 186    }
 187    assert(val.type() == RegType::vgpr);
 188    return val;
 189 }
 190
 191 //assumes a != 0xffffffff
 192 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 193 {
 194    assert(b != 0);
 195    Builder bld(ctx->program, ctx->block);
 196
 197    if (util_is_power_of_two_or_zero(b)) {
 198       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 199       return;
 200    }
 201
 202    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 203
 204    assert(info.multiplier <= 0xffffffff);
 205
 206    bool pre_shift = info.pre_shift != 0;
 207    bool increment = info.increment != 0;
 208    bool multiply = true;
 209    bool post_shift = info.post_shift != 0;
 210
 211    if (!pre_shift && !increment && !multiply && !post_shift) {
 212       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 213       return;
 214    }
 215
 216    Temp pre_shift_dst = a;
 217    if (pre_shift) {
 218       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 219       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 220    }
 221
 222    Temp increment_dst = pre_shift_dst;
 223    if (increment) {
 224       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 225       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 226    }
 227
 228    Temp multiply_dst = increment_dst;
 229    if (multiply) {
 230       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 231       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 232                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 233    }
 234
 235    if (post_shift) {
 236       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 237    }
 238 }
 239
 240 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 241 {
 242    Builder bld(ctx->program, ctx->block);
 243    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 244 }
 245
 246
 247 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 248 {
 249    /* no need to extract the whole vector */
 250    if (src.regClass() == dst_rc) {
 251       assert(idx == 0);
 252       return src;
 253    }
 254    assert(src.size() > idx);
 255    Builder bld(ctx->program, ctx->block);
 256    auto it = ctx->allocated_vec.find(src.id());
 257    /* the size check needs to be early because elements other than 0 may be garbage */
 258    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 259       if (it->second[idx].regClass() == dst_rc) {
 260          return it->second[idx];
 261       } else {
 262          assert(dst_rc.size() == it->second[idx].regClass().size());
 263          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 264          return bld.copy(bld.def(dst_rc), it->second[idx]);
 265       }
 266    }
 267
 268    if (src.size() == dst_rc.size()) {
 269       assert(idx == 0);
 270       return bld.copy(bld.def(dst_rc), src);
 271    } else {
 272       Temp dst = bld.tmp(dst_rc);
 273       emit_extract_vector(ctx, src, idx, dst);
 274       return dst;
 275    }
 276 }
 277
 278 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 279 {
 280    if (num_components == 1)
 281       return;
 282    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 283       return;
 284    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 285    split->operands[0] = Operand(vec_src);
 286    std::array<Temp,4> elems;
 287    for (unsigned i = 0; i < num_components; i++) {
 288       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 289       split->definitions[i] = Definition(elems[i]);
 290    }
 291    ctx->block->instructions.emplace_back(std::move(split));
 292    ctx->allocated_vec.emplace(vec_src.id(), elems);
 293 }
 294
 295 /* This vector expansion uses a mask to determine which elements in the new vector
 296  * come from the original vector. The other elements are undefined. */
 297 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 298 {
 299    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 300
 301    if (vec_src == dst)
 302       return;
 303
 304    Builder bld(ctx->program, ctx->block);
 305    if (num_components == 1) {
 306       if (dst.type() == RegType::sgpr)
 307          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 308       else
 309          bld.copy(Definition(dst), vec_src);
 310       return;
 311    }
 312
 313    unsigned component_size = dst.size() / num_components;
 314    std::array<Temp,4> elems;
 315
 316    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 317    vec->definitions[0] = Definition(dst);
 318    unsigned k = 0;
 319    for (unsigned i = 0; i < num_components; i++) {
 320       if (mask & (1 << i)) {
 321          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 322          if (dst.type() == RegType::sgpr)
 323             src = bld.as_uniform(src);
 324          vec->operands[i] = Operand(src);
 325       } else {
 326          vec->operands[i] = Operand(0u);
 327       }
 328       elems[i] = vec->operands[i].getTemp();
 329    }
 330    ctx->block->instructions.emplace_back(std::move(vec));
 331    ctx->allocated_vec.emplace(dst.id(), elems);
 332 }
 333
 334 Temp as_divergent_bool(isel_context *ctx, Temp val, bool vcc_hint)
 335 {
 336    if (val.regClass() == s2) {
 337       return val;
 338    } else {
 339       assert(val.regClass() == s1);
 340       Builder bld(ctx->program, ctx->block);
 341       Definition& def = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2),
 342                                  Operand((uint32_t) -1), Operand(0u), bld.scc(val)).def(0);
 343       if (vcc_hint)
 344          def.setHint(vcc);
 345       return def.getTemp();
 346    }
 347 }
 348
 349 Temp as_uniform_bool(isel_context *ctx, Temp val)
 350 {
 351    if (val.regClass() == s1) {
 352       return val;
 353    } else {
 354       assert(val.regClass() == s2);
 355       Builder bld(ctx->program, ctx->block);
 356       /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 357       Temp tmp = bld.tmp(s1);
 358       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(Definition(tmp)), val, Operand(exec, s2)).def(1).getTemp();
 359       return emit_wqm(ctx, tmp);
 360    }
 361 }
 362
 363 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 364 {
 365    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 366       return get_ssa_temp(ctx, src.src.ssa);
 367
 368    if (src.src.ssa->num_components == size) {
 369       bool identity_swizzle = true;
 370       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 371          if (src.swizzle[i] != i)
 372             identity_swizzle = false;
 373       }
 374       if (identity_swizzle)
 375          return get_ssa_temp(ctx, src.src.ssa);
 376    }
 377
 378    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 379    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 380    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 381    assert(vec.size() % elem_size == 0);
 382
 383    RegClass elem_rc = RegClass(vec.type(), elem_size);
 384    if (size == 1) {
 385       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 386    } else {
 387       assert(size <= 4);
 388       std::array<Temp,4> elems;
 389       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 390       for (unsigned i = 0; i < size; ++i) {
 391          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 392          vec_instr->operands[i] = Operand{elems[i]};
 393       }
 394       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 395       vec_instr->definitions[0] = Definition(dst);
 396       ctx->block->instructions.emplace_back(std::move(vec_instr));
 397       ctx->allocated_vec.emplace(dst.id(), elems);
 398       return dst;
 399    }
 400 }
 401
 402 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 403 {
 404    if (ptr.size() == 2)
 405       return ptr;
 406    Builder bld(ctx->program, ctx->block);
 407    if (ptr.type() == RegType::vgpr)
 408       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 409    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 410                      ptr, Operand((unsigned)ctx->options->address32_hi));
 411 }
 412
 413 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 414 {
 415    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 416    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 417    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 418    sop2->definitions[0] = Definition(dst);
 419    if (writes_scc)
 420       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 421    ctx->block->instructions.emplace_back(std::move(sop2));
 422 }
 423
 424 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 425 {
 426    Builder bld(ctx->program, ctx->block);
 427    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 428    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 429    if (src1.type() == RegType::sgpr) {
 430       if (commutative && src0.type() == RegType::vgpr) {
 431          Temp t = src0;
 432          src0 = src1;
 433          src1 = t;
 434       } else if (src0.type() == RegType::vgpr &&
 435                  op != aco_opcode::v_madmk_f32 &&
 436                  op != aco_opcode::v_madak_f32 &&
 437                  op != aco_opcode::v_madmk_f16 &&
 438                  op != aco_opcode::v_madak_f16) {
 439          /* If the instruction is not commutative, we emit a VOP3A instruction */
 440          bld.vop2_e64(op, Definition(dst), src0, src1);
 441          return;
 442       } else {
 443          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 444       }
 445    }
 446    bld.vop2(op, Definition(dst), src0, src1);
 447 }
 448
 449 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 450 {
 451    Temp src0 = get_alu_src(ctx, instr->src[0]);
 452    Temp src1 = get_alu_src(ctx, instr->src[1]);
 453    Temp src2 = get_alu_src(ctx, instr->src[2]);
 454
 455    /* ensure that the instruction has at most 1 sgpr operand
 456     * The optimizer will inline constants for us */
 457    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 458       src0 = as_vgpr(ctx, src0);
 459    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 460       src1 = as_vgpr(ctx, src1);
 461    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 462       src2 = as_vgpr(ctx, src2);
 463
 464    Builder bld(ctx->program, ctx->block);
 465    bld.vop3(op, Definition(dst), src0, src1, src2);
 466 }
 467
 468 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 469 {
 470    Builder bld(ctx->program, ctx->block);
 471    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 472 }
 473
 474 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 475 {
 476    Temp src0 = get_alu_src(ctx, instr->src[0]);
 477    Temp src1 = get_alu_src(ctx, instr->src[1]);
 478    aco_ptr<Instruction> vopc;
 479    if (src1.type() == RegType::sgpr) {
 480       if (src0.type() == RegType::vgpr) {
 481          /* to swap the operands, we might also have to change the opcode */
 482          switch (op) {
 483             case aco_opcode::v_cmp_lt_f32:
 484                op = aco_opcode::v_cmp_gt_f32;
 485                break;
 486             case aco_opcode::v_cmp_ge_f32:
 487                op = aco_opcode::v_cmp_le_f32;
 488                break;
 489             case aco_opcode::v_cmp_lt_i32:
 490                op = aco_opcode::v_cmp_gt_i32;
 491                break;
 492             case aco_opcode::v_cmp_ge_i32:
 493                op = aco_opcode::v_cmp_le_i32;
 494                break;
 495             case aco_opcode::v_cmp_lt_u32:
 496                op = aco_opcode::v_cmp_gt_u32;
 497                break;
 498             case aco_opcode::v_cmp_ge_u32:
 499                op = aco_opcode::v_cmp_le_u32;
 500                break;
 501             case aco_opcode::v_cmp_lt_f64:
 502                op = aco_opcode::v_cmp_gt_f64;
 503                break;
 504             case aco_opcode::v_cmp_ge_f64:
 505                op = aco_opcode::v_cmp_le_f64;
 506                break;
 507             case aco_opcode::v_cmp_lt_i64:
 508                op = aco_opcode::v_cmp_gt_i64;
 509                break;
 510             case aco_opcode::v_cmp_ge_i64:
 511                op = aco_opcode::v_cmp_le_i64;
 512                break;
 513             case aco_opcode::v_cmp_lt_u64:
 514                op = aco_opcode::v_cmp_gt_u64;
 515                break;
 516             case aco_opcode::v_cmp_ge_u64:
 517                op = aco_opcode::v_cmp_le_u64;
 518                break;
 519             default: /* eq and ne are commutative */
 520                break;
 521          }
 522          Temp t = src0;
 523          src0 = src1;
 524          src1 = t;
 525       } else {
 526          src1 = as_vgpr(ctx, src1);
 527       }
 528    }
 529    Builder bld(ctx->program, ctx->block);
 530    bld.vopc(op, Definition(dst), src0, src1).def(0).setHint(vcc);
 531 }
 532
 533 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 534 {
 535    if (dst.regClass() == s2) {
 536       emit_vopc_instruction(ctx, instr, op, dst);
 537       if (!ctx->divergent_vals[instr->dest.dest.ssa.index])
 538          emit_split_vector(ctx, dst, 2);
 539    } else if (dst.regClass() == s1) {
 540       Temp src0 = get_alu_src(ctx, instr->src[0]);
 541       Temp src1 = get_alu_src(ctx, instr->src[1]);
 542       assert(src0.type() == RegType::sgpr && src1.type() == RegType::sgpr);
 543
 544       Builder bld(ctx->program, ctx->block);
 545       bld.sopc(op, bld.scc(Definition(dst)), src0, src1);
 546
 547    } else {
 548       assert(false);
 549    }
 550 }
 551
 552 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 553 {
 554    Builder bld(ctx->program, ctx->block);
 555    Temp src0 = get_alu_src(ctx, instr->src[0]);
 556    Temp src1 = get_alu_src(ctx, instr->src[1]);
 557    if (dst.regClass() == s2) {
 558       bld.sop2(op64, Definition(dst), bld.def(s1, scc),
 559                as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
 560    } else {
 561       assert(dst.regClass() == s1);
 562       bld.sop2(op32, bld.def(s1), bld.scc(Definition(dst)),
 563                as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
 564    }
 565 }
 566
 567
 568 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 569 {
 570    Builder bld(ctx->program, ctx->block);
 571    Temp cond = get_alu_src(ctx, instr->src[0]);
 572    Temp then = get_alu_src(ctx, instr->src[1]);
 573    Temp els = get_alu_src(ctx, instr->src[2]);
 574
 575    if (dst.type() == RegType::vgpr) {
 576       cond = as_divergent_bool(ctx, cond, true);
 577
 578       aco_ptr<Instruction> bcsel;
 579       if (dst.size() == 1) {
 580          then = as_vgpr(ctx, then);
 581          els = as_vgpr(ctx, els);
 582
 583          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 584       } else if (dst.size() == 2) {
 585          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 586          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 587          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 588          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 589
 590          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 591          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 592
 593          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 594       } else {
 595          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 596          nir_print_instr(&instr->instr, stderr);
 597          fprintf(stderr, "\n");
 598       }
 599       return;
 600    }
 601
 602    if (instr->dest.dest.ssa.bit_size != 1) { /* uniform condition and values in sgpr */
 603       if (dst.regClass() == s1 || dst.regClass() == s2) {
 604          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 605          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 606          bld.sop2(op, Definition(dst), then, els, bld.scc(as_uniform_bool(ctx, cond)));
 607       } else {
 608          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 609          nir_print_instr(&instr->instr, stderr);
 610          fprintf(stderr, "\n");
 611       }
 612       return;
 613    }
 614
 615    /* boolean bcsel */
 616    assert(instr->dest.dest.ssa.bit_size == 1);
 617
 618    if (dst.regClass() == s1)
 619       cond = as_uniform_bool(ctx, cond);
 620
 621    if (cond.regClass() == s1) { /* uniform selection */
 622       aco_opcode op;
 623       if (dst.regClass() == s2) {
 624          op = aco_opcode::s_cselect_b64;
 625          then = as_divergent_bool(ctx, then, false);
 626          els = as_divergent_bool(ctx, els, false);
 627       } else {
 628          assert(dst.regClass() == s1);
 629          op = aco_opcode::s_cselect_b32;
 630          then = as_uniform_bool(ctx, then);
 631          els = as_uniform_bool(ctx, els);
 632       }
 633       bld.sop2(op, Definition(dst), then, els, bld.scc(cond));
 634       return;
 635    }
 636
 637    /* divergent boolean bcsel
 638     * this implements bcsel on bools: dst = s0 ? s1 : s2
 639     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 640    assert (dst.regClass() == s2);
 641    then = as_divergent_bool(ctx, then, false);
 642    els = as_divergent_bool(ctx, els, false);
 643
 644    if (cond.id() != then.id())
 645       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 646
 647    if (cond.id() == els.id())
 648       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 649    else
 650       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 651                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 652 }
 653
 654 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 655 {
 656    if (!instr->dest.dest.is_ssa) {
 657       fprintf(stderr, "nir alu dst not in ssa: ");
 658       nir_print_instr(&instr->instr, stderr);
 659       fprintf(stderr, "\n");
 660       abort();
 661    }
 662    Builder bld(ctx->program, ctx->block);
 663    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 664    switch(instr->op) {
 665    case nir_op_vec2:
 666    case nir_op_vec3:
 667    case nir_op_vec4: {
 668       std::array<Temp,4> elems;
 669       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 670       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 671          elems[i] = get_alu_src(ctx, instr->src[i]);
 672          vec->operands[i] = Operand{elems[i]};
 673       }
 674       vec->definitions[0] = Definition(dst);
 675       ctx->block->instructions.emplace_back(std::move(vec));
 676       ctx->allocated_vec.emplace(dst.id(), elems);
 677       break;
 678    }
 679    case nir_op_mov: {
 680       Temp src = get_alu_src(ctx, instr->src[0]);
 681       aco_ptr<Instruction> mov;
 682       if (dst.type() == RegType::sgpr) {
 683          if (src.type() == RegType::vgpr)
 684             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 685          else if (src.regClass() == s1)
 686             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 687          else if (src.regClass() == s2)
 688             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 689          else
 690             unreachable("wrong src register class for nir_op_imov");
 691       } else if (dst.regClass() == v1) {
 692          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 693       } else if (dst.regClass() == v2) {
 694          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 695       } else {
 696          nir_print_instr(&instr->instr, stderr);
 697          unreachable("Should have been lowered to scalar.");
 698       }
 699       break;
 700    }
 701    case nir_op_inot: {
 702       Temp src = get_alu_src(ctx, instr->src[0]);
 703       /* uniform booleans */
 704       if (instr->dest.dest.ssa.bit_size == 1 && dst.regClass() == s1) {
 705          if (src.regClass() == s1) {
 706             /* in this case, src is either 1 or 0 */
 707             bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.scc(Definition(dst)), Operand(1u), src);
 708          } else {
 709             /* src is either exec_mask or 0 */
 710             assert(src.regClass() == s2);
 711             bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(Definition(dst)), Operand(0u), src);
 712          }
 713       } else if (dst.regClass() == v1) {
 714          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 715       } else if (dst.type() == RegType::sgpr) {
 716          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 717          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 718       } else {
 719          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 720          nir_print_instr(&instr->instr, stderr);
 721          fprintf(stderr, "\n");
 722       }
 723       break;
 724    }
 725    case nir_op_ineg: {
 726       Temp src = get_alu_src(ctx, instr->src[0]);
 727       if (dst.regClass() == v1) {
 728          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 729       } else if (dst.regClass() == s1) {
 730          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 731       } else if (dst.size() == 2) {
 732          Temp src0 = bld.tmp(dst.type(), 1);
 733          Temp src1 = bld.tmp(dst.type(), 1);
 734          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 735
 736          if (dst.regClass() == s2) {
 737             Temp carry = bld.tmp(s1);
 738             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 739             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 740             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 741          } else {
 742             Temp lower = bld.tmp(v1);
 743             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 744             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 745             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 746          }
 747       } else {
 748          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 749          nir_print_instr(&instr->instr, stderr);
 750          fprintf(stderr, "\n");
 751       }
 752       break;
 753    }
 754    case nir_op_iabs: {
 755       if (dst.regClass() == s1) {
 756          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 757       } else if (dst.regClass() == v1) {
 758          Temp src = get_alu_src(ctx, instr->src[0]);
 759          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 760       } else {
 761          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 762          nir_print_instr(&instr->instr, stderr);
 763          fprintf(stderr, "\n");
 764       }
 765       break;
 766    }
 767    case nir_op_isign: {
 768       Temp src = get_alu_src(ctx, instr->src[0]);
 769       if (dst.regClass() == s1) {
 770          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 771          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 772          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 773       } else if (dst.regClass() == s2) {
 774          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 775          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 776          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 777       } else if (dst.regClass() == v1) {
 778          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 779          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 780          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 781       } else if (dst.regClass() == v2) {
 782          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 783          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 784          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 785          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 786          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 787          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 788       } else {
 789          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 790          nir_print_instr(&instr->instr, stderr);
 791          fprintf(stderr, "\n");
 792       }
 793       break;
 794    }
 795    case nir_op_imax: {
 796       if (dst.regClass() == v1) {
 797          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 798       } else if (dst.regClass() == s1) {
 799          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 800       } else {
 801          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 802          nir_print_instr(&instr->instr, stderr);
 803          fprintf(stderr, "\n");
 804       }
 805       break;
 806    }
 807    case nir_op_umax: {
 808       if (dst.regClass() == v1) {
 809          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 810       } else if (dst.regClass() == s1) {
 811          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 812       } else {
 813          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 814          nir_print_instr(&instr->instr, stderr);
 815          fprintf(stderr, "\n");
 816       }
 817       break;
 818    }
 819    case nir_op_imin: {
 820       if (dst.regClass() == v1) {
 821          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 822       } else if (dst.regClass() == s1) {
 823          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 824       } else {
 825          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 826          nir_print_instr(&instr->instr, stderr);
 827          fprintf(stderr, "\n");
 828       }
 829       break;
 830    }
 831    case nir_op_umin: {
 832       if (dst.regClass() == v1) {
 833          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 834       } else if (dst.regClass() == s1) {
 835          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 836       } else {
 837          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 838          nir_print_instr(&instr->instr, stderr);
 839          fprintf(stderr, "\n");
 840       }
 841       break;
 842    }
 843    case nir_op_ior: {
 844       if (instr->dest.dest.ssa.bit_size == 1) {
 845          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 846       } else if (dst.regClass() == v1) {
 847          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 848       } else if (dst.regClass() == s1) {
 849          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 850       } else if (dst.regClass() == s2) {
 851          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 852       } else {
 853          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 854          nir_print_instr(&instr->instr, stderr);
 855          fprintf(stderr, "\n");
 856       }
 857       break;
 858    }
 859    case nir_op_iand: {
 860       if (instr->dest.dest.ssa.bit_size == 1) {
 861          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 862       } else if (dst.regClass() == v1) {
 863          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 864       } else if (dst.regClass() == s1) {
 865          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 866       } else if (dst.regClass() == s2) {
 867          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 868       } else {
 869          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 870          nir_print_instr(&instr->instr, stderr);
 871          fprintf(stderr, "\n");
 872       }
 873       break;
 874    }
 875    case nir_op_ixor: {
 876       if (instr->dest.dest.ssa.bit_size == 1) {
 877          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 878       } else if (dst.regClass() == v1) {
 879          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 880       } else if (dst.regClass() == s1) {
 881          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 882       } else if (dst.regClass() == s2) {
 883          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 884       } else {
 885          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 886          nir_print_instr(&instr->instr, stderr);
 887          fprintf(stderr, "\n");
 888       }
 889       break;
 890    }
 891    case nir_op_ushr: {
 892       if (dst.regClass() == v1) {
 893          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 894       } else if (dst.regClass() == v2) {
 895          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 896                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 897       } else if (dst.regClass() == s2) {
 898          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 899       } else if (dst.regClass() == s1) {
 900          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 901       } else {
 902          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 903          nir_print_instr(&instr->instr, stderr);
 904          fprintf(stderr, "\n");
 905       }
 906       break;
 907    }
 908    case nir_op_ishl: {
 909       if (dst.regClass() == v1) {
 910          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 911       } else if (dst.regClass() == v2) {
 912          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 913                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 914       } else if (dst.regClass() == s1) {
 915          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 916       } else if (dst.regClass() == s2) {
 917          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 918       } else {
 919          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 920          nir_print_instr(&instr->instr, stderr);
 921          fprintf(stderr, "\n");
 922       }
 923       break;
 924    }
 925    case nir_op_ishr: {
 926       if (dst.regClass() == v1) {
 927          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 928       } else if (dst.regClass() == v2) {
 929          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 930                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 931       } else if (dst.regClass() == s1) {
 932          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 933       } else if (dst.regClass() == s2) {
 934          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 935       } else {
 936          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 937          nir_print_instr(&instr->instr, stderr);
 938          fprintf(stderr, "\n");
 939       }
 940       break;
 941    }
 942    case nir_op_find_lsb: {
 943       Temp src = get_alu_src(ctx, instr->src[0]);
 944       if (src.regClass() == s1) {
 945          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 946       } else if (src.regClass() == v1) {
 947          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 948       } else if (src.regClass() == s2) {
 949          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 950       } else {
 951          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 952          nir_print_instr(&instr->instr, stderr);
 953          fprintf(stderr, "\n");
 954       }
 955       break;
 956    }
 957    case nir_op_ufind_msb:
 958    case nir_op_ifind_msb: {
 959       Temp src = get_alu_src(ctx, instr->src[0]);
 960       if (src.regClass() == s1 || src.regClass() == s2) {
 961          aco_opcode op = src.regClass() == s2 ?
 962                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 963                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 964          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 965
 966          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 967                                         Operand(src.size() * 32u - 1u), msb_rev);
 968          Temp msb = sub.def(0).getTemp();
 969          Temp carry = sub.def(1).getTemp();
 970
 971          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 972       } else if (src.regClass() == v1) {
 973          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 974          Temp msb_rev = bld.tmp(v1);
 975          emit_vop1_instruction(ctx, instr, op, msb_rev);
 976          Temp msb = bld.tmp(v1);
 977          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 978          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 979       } else {
 980          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 981          nir_print_instr(&instr->instr, stderr);
 982          fprintf(stderr, "\n");
 983       }
 984       break;
 985    }
 986    case nir_op_bitfield_reverse: {
 987       if (dst.regClass() == s1) {
 988          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 989       } else if (dst.regClass() == v1) {
 990          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 991       } else {
 992          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 993          nir_print_instr(&instr->instr, stderr);
 994          fprintf(stderr, "\n");
 995       }
 996       break;
 997    }
 998    case nir_op_iadd: {
 999       if (dst.regClass() == s1) {
1000          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1001          break;
1002       }
1003
1004       Temp src0 = get_alu_src(ctx, instr->src[0]);
1005       Temp src1 = get_alu_src(ctx, instr->src[1]);
1006       if (dst.regClass() == v1) {
1007          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1008          break;
1009       }
1010
1011       assert(src0.size() == 2 && src1.size() == 2);
1012       Temp src00 = bld.tmp(src0.type(), 1);
1013       Temp src01 = bld.tmp(dst.type(), 1);
1014       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1015       Temp src10 = bld.tmp(src1.type(), 1);
1016       Temp src11 = bld.tmp(dst.type(), 1);
1017       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1018
1019       if (dst.regClass() == s2) {
1020          Temp carry = bld.tmp(s1);
1021          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1022          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1023          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1024       } else if (dst.regClass() == v2) {
1025          Temp dst0 = bld.tmp(v1);
1026          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1027          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1028          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1029       } else {
1030          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1031          nir_print_instr(&instr->instr, stderr);
1032          fprintf(stderr, "\n");
1033       }
1034       break;
1035    }
1036    case nir_op_uadd_sat: {
1037       Temp src0 = get_alu_src(ctx, instr->src[0]);
1038       Temp src1 = get_alu_src(ctx, instr->src[1]);
1039       if (dst.regClass() == s1) {
1040          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1041          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1042                   src0, src1);
1043          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1044       } else if (dst.regClass() == v1) {
1045          if (ctx->options->chip_class >= GFX9) {
1046             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1047             add->operands[0] = Operand(src0);
1048             add->operands[1] = Operand(src1);
1049             add->definitions[0] = Definition(dst);
1050             add->clamp = 1;
1051             ctx->block->instructions.emplace_back(std::move(add));
1052          } else {
1053             if (src1.regClass() != v1)
1054                std::swap(src0, src1);
1055             assert(src1.regClass() == v1);
1056             Temp tmp = bld.tmp(v1);
1057             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1058             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1059          }
1060       } else {
1061          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1062          nir_print_instr(&instr->instr, stderr);
1063          fprintf(stderr, "\n");
1064       }
1065       break;
1066    }
1067    case nir_op_uadd_carry: {
1068       Temp src0 = get_alu_src(ctx, instr->src[0]);
1069       Temp src1 = get_alu_src(ctx, instr->src[1]);
1070       if (dst.regClass() == s1) {
1071          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1072          break;
1073       }
1074       if (dst.regClass() == v1) {
1075          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1076          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1077          break;
1078       }
1079
1080       Temp src00 = bld.tmp(src0.type(), 1);
1081       Temp src01 = bld.tmp(dst.type(), 1);
1082       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1083       Temp src10 = bld.tmp(src1.type(), 1);
1084       Temp src11 = bld.tmp(dst.type(), 1);
1085       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1086       if (dst.regClass() == s2) {
1087          Temp carry = bld.tmp(s1);
1088          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1089          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1090          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1091       } else if (dst.regClass() == v2) {
1092          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1093          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1094          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1095          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1096       } else {
1097          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1098          nir_print_instr(&instr->instr, stderr);
1099          fprintf(stderr, "\n");
1100       }
1101       break;
1102    }
1103    case nir_op_isub: {
1104       if (dst.regClass() == s1) {
1105          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1106          break;
1107       }
1108
1109       Temp src0 = get_alu_src(ctx, instr->src[0]);
1110       Temp src1 = get_alu_src(ctx, instr->src[1]);
1111       if (dst.regClass() == v1) {
1112          bld.vsub32(Definition(dst), src0, src1);
1113          break;
1114       }
1115
1116       Temp src00 = bld.tmp(src0.type(), 1);
1117       Temp src01 = bld.tmp(dst.type(), 1);
1118       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1119       Temp src10 = bld.tmp(src1.type(), 1);
1120       Temp src11 = bld.tmp(dst.type(), 1);
1121       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1122       if (dst.regClass() == s2) {
1123          Temp carry = bld.tmp(s1);
1124          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1125          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1126          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1127       } else if (dst.regClass() == v2) {
1128          Temp lower = bld.tmp(v1);
1129          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1130          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1131          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1132       } else {
1133          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1134          nir_print_instr(&instr->instr, stderr);
1135          fprintf(stderr, "\n");
1136       }
1137       break;
1138    }
1139    case nir_op_usub_borrow: {
1140       Temp src0 = get_alu_src(ctx, instr->src[0]);
1141       Temp src1 = get_alu_src(ctx, instr->src[1]);
1142       if (dst.regClass() == s1) {
1143          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1144          break;
1145       } else if (dst.regClass() == v1) {
1146          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1147          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1148          break;
1149       }
1150
1151       Temp src00 = bld.tmp(src0.type(), 1);
1152       Temp src01 = bld.tmp(dst.type(), 1);
1153       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1154       Temp src10 = bld.tmp(src1.type(), 1);
1155       Temp src11 = bld.tmp(dst.type(), 1);
1156       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1157       if (dst.regClass() == s2) {
1158          Temp borrow = bld.tmp(s1);
1159          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1160          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1161          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1162       } else if (dst.regClass() == v2) {
1163          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1164          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1165          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1166          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1167       } else {
1168          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1169          nir_print_instr(&instr->instr, stderr);
1170          fprintf(stderr, "\n");
1171       }
1172       break;
1173    }
1174    case nir_op_imul: {
1175       if (dst.regClass() == v1) {
1176          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1177                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1178       } else if (dst.regClass() == s1) {
1179          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1180       } else {
1181          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1182          nir_print_instr(&instr->instr, stderr);
1183          fprintf(stderr, "\n");
1184       }
1185       break;
1186    }
1187    case nir_op_umul_high: {
1188       if (dst.regClass() == v1) {
1189          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1190       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1191          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1192       } else if (dst.regClass() == s1) {
1193          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1194                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1195          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1196       } else {
1197          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1198          nir_print_instr(&instr->instr, stderr);
1199          fprintf(stderr, "\n");
1200       }
1201       break;
1202    }
1203    case nir_op_imul_high: {
1204       if (dst.regClass() == v1) {
1205          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1206       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1207          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1208       } else if (dst.regClass() == s1) {
1209          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1210                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1211          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1212       } else {
1213          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1214          nir_print_instr(&instr->instr, stderr);
1215          fprintf(stderr, "\n");
1216       }
1217       break;
1218    }
1219    case nir_op_fmul: {
1220       if (dst.size() == 1) {
1221          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1222       } else if (dst.size() == 2) {
1223          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1224                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1225       } else {
1226          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1227          nir_print_instr(&instr->instr, stderr);
1228          fprintf(stderr, "\n");
1229       }
1230       break;
1231    }
1232    case nir_op_fadd: {
1233       if (dst.size() == 1) {
1234          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1235       } else if (dst.size() == 2) {
1236          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1237                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1238       } else {
1239          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1240          nir_print_instr(&instr->instr, stderr);
1241          fprintf(stderr, "\n");
1242       }
1243       break;
1244    }
1245    case nir_op_fsub: {
1246       Temp src0 = get_alu_src(ctx, instr->src[0]);
1247       Temp src1 = get_alu_src(ctx, instr->src[1]);
1248       if (dst.size() == 1) {
1249          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1250             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1251          else
1252             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1253       } else if (dst.size() == 2) {
1254          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1255                                      get_alu_src(ctx, instr->src[0]),
1256                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1257          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1258          sub->neg[1] = true;
1259       } else {
1260          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1261          nir_print_instr(&instr->instr, stderr);
1262          fprintf(stderr, "\n");
1263       }
1264       break;
1265    }
1266    case nir_op_fmax: {
1267       if (dst.size() == 1) {
1268          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1269       } else if (dst.size() == 2) {
1270          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1271                   get_alu_src(ctx, instr->src[0]),
1272                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1273       } else {
1274          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1275          nir_print_instr(&instr->instr, stderr);
1276          fprintf(stderr, "\n");
1277       }
1278       break;
1279    }
1280    case nir_op_fmin: {
1281       if (dst.size() == 1) {
1282          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1283       } else if (dst.size() == 2) {
1284          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1285                   get_alu_src(ctx, instr->src[0]),
1286                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1287       } else {
1288          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1289          nir_print_instr(&instr->instr, stderr);
1290          fprintf(stderr, "\n");
1291       }
1292       break;
1293    }
1294    case nir_op_fmax3: {
1295       if (dst.size() == 1) {
1296          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1297       } else {
1298          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1299          nir_print_instr(&instr->instr, stderr);
1300          fprintf(stderr, "\n");
1301       }
1302       break;
1303    }
1304    case nir_op_fmin3: {
1305       if (dst.size() == 1) {
1306          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1307       } else {
1308          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1309          nir_print_instr(&instr->instr, stderr);
1310          fprintf(stderr, "\n");
1311       }
1312       break;
1313    }
1314    case nir_op_fmed3: {
1315       if (dst.size() == 1) {
1316          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1317       } else {
1318          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1319          nir_print_instr(&instr->instr, stderr);
1320          fprintf(stderr, "\n");
1321       }
1322       break;
1323    }
1324    case nir_op_umax3: {
1325       if (dst.size() == 1) {
1326          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1327       } else {
1328          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1329          nir_print_instr(&instr->instr, stderr);
1330          fprintf(stderr, "\n");
1331       }
1332       break;
1333    }
1334    case nir_op_umin3: {
1335       if (dst.size() == 1) {
1336          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1337       } else {
1338          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1339          nir_print_instr(&instr->instr, stderr);
1340          fprintf(stderr, "\n");
1341       }
1342       break;
1343    }
1344    case nir_op_umed3: {
1345       if (dst.size() == 1) {
1346          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1347       } else {
1348          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1349          nir_print_instr(&instr->instr, stderr);
1350          fprintf(stderr, "\n");
1351       }
1352       break;
1353    }
1354    case nir_op_imax3: {
1355       if (dst.size() == 1) {
1356          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1357       } else {
1358          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1359          nir_print_instr(&instr->instr, stderr);
1360          fprintf(stderr, "\n");
1361       }
1362       break;
1363    }
1364    case nir_op_imin3: {
1365       if (dst.size() == 1) {
1366          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_imed3: {
1375       if (dst.size() == 1) {
1376          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1377       } else {
1378          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1379          nir_print_instr(&instr->instr, stderr);
1380          fprintf(stderr, "\n");
1381       }
1382       break;
1383    }
1384    case nir_op_cube_face_coord: {
1385       Temp in = get_alu_src(ctx, instr->src[0], 3);
1386       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1387                       emit_extract_vector(ctx, in, 1, v1),
1388                       emit_extract_vector(ctx, in, 2, v1) };
1389       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1390       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1391       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1392       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1393       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1394       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1395       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1396       break;
1397    }
1398    case nir_op_cube_face_index: {
1399       Temp in = get_alu_src(ctx, instr->src[0], 3);
1400       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1401                       emit_extract_vector(ctx, in, 1, v1),
1402                       emit_extract_vector(ctx, in, 2, v1) };
1403       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1404       break;
1405    }
1406    case nir_op_bcsel: {
1407       emit_bcsel(ctx, instr, dst);
1408       break;
1409    }
1410    case nir_op_frsq: {
1411       if (dst.size() == 1) {
1412          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1413       } else if (dst.size() == 2) {
1414          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1415       } else {
1416          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1417          nir_print_instr(&instr->instr, stderr);
1418          fprintf(stderr, "\n");
1419       }
1420       break;
1421    }
1422    case nir_op_fneg: {
1423       Temp src = get_alu_src(ctx, instr->src[0]);
1424       if (dst.size() == 1) {
1425          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1426       } else if (dst.size() == 2) {
1427          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1428          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1429          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1430          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1431       } else {
1432          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1433          nir_print_instr(&instr->instr, stderr);
1434          fprintf(stderr, "\n");
1435       }
1436       break;
1437    }
1438    case nir_op_fabs: {
1439       Temp src = get_alu_src(ctx, instr->src[0]);
1440       if (dst.size() == 1) {
1441          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1442       } else if (dst.size() == 2) {
1443          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1444          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1445          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1446          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1447       } else {
1448          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1449          nir_print_instr(&instr->instr, stderr);
1450          fprintf(stderr, "\n");
1451       }
1452       break;
1453    }
1454    case nir_op_fsat: {
1455       Temp src = get_alu_src(ctx, instr->src[0]);
1456       if (dst.size() == 1) {
1457          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1458       } else if (dst.size() == 2) {
1459          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1460          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1461          vop3->clamp = true;
1462       } else {
1463          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1464          nir_print_instr(&instr->instr, stderr);
1465          fprintf(stderr, "\n");
1466       }
1467       break;
1468    }
1469    case nir_op_flog2: {
1470       if (dst.size() == 1) {
1471          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1472       } else {
1473          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1474          nir_print_instr(&instr->instr, stderr);
1475          fprintf(stderr, "\n");
1476       }
1477       break;
1478    }
1479    case nir_op_frcp: {
1480       if (dst.size() == 1) {
1481          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1482       } else if (dst.size() == 2) {
1483          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fexp2: {
1492       if (dst.size() == 1) {
1493          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1494       } else {
1495          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1496          nir_print_instr(&instr->instr, stderr);
1497          fprintf(stderr, "\n");
1498       }
1499       break;
1500    }
1501    case nir_op_fsqrt: {
1502       if (dst.size() == 1) {
1503          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1504       } else if (dst.size() == 2) {
1505          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1506       } else {
1507          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1508          nir_print_instr(&instr->instr, stderr);
1509          fprintf(stderr, "\n");
1510       }
1511       break;
1512    }
1513    case nir_op_ffract: {
1514       if (dst.size() == 1) {
1515          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1516       } else if (dst.size() == 2) {
1517          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1518       } else {
1519          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1520          nir_print_instr(&instr->instr, stderr);
1521          fprintf(stderr, "\n");
1522       }
1523       break;
1524    }
1525    case nir_op_ffloor: {
1526       if (dst.size() == 1) {
1527          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1528       } else if (dst.size() == 2) {
1529          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1530       } else {
1531          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1532          nir_print_instr(&instr->instr, stderr);
1533          fprintf(stderr, "\n");
1534       }
1535       break;
1536    }
1537    case nir_op_fceil: {
1538       if (dst.size() == 1) {
1539          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1540       } else if (dst.size() == 2) {
1541          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1542       } else {
1543          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1544          nir_print_instr(&instr->instr, stderr);
1545          fprintf(stderr, "\n");
1546       }
1547       break;
1548    }
1549    case nir_op_ftrunc: {
1550       if (dst.size() == 1) {
1551          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1552       } else if (dst.size() == 2) {
1553          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1554       } else {
1555          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1556          nir_print_instr(&instr->instr, stderr);
1557          fprintf(stderr, "\n");
1558       }
1559       break;
1560    }
1561    case nir_op_fround_even: {
1562       if (dst.size() == 1) {
1563          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1564       } else if (dst.size() == 2) {
1565          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1566       } else {
1567          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1568          nir_print_instr(&instr->instr, stderr);
1569          fprintf(stderr, "\n");
1570       }
1571       break;
1572    }
1573    case nir_op_fsin:
1574    case nir_op_fcos: {
1575       Temp src = get_alu_src(ctx, instr->src[0]);
1576       aco_ptr<Instruction> norm;
1577       if (dst.size() == 1) {
1578          Temp tmp;
1579          Operand half_pi(0x3e22f983u);
1580          if (src.type() == RegType::sgpr)
1581             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1582          else
1583             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1584
1585          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1586          if (ctx->options->chip_class < GFX9)
1587             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1588
1589          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1590          bld.vop1(opcode, Definition(dst), tmp);
1591       } else {
1592          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1593          nir_print_instr(&instr->instr, stderr);
1594          fprintf(stderr, "\n");
1595       }
1596       break;
1597    }
1598    case nir_op_ldexp: {
1599       if (dst.size() == 1) {
1600          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1601                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1602                   get_alu_src(ctx, instr->src[1]));
1603       } else if (dst.size() == 2) {
1604          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1605                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1606                   get_alu_src(ctx, instr->src[1]));
1607       } else {
1608          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1609          nir_print_instr(&instr->instr, stderr);
1610          fprintf(stderr, "\n");
1611       }
1612       break;
1613    }
1614    case nir_op_frexp_sig: {
1615       if (dst.size() == 1) {
1616          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1617                   get_alu_src(ctx, instr->src[0]));
1618       } else if (dst.size() == 2) {
1619          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1620                   get_alu_src(ctx, instr->src[0]));
1621       } else {
1622          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1623          nir_print_instr(&instr->instr, stderr);
1624          fprintf(stderr, "\n");
1625       }
1626       break;
1627    }
1628    case nir_op_frexp_exp: {
1629       if (instr->src[0].src.ssa->bit_size == 32) {
1630          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1631                   get_alu_src(ctx, instr->src[0]));
1632       } else if (instr->src[0].src.ssa->bit_size == 64) {
1633          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1634                   get_alu_src(ctx, instr->src[0]));
1635       } else {
1636          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1637          nir_print_instr(&instr->instr, stderr);
1638          fprintf(stderr, "\n");
1639       }
1640       break;
1641    }
1642    case nir_op_fsign: {
1643       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1644       if (dst.size() == 1) {
1645          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1646          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1647          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1648          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1649       } else if (dst.size() == 2) {
1650          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1651          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1652          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, src, cond);
1653
1654          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1655          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1656          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1657
1658          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_f2f32: {
1667       if (instr->src[0].src.ssa->bit_size == 64) {
1668          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_f2f64: {
1677       if (instr->src[0].src.ssa->bit_size == 32) {
1678          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1679       } else {
1680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1681          nir_print_instr(&instr->instr, stderr);
1682          fprintf(stderr, "\n");
1683       }
1684       break;
1685    }
1686    case nir_op_i2f32: {
1687       assert(dst.size() == 1);
1688       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1689       break;
1690    }
1691    case nir_op_i2f64: {
1692       if (instr->src[0].src.ssa->bit_size == 32) {
1693          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1694       } else if (instr->src[0].src.ssa->bit_size == 64) {
1695          Temp src = get_alu_src(ctx, instr->src[0]);
1696          RegClass rc = RegClass(src.type(), 1);
1697          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1698          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1699          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1700          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1701          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1702          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1703
1704       } else {
1705          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1706          nir_print_instr(&instr->instr, stderr);
1707          fprintf(stderr, "\n");
1708       }
1709       break;
1710    }
1711    case nir_op_u2f32: {
1712       assert(dst.size() == 1);
1713       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1714       break;
1715    }
1716    case nir_op_u2f64: {
1717       if (instr->src[0].src.ssa->bit_size == 32) {
1718          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1719       } else if (instr->src[0].src.ssa->bit_size == 64) {
1720          Temp src = get_alu_src(ctx, instr->src[0]);
1721          RegClass rc = RegClass(src.type(), 1);
1722          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1723          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1724          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1725          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1726          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1727          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1728       } else {
1729          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1730          nir_print_instr(&instr->instr, stderr);
1731          fprintf(stderr, "\n");
1732       }
1733       break;
1734    }
1735    case nir_op_f2i32: {
1736       Temp src = get_alu_src(ctx, instr->src[0]);
1737       if (instr->src[0].src.ssa->bit_size == 32) {
1738          if (dst.type() == RegType::vgpr)
1739             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1740          else
1741             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1742                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1743
1744       } else if (instr->src[0].src.ssa->bit_size == 64) {
1745          if (dst.type() == RegType::vgpr)
1746             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1747          else
1748             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1749                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1750
1751       } else {
1752          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1753          nir_print_instr(&instr->instr, stderr);
1754          fprintf(stderr, "\n");
1755       }
1756       break;
1757    }
1758    case nir_op_f2u32: {
1759       Temp src = get_alu_src(ctx, instr->src[0]);
1760       if (instr->src[0].src.ssa->bit_size == 32) {
1761          if (dst.type() == RegType::vgpr)
1762             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1763          else
1764             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1765                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1766
1767       } else if (instr->src[0].src.ssa->bit_size == 64) {
1768          if (dst.type() == RegType::vgpr)
1769             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1770          else
1771             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1772                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1773
1774       } else {
1775          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1776          nir_print_instr(&instr->instr, stderr);
1777          fprintf(stderr, "\n");
1778       }
1779       break;
1780    }
1781    case nir_op_f2i64: {
1782       Temp src = get_alu_src(ctx, instr->src[0]);
1783       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1784          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1785          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1786          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1787          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1788          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1789          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1790          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1791          Temp new_exponent = bld.tmp(v1);
1792          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1793          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1794          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1795          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1796          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1797          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1798          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1799          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1800          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1801          Temp new_lower = bld.tmp(v1);
1802          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1803          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1804          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1805
1806       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1807          if (src.type() == RegType::vgpr)
1808             src = bld.as_uniform(src);
1809          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1810          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1811          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1812          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1813          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1814          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1815          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1816          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1817          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1818          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1819          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1820          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1821          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1822          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1823          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1824          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1825          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1826          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1827          Temp borrow = bld.tmp(s1);
1828          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1829          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1830          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1831
1832       } else if (instr->src[0].src.ssa->bit_size == 64) {
1833          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1834          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1835          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1836          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1837          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1838          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1839          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1840          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1841          if (dst.type() == RegType::sgpr) {
1842             lower = bld.as_uniform(lower);
1843             upper = bld.as_uniform(upper);
1844          }
1845          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1846
1847       } else {
1848          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1849          nir_print_instr(&instr->instr, stderr);
1850          fprintf(stderr, "\n");
1851       }
1852       break;
1853    }
1854    case nir_op_f2u64: {
1855       Temp src = get_alu_src(ctx, instr->src[0]);
1856       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1857          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1858          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1859          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1860          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1861          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1862          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1863          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1864          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1865          Temp new_exponent = bld.tmp(v1);
1866          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1867          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1868          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1869          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1870          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1871          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1872          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1873          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1874          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1875
1876       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1877          if (src.type() == RegType::vgpr)
1878             src = bld.as_uniform(src);
1879          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1880          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1881          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1882          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1883          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1884          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1885          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1886          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1887          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1888          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1889          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1890          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1891          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1892          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1893          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1894          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1895          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1896          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1897
1898       } else if (instr->src[0].src.ssa->bit_size == 64) {
1899          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1900          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1901          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1902          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1903          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1904          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1905          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1906          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1907          if (dst.type() == RegType::sgpr) {
1908             lower = bld.as_uniform(lower);
1909             upper = bld.as_uniform(upper);
1910          }
1911          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1912
1913       } else {
1914          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1915          nir_print_instr(&instr->instr, stderr);
1916          fprintf(stderr, "\n");
1917       }
1918       break;
1919    }
1920    case nir_op_b2f32: {
1921       Temp src = get_alu_src(ctx, instr->src[0]);
1922       if (dst.regClass() == s1) {
1923          src = as_uniform_bool(ctx, src);
1924          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1925       } else if (dst.regClass() == v1) {
1926          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u),
1927                       as_divergent_bool(ctx, src, true));
1928       } else {
1929          unreachable("Wrong destination register class for nir_op_b2f32.");
1930       }
1931       break;
1932    }
1933    case nir_op_b2f64: {
1934       Temp src = get_alu_src(ctx, instr->src[0]);
1935       if (dst.regClass() == s2) {
1936          src = as_uniform_bool(ctx, src);
1937          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1938       } else if (dst.regClass() == v2) {
1939          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1940          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one,
1941                       as_divergent_bool(ctx, src, true));
1942          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1943       } else {
1944          unreachable("Wrong destination register class for nir_op_b2f64.");
1945       }
1946       break;
1947    }
1948    case nir_op_i2i32: {
1949       Temp src = get_alu_src(ctx, instr->src[0]);
1950       if (instr->src[0].src.ssa->bit_size == 64) {
1951          /* we can actually just say dst = src, as it would map the lower register */
1952          emit_extract_vector(ctx, src, 0, dst);
1953       } else {
1954          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1955          nir_print_instr(&instr->instr, stderr);
1956          fprintf(stderr, "\n");
1957       }
1958       break;
1959    }
1960    case nir_op_u2u32: {
1961       Temp src = get_alu_src(ctx, instr->src[0]);
1962       if (instr->src[0].src.ssa->bit_size == 16) {
1963          if (dst.regClass() == s1) {
1964             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1965          } else {
1966             // TODO: do better with SDWA
1967             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1968          }
1969       } else if (instr->src[0].src.ssa->bit_size == 64) {
1970          /* we can actually just say dst = src, as it would map the lower register */
1971          emit_extract_vector(ctx, src, 0, dst);
1972       } else {
1973          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1974          nir_print_instr(&instr->instr, stderr);
1975          fprintf(stderr, "\n");
1976       }
1977       break;
1978    }
1979    case nir_op_i2i64: {
1980       Temp src = get_alu_src(ctx, instr->src[0]);
1981       if (instr->src[0].src.ssa->bit_size == 32) {
1982          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1983       } else {
1984          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1985          nir_print_instr(&instr->instr, stderr);
1986          fprintf(stderr, "\n");
1987       }
1988       break;
1989    }
1990    case nir_op_u2u64: {
1991       Temp src = get_alu_src(ctx, instr->src[0]);
1992       if (instr->src[0].src.ssa->bit_size == 32) {
1993          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1994       } else {
1995          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1996          nir_print_instr(&instr->instr, stderr);
1997          fprintf(stderr, "\n");
1998       }
1999       break;
2000    }
2001    case nir_op_b2i32: {
2002       Temp src = get_alu_src(ctx, instr->src[0]);
2003       if (dst.regClass() == s1) {
2004          if (src.regClass() == s1) {
2005             bld.copy(Definition(dst), src);
2006          } else {
2007             // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2008             assert(src.regClass() == s2);
2009             bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(dst)), Operand(0u), src);
2010          }
2011       } else {
2012          assert(dst.regClass() == v1 && src.regClass() == s2);
2013          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2014       }
2015       break;
2016    }
2017    case nir_op_i2b1: {
2018       Temp src = get_alu_src(ctx, instr->src[0]);
2019       if (dst.regClass() == s2) {
2020          assert(src.regClass() == v1 || src.regClass() == v2);
2021          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2022                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2023       } else {
2024          assert(src.regClass() == s1 && dst.regClass() == s1);
2025          bld.sopc(aco_opcode::s_cmp_lg_u32, bld.scc(Definition(dst)), Operand(0u), src);
2026       }
2027       break;
2028    }
2029    case nir_op_pack_64_2x32_split: {
2030       Temp src0 = get_alu_src(ctx, instr->src[0]);
2031       Temp src1 = get_alu_src(ctx, instr->src[1]);
2032
2033       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2034       break;
2035    }
2036    case nir_op_unpack_64_2x32_split_x:
2037       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2038       break;
2039    case nir_op_unpack_64_2x32_split_y:
2040       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2041       break;
2042    case nir_op_pack_half_2x16: {
2043       Temp src = get_alu_src(ctx, instr->src[0], 2);
2044
2045       if (dst.regClass() == v1) {
2046          Temp src0 = bld.tmp(v1);
2047          Temp src1 = bld.tmp(v1);
2048          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2049          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2050
2051       } else {
2052          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2053          nir_print_instr(&instr->instr, stderr);
2054          fprintf(stderr, "\n");
2055       }
2056       break;
2057    }
2058    case nir_op_unpack_half_2x16_split_x: {
2059       if (dst.regClass() == v1) {
2060          Builder bld(ctx->program, ctx->block);
2061          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2062       } else {
2063          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2064          nir_print_instr(&instr->instr, stderr);
2065          fprintf(stderr, "\n");
2066       }
2067       break;
2068    }
2069    case nir_op_unpack_half_2x16_split_y: {
2070       if (dst.regClass() == v1) {
2071          Builder bld(ctx->program, ctx->block);
2072          /* TODO: use SDWA here */
2073          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2074                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2075       } else {
2076          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2077          nir_print_instr(&instr->instr, stderr);
2078          fprintf(stderr, "\n");
2079       }
2080       break;
2081    }
2082    case nir_op_fquantize2f16: {
2083       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2084
2085       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2086
2087       Temp cmp_res = bld.tmp(s2);
2088       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2089
2090       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2091
2092       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2093       break;
2094    }
2095    case nir_op_bfm: {
2096       Temp bits = get_alu_src(ctx, instr->src[0]);
2097       Temp offset = get_alu_src(ctx, instr->src[1]);
2098
2099       if (dst.regClass() == s1) {
2100          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2101       } else if (dst.regClass() == v1) {
2102          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2103       } else {
2104          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2105          nir_print_instr(&instr->instr, stderr);
2106          fprintf(stderr, "\n");
2107       }
2108       break;
2109    }
2110    case nir_op_bitfield_select: {
2111       /* (mask & insert) | (~mask & base) */
2112       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2113       Temp insert = get_alu_src(ctx, instr->src[1]);
2114       Temp base = get_alu_src(ctx, instr->src[2]);
2115
2116       /* dst = (insert & bitmask) | (base & ~bitmask) */
2117       if (dst.regClass() == s1) {
2118          aco_ptr<Instruction> sop2;
2119          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2120          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2121          Operand lhs;
2122          if (const_insert && const_bitmask) {
2123             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2124          } else {
2125             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2126             lhs = Operand(insert);
2127          }
2128
2129          Operand rhs;
2130          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2131          if (const_base && const_bitmask) {
2132             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2133          } else {
2134             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2135             rhs = Operand(base);
2136          }
2137
2138          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2139
2140       } else if (dst.regClass() == v1) {
2141          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2142             base = as_vgpr(ctx, base);
2143          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2144             insert = as_vgpr(ctx, insert);
2145
2146          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2147
2148       } else {
2149          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2150          nir_print_instr(&instr->instr, stderr);
2151          fprintf(stderr, "\n");
2152       }
2153       break;
2154    }
2155    case nir_op_ubfe:
2156    case nir_op_ibfe: {
2157       Temp base = get_alu_src(ctx, instr->src[0]);
2158       Temp offset = get_alu_src(ctx, instr->src[1]);
2159       Temp bits = get_alu_src(ctx, instr->src[2]);
2160
2161       if (dst.type() == RegType::sgpr) {
2162          Operand extract;
2163          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2164          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2165          if (const_offset && const_bits) {
2166             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2167             extract = Operand(const_extract);
2168          } else {
2169             Operand width;
2170             if (const_bits) {
2171                width = Operand(const_bits->u32 << 16);
2172             } else {
2173                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2174             }
2175             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2176          }
2177
2178          aco_opcode opcode;
2179          if (dst.regClass() == s1) {
2180             if (instr->op == nir_op_ubfe)
2181                opcode = aco_opcode::s_bfe_u32;
2182             else
2183                opcode = aco_opcode::s_bfe_i32;
2184          } else if (dst.regClass() == s2) {
2185             if (instr->op == nir_op_ubfe)
2186                opcode = aco_opcode::s_bfe_u64;
2187             else
2188                opcode = aco_opcode::s_bfe_i64;
2189          } else {
2190             unreachable("Unsupported BFE bit size");
2191          }
2192
2193          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2194
2195       } else {
2196          aco_opcode opcode;
2197          if (dst.regClass() == v1) {
2198             if (instr->op == nir_op_ubfe)
2199                opcode = aco_opcode::v_bfe_u32;
2200             else
2201                opcode = aco_opcode::v_bfe_i32;
2202          } else {
2203             unreachable("Unsupported BFE bit size");
2204          }
2205
2206          emit_vop3a_instruction(ctx, instr, opcode, dst);
2207       }
2208       break;
2209    }
2210    case nir_op_bit_count: {
2211       Temp src = get_alu_src(ctx, instr->src[0]);
2212       if (src.regClass() == s1) {
2213          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2214       } else if (src.regClass() == v1) {
2215          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2216       } else if (src.regClass() == v2) {
2217          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2218                   emit_extract_vector(ctx, src, 1, v1),
2219                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2220                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2221       } else if (src.regClass() == s2) {
2222          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2223       } else {
2224          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2225          nir_print_instr(&instr->instr, stderr);
2226          fprintf(stderr, "\n");
2227       }
2228       break;
2229    }
2230    case nir_op_flt: {
2231       if (instr->src[0].src.ssa->bit_size == 32)
2232          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f32, dst);
2233       else if (instr->src[0].src.ssa->bit_size == 64)
2234          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f64, dst);
2235       break;
2236    }
2237    case nir_op_fge: {
2238       if (instr->src[0].src.ssa->bit_size == 32)
2239          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f32, dst);
2240       else if (instr->src[0].src.ssa->bit_size == 64)
2241          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f64, dst);
2242       break;
2243    }
2244    case nir_op_feq: {
2245       if (instr->src[0].src.ssa->bit_size == 32)
2246          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f32, dst);
2247       else if (instr->src[0].src.ssa->bit_size == 64)
2248          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f64, dst);
2249       break;
2250    }
2251    case nir_op_fne: {
2252       if (instr->src[0].src.ssa->bit_size == 32)
2253          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f32, dst);
2254       else if (instr->src[0].src.ssa->bit_size == 64)
2255          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f64, dst);
2256       break;
2257    }
2258    case nir_op_ilt: {
2259       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2260          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i32, dst);
2261       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2262          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_i32, dst);
2263       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2264          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i64, dst);
2265       break;
2266    }
2267    case nir_op_ige: {
2268       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2269          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i32, dst);
2270       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2271          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_i32, dst);
2272       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2273          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i64, dst);
2274       break;
2275    }
2276    case nir_op_ieq: {
2277       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2278          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i32, dst);
2279       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2280          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_i32, dst);
2281       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2282          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i64, dst);
2283       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2284          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_u64, dst);
2285       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2286          Temp src0 = get_alu_src(ctx, instr->src[0]);
2287          Temp src1 = get_alu_src(ctx, instr->src[1]);
2288          bld.sopc(aco_opcode::s_cmp_eq_i32, bld.scc(Definition(dst)),
2289                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2290       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2291          Temp src0 = get_alu_src(ctx, instr->src[0]);
2292          Temp src1 = get_alu_src(ctx, instr->src[1]);
2293          bld.sop2(aco_opcode::s_xnor_b64, Definition(dst), bld.def(s1, scc),
2294                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2295       } else {
2296          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2297          nir_print_instr(&instr->instr, stderr);
2298          fprintf(stderr, "\n");
2299       }
2300       break;
2301    }
2302    case nir_op_ine: {
2303       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2304          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i32, dst);
2305       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2306          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i64, dst);
2307       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2308          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_i32, dst);
2309       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2310          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_u64, dst);
2311       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2312          Temp src0 = get_alu_src(ctx, instr->src[0]);
2313          Temp src1 = get_alu_src(ctx, instr->src[1]);
2314          bld.sopc(aco_opcode::s_cmp_lg_i32, bld.scc(Definition(dst)),
2315                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2316       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2317          Temp src0 = get_alu_src(ctx, instr->src[0]);
2318          Temp src1 = get_alu_src(ctx, instr->src[1]);
2319          bld.sop2(aco_opcode::s_xor_b64, Definition(dst), bld.def(s1, scc),
2320                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2321       } else {
2322          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2323          nir_print_instr(&instr->instr, stderr);
2324          fprintf(stderr, "\n");
2325       }
2326       break;
2327    }
2328    case nir_op_ult: {
2329       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2330          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u32, dst);
2331       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2332          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_u32, dst);
2333       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2334          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u64, dst);
2335       break;
2336    }
2337    case nir_op_uge: {
2338       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2339          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u32, dst);
2340       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2341          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_u32, dst);
2342       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2343          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u64, dst);
2344       break;
2345    }
2346    case nir_op_fddx:
2347    case nir_op_fddy:
2348    case nir_op_fddx_fine:
2349    case nir_op_fddy_fine:
2350    case nir_op_fddx_coarse:
2351    case nir_op_fddy_coarse: {
2352       Definition tl = bld.def(v1);
2353       uint16_t dpp_ctrl;
2354       if (instr->op == nir_op_fddx_fine) {
2355          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2356          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2357       } else if (instr->op == nir_op_fddy_fine) {
2358          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2359          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2360       } else {
2361          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2362          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2363             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2364          else
2365             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2366       }
2367
2368       Definition tmp = bld.def(v1);
2369       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2370       emit_wqm(ctx, tmp.getTemp(), dst, true);
2371       break;
2372    }
2373    default:
2374       fprintf(stderr, "Unknown NIR ALU instr: ");
2375       nir_print_instr(&instr->instr, stderr);
2376       fprintf(stderr, "\n");
2377    }
2378 }
2379
2380 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2381 {
2382    Temp dst = get_ssa_temp(ctx, &instr->def);
2383
2384    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2385    // which get truncated the lsb if double and msb if int
2386    // for now, we only use s_mov_b64 with 64bit inline constants
2387    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2388    assert(dst.type() == RegType::sgpr);
2389
2390    if (dst.size() == 1)
2391    {
2392       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(instr->value[0].u32));
2393    } else {
2394       assert(dst.size() != 1);
2395       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2396       if (instr->def.bit_size == 64)
2397          for (unsigned i = 0; i < dst.size(); i++)
2398             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2399       else {
2400          for (unsigned i = 0; i < dst.size(); i++)
2401             vec->operands[i] = Operand{instr->value[i].u32};
2402       }
2403       vec->definitions[0] = Definition(dst);
2404       ctx->block->instructions.emplace_back(std::move(vec));
2405    }
2406 }
2407
2408 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2409 {
2410    uint32_t new_mask = 0;
2411    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2412       if (mask & (1u << i))
2413          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2414    return new_mask;
2415 }
2416
2417 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2418 {
2419    /* This wouldn't work inside control flow or with indirect offsets but
2420     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2421
2422    unsigned write_mask = nir_intrinsic_write_mask(instr);
2423    unsigned component = nir_intrinsic_component(instr);
2424    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2425    unsigned idx = nir_intrinsic_base(instr) + component;
2426
2427    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2428    if (off_instr->type != nir_instr_type_load_const) {
2429       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2430       nir_print_instr(off_instr, stderr);
2431       fprintf(stderr, "\n");
2432    }
2433    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2434
2435    if (instr->src[0].ssa->bit_size == 64)
2436       write_mask = widen_mask(write_mask, 2);
2437
2438    for (unsigned i = 0; i < 8; ++i) {
2439       if (write_mask & (1 << i)) {
2440          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2441          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2442       }
2443       idx++;
2444    }
2445 }
2446
2447 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2448 {
2449    unsigned write_mask = nir_intrinsic_write_mask(instr);
2450    Operand values[4];
2451    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2452    for (unsigned i = 0; i < 4; ++i) {
2453       if (write_mask & (1 << i)) {
2454          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2455          values[i] = Operand(tmp);
2456       } else {
2457          values[i] = Operand(v1);
2458       }
2459    }
2460
2461    unsigned index = nir_intrinsic_base(instr) / 4;
2462    unsigned target, col_format;
2463    unsigned enabled_channels = 0xF;
2464    aco_opcode compr_op = (aco_opcode)0;
2465
2466    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2467    assert(offset && "Non-const offsets on exports not yet supported");
2468    index += offset->u32;
2469
2470    assert(index != FRAG_RESULT_COLOR);
2471
2472    /* Unlike vertex shader exports, it's fine to use multiple exports to
2473     * export separate channels of one target. So shaders which export both
2474     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2475     * TODO: combine the exports in those cases and create better code
2476     */
2477
2478    if (index == FRAG_RESULT_SAMPLE_MASK) {
2479
2480       if (ctx->program->info->ps.writes_z) {
2481          target = V_008DFC_SQ_EXP_MRTZ;
2482          enabled_channels = 0x4;
2483          col_format = (unsigned) -1;
2484
2485          values[2] = values[0];
2486          values[0] = Operand(v1);
2487       } else {
2488          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2489          exp->valid_mask = false;
2490          exp->done = false;
2491          exp->compressed = true;
2492          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2493          exp->enabled_mask = 0xc;
2494          for (int i = 0; i < 4; i++)
2495             exp->operands[i] = Operand(v1);
2496          exp->operands[1] = Operand(values[0]);
2497          ctx->block->instructions.emplace_back(std::move(exp));
2498          return;
2499       }
2500
2501    } else if (index == FRAG_RESULT_DEPTH) {
2502
2503       target = V_008DFC_SQ_EXP_MRTZ;
2504       enabled_channels = 0x1;
2505       col_format = (unsigned) -1;
2506
2507    } else if (index == FRAG_RESULT_STENCIL) {
2508
2509       if (ctx->program->info->ps.writes_z) {
2510          target = V_008DFC_SQ_EXP_MRTZ;
2511          enabled_channels = 0x2;
2512          col_format = (unsigned) -1;
2513
2514          values[1] = values[0];
2515          values[0] = Operand(v1);
2516       } else {
2517          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2518          shift->operands[0] = Operand((uint32_t) 16);
2519          shift->operands[1] = values[0];
2520          Temp tmp = {ctx->program->allocateId(), v1};
2521          shift->definitions[0] = Definition(tmp);
2522          ctx->block->instructions.emplace_back(std::move(shift));
2523
2524          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2525          exp->valid_mask = false;
2526          exp->done = false;
2527          exp->compressed = true;
2528          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2529          exp->enabled_mask = 0x3;
2530          exp->operands[0] = Operand(tmp);
2531          for (int i = 1; i < 4; i++)
2532             exp->operands[i] = Operand(v1);
2533          ctx->block->instructions.emplace_back(std::move(exp));
2534          return;
2535       }
2536
2537    } else {
2538       index -= FRAG_RESULT_DATA0;
2539       target = V_008DFC_SQ_EXP_MRT + index;
2540       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2541    }
2542    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2543    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2544    assert(!is_int8 && !is_int10);
2545
2546    switch (col_format)
2547    {
2548    case V_028714_SPI_SHADER_ZERO:
2549       enabled_channels = 0; /* writemask */
2550       target = V_008DFC_SQ_EXP_NULL;
2551       break;
2552
2553    case V_028714_SPI_SHADER_32_R:
2554       enabled_channels = 1;
2555       break;
2556
2557    case V_028714_SPI_SHADER_32_GR:
2558       enabled_channels = 0x3;
2559       break;
2560
2561    case V_028714_SPI_SHADER_32_AR:
2562       if (ctx->options->chip_class >= GFX10) {
2563          /* Special case: on GFX10, the outputs are different for 32_AR */
2564          enabled_channels = 0x3;
2565          values[1] = values[3];
2566       } else {
2567          enabled_channels = 0x9;
2568       }
2569       break;
2570
2571    case V_028714_SPI_SHADER_FP16_ABGR:
2572       enabled_channels = 0x5;
2573       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2574       break;
2575
2576    case V_028714_SPI_SHADER_UNORM16_ABGR:
2577       enabled_channels = 0x5;
2578       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2579       break;
2580
2581    case V_028714_SPI_SHADER_SNORM16_ABGR:
2582       enabled_channels = 0x5;
2583       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2584       break;
2585
2586    case V_028714_SPI_SHADER_UINT16_ABGR:
2587       enabled_channels = 0x5;
2588       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2589       break;
2590
2591    case V_028714_SPI_SHADER_SINT16_ABGR:
2592       enabled_channels = 0x5;
2593       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2594       break;
2595
2596    case V_028714_SPI_SHADER_32_ABGR:
2597       enabled_channels = 0xF;
2598       break;
2599
2600    default:
2601       break;
2602    }
2603
2604    if (target == V_008DFC_SQ_EXP_NULL)
2605       return;
2606
2607    if ((bool)compr_op)
2608    {
2609       for (int i = 0; i < 2; i++)
2610       {
2611          /* check if at least one of the values to be compressed is enabled */
2612          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2613          if (enabled) {
2614             enabled_channels |= enabled << (i*2);
2615             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2616             Temp tmp{ctx->program->allocateId(), v1};
2617             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2618             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2619             compr->definitions[0] = Definition(tmp);
2620             values[i] = Operand(tmp);
2621             ctx->block->instructions.emplace_back(std::move(compr));
2622          } else {
2623             values[i] = Operand(v1);
2624          }
2625       }
2626    }
2627
2628    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2629    exp->valid_mask = false;
2630    exp->done = false;
2631    exp->compressed = (bool) compr_op;
2632    exp->dest = target;
2633    exp->enabled_mask = enabled_channels;
2634    if ((bool) compr_op) {
2635       for (int i = 0; i < 2; i++)
2636          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2637       exp->operands[2] = Operand(v1);
2638       exp->operands[3] = Operand(v1);
2639    } else {
2640       for (int i = 0; i < 4; i++)
2641          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2642    }
2643
2644    ctx->block->instructions.emplace_back(std::move(exp));
2645 }
2646
2647 Operand load_lds_size_m0(isel_context *ctx)
2648 {
2649    /* TODO: m0 does not need to be initialized on GFX9+ */
2650    Builder bld(ctx->program, ctx->block);
2651    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2652 }
2653
2654 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2655               Temp address, unsigned base_offset, unsigned align)
2656 {
2657    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2658
2659    Builder bld(ctx->program, ctx->block);
2660
2661    Operand m = load_lds_size_m0(ctx);
2662
2663    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2664    unsigned bytes_read = 0;
2665    unsigned result_size = 0;
2666    unsigned total_bytes = num_components * elem_size_bytes;
2667    std::array<Temp, 4> result;
2668
2669    while (bytes_read < total_bytes) {
2670       unsigned todo = total_bytes - bytes_read;
2671       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2672       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2673
2674       aco_opcode op = aco_opcode::last_opcode;
2675       bool read2 = false;
2676       if (todo >= 16 && aligned16) {
2677          op = aco_opcode::ds_read_b128;
2678          todo = 16;
2679       } else if (todo >= 16 && aligned8) {
2680          op = aco_opcode::ds_read2_b64;
2681          read2 = true;
2682          todo = 16;
2683       } else if (todo >= 12 && aligned16) {
2684          op = aco_opcode::ds_read_b96;
2685          todo = 12;
2686       } else if (todo >= 8 && aligned8) {
2687          op = aco_opcode::ds_read_b64;
2688          todo = 8;
2689       } else if (todo >= 8) {
2690          op = aco_opcode::ds_read2_b32;
2691          read2 = true;
2692          todo = 8;
2693       } else if (todo >= 4) {
2694          op = aco_opcode::ds_read_b32;
2695          todo = 4;
2696       } else {
2697          assert(false);
2698       }
2699       assert(todo % elem_size_bytes == 0);
2700       unsigned num_elements = todo / elem_size_bytes;
2701       unsigned offset = base_offset + bytes_read;
2702       unsigned max_offset = read2 ? 1019 : 65535;
2703
2704       Temp address_offset = address;
2705       if (offset > max_offset) {
2706          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2707          offset = bytes_read;
2708       }
2709       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2710
2711       Temp res;
2712       if (num_components == 1 && dst.type() == RegType::vgpr)
2713          res = dst;
2714       else
2715          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2716
2717       if (read2)
2718          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2719       else
2720          res = bld.ds(op, Definition(res), address_offset, m, offset);
2721
2722       if (num_components == 1) {
2723          assert(todo == total_bytes);
2724          if (dst.type() == RegType::sgpr)
2725             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2726          return;
2727       }
2728
2729       if (dst.type() == RegType::sgpr)
2730          res = bld.as_uniform(res);
2731
2732       if (num_elements == 1) {
2733          result[result_size++] = res;
2734       } else {
2735          assert(res != dst && res.size() % num_elements == 0);
2736          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2737          split->operands[0] = Operand(res);
2738          for (unsigned i = 0; i < num_elements; i++)
2739             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2740          ctx->block->instructions.emplace_back(std::move(split));
2741       }
2742
2743       bytes_read += todo;
2744    }
2745
2746    assert(result_size == num_components && result_size > 1);
2747    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2748    for (unsigned i = 0; i < result_size; i++)
2749       vec->operands[i] = Operand(result[i]);
2750    vec->definitions[0] = Definition(dst);
2751    ctx->block->instructions.emplace_back(std::move(vec));
2752    ctx->allocated_vec.emplace(dst.id(), result);
2753 }
2754
2755 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2756 {
2757    if (start == 0 && size == data.size())
2758       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2759
2760    unsigned size_hint = 1;
2761    auto it = ctx->allocated_vec.find(data.id());
2762    if (it != ctx->allocated_vec.end())
2763       size_hint = it->second[0].size();
2764    if (size % size_hint || start % size_hint)
2765       size_hint = 1;
2766
2767    start /= size_hint;
2768    size /= size_hint;
2769
2770    Temp elems[size];
2771    for (unsigned i = 0; i < size; i++)
2772       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2773
2774    if (size == 1)
2775       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2776
2777    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2778    for (unsigned i = 0; i < size; i++)
2779       vec->operands[i] = Operand(elems[i]);
2780    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2781    vec->definitions[0] = Definition(res);
2782    ctx->block->instructions.emplace_back(std::move(vec));
2783    return res;
2784 }
2785
2786 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2787 {
2788    Builder bld(ctx->program, ctx->block);
2789    unsigned bytes_written = 0;
2790    while (bytes_written < total_size * 4) {
2791       unsigned todo = total_size * 4 - bytes_written;
2792       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2793       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2794
2795       aco_opcode op = aco_opcode::last_opcode;
2796       bool write2 = false;
2797       unsigned size = 0;
2798       if (todo >= 16 && aligned16) {
2799          op = aco_opcode::ds_write_b128;
2800          size = 4;
2801       } else if (todo >= 16 && aligned8) {
2802          op = aco_opcode::ds_write2_b64;
2803          write2 = true;
2804          size = 4;
2805       } else if (todo >= 12 && aligned16) {
2806          op = aco_opcode::ds_write_b96;
2807          size = 3;
2808       } else if (todo >= 8 && aligned8) {
2809          op = aco_opcode::ds_write_b64;
2810          size = 2;
2811       } else if (todo >= 8) {
2812          op = aco_opcode::ds_write2_b32;
2813          write2 = true;
2814          size = 2;
2815       } else if (todo >= 4) {
2816          op = aco_opcode::ds_write_b32;
2817          size = 1;
2818       } else {
2819          assert(false);
2820       }
2821
2822       unsigned offset = offset0 + offset1 + bytes_written;
2823       unsigned max_offset = write2 ? 1020 : 65535;
2824       Temp address_offset = address;
2825       if (offset > max_offset) {
2826          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2827          offset = offset1 + bytes_written;
2828       }
2829       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2830
2831       if (write2) {
2832          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2833          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2834          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2835       } else {
2836          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2837          bld.ds(op, address_offset, val, m, offset);
2838       }
2839
2840       bytes_written += size * 4;
2841    }
2842 }
2843
2844 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2845                Temp address, unsigned base_offset, unsigned align)
2846 {
2847    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2848
2849    Operand m = load_lds_size_m0(ctx);
2850
2851    /* we need at most two stores for 32bit variables */
2852    int start[2], count[2];
2853    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2854    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2855    assert(wrmask == 0);
2856
2857    /* one combined store is sufficient */
2858    if (count[0] == count[1]) {
2859       Builder bld(ctx->program, ctx->block);
2860
2861       Temp address_offset = address;
2862       if ((base_offset >> 2) + start[1] > 255) {
2863          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2864          base_offset = 0;
2865       }
2866
2867       assert(count[0] == 1);
2868       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2869       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2870       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2871       base_offset = base_offset / elem_size_bytes;
2872       bld.ds(op, address_offset, val0, val1, m,
2873              base_offset + start[0], base_offset + start[1]);
2874       return;
2875    }
2876
2877    for (unsigned i = 0; i < 2; i++) {
2878       if (count[i] == 0)
2879          continue;
2880
2881       unsigned elem_size_words = elem_size_bytes / 4;
2882       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2883                       base_offset, start[i] * elem_size_bytes, align);
2884    }
2885    return;
2886 }
2887
2888 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2889 {
2890    if (ctx->stage == vertex_vs) {
2891       visit_store_vs_output(ctx, instr);
2892    } else if (ctx->stage == fragment_fs) {
2893       visit_store_fs_output(ctx, instr);
2894    } else {
2895       unreachable("Shader stage not implemented");
2896    }
2897 }
2898
2899 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2900 {
2901    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2902    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2903
2904    Builder bld(ctx->program, ctx->block);
2905    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2906    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2907 }
2908
2909 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2910 {
2911    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2912    for (unsigned i = 0; i < num_components; i++)
2913       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2914
2915    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2916       assert(num_components == 4);
2917       Builder bld(ctx->program, ctx->block);
2918       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2919    }
2920
2921    for (Operand& op : vec->operands)
2922       op = op.isUndefined() ? Operand(0u) : op;
2923
2924    vec->definitions[0] = Definition(dst);
2925    ctx->block->instructions.emplace_back(std::move(vec));
2926    emit_split_vector(ctx, dst, num_components);
2927    return;
2928 }
2929
2930 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2931 {
2932    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2933    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2934    unsigned idx = nir_intrinsic_base(instr);
2935    unsigned component = nir_intrinsic_component(instr);
2936    Temp prim_mask = ctx->prim_mask;
2937
2938    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2939    if (offset) {
2940       assert(offset->u32 == 0);
2941    } else {
2942       /* the lower 15bit of the prim_mask contain the offset into LDS
2943        * while the upper bits contain the number of prims */
2944       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2945       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2946       Builder bld(ctx->program, ctx->block);
2947       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2948       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2949       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2950       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2951       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2952    }
2953
2954    if (instr->dest.ssa.num_components == 1) {
2955       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2956    } else {
2957       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2958       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2959       {
2960          Temp tmp = {ctx->program->allocateId(), v1};
2961          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2962          vec->operands[i] = Operand(tmp);
2963       }
2964       vec->definitions[0] = Definition(dst);
2965       ctx->block->instructions.emplace_back(std::move(vec));
2966    }
2967 }
2968
2969 unsigned get_num_channels_from_data_format(unsigned data_format)
2970 {
2971    switch (data_format) {
2972    case V_008F0C_BUF_DATA_FORMAT_8:
2973    case V_008F0C_BUF_DATA_FORMAT_16:
2974    case V_008F0C_BUF_DATA_FORMAT_32:
2975       return 1;
2976    case V_008F0C_BUF_DATA_FORMAT_8_8:
2977    case V_008F0C_BUF_DATA_FORMAT_16_16:
2978    case V_008F0C_BUF_DATA_FORMAT_32_32:
2979       return 2;
2980    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2981    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2982    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2983       return 3;
2984    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2985    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2986    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2987    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2988    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2989       return 4;
2990    default:
2991       break;
2992    }
2993
2994    return 4;
2995 }
2996
2997 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2998  * so we may need to fix it up. */
2999 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3000 {
3001    Builder bld(ctx->program, ctx->block);
3002
3003    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3004       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3005
3006    /* For the integer-like cases, do a natural sign extension.
3007     *
3008     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3009     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3010     * exponent.
3011     */
3012    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3013    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3014
3015    /* Convert back to the right type. */
3016    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3017       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3018       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
3019       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3020    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3021       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3022    }
3023
3024    return alpha;
3025 }
3026
3027 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3028 {
3029    Builder bld(ctx->program, ctx->block);
3030    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3031    if (ctx->stage & sw_vs) {
3032
3033       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3034       if (off_instr->type != nir_instr_type_load_const) {
3035          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3036          nir_print_instr(off_instr, stderr);
3037          fprintf(stderr, "\n");
3038       }
3039       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3040
3041       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
3042
3043       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3044       unsigned component = nir_intrinsic_component(instr);
3045       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3046       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3047       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3048       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3049
3050       unsigned dfmt = attrib_format & 0xf;
3051
3052       unsigned nfmt = (attrib_format >> 4) & 0x7;
3053       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3054       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3055       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3056       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3057       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3058       if (post_shuffle)
3059          num_channels = MAX2(num_channels, 3);
3060
3061       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3062
3063       Temp index;
3064       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3065          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3066          if (divisor) {
3067             ctx->needs_instance_id = true;
3068
3069             if (divisor != 1) {
3070                Temp divided = bld.tmp(v1);
3071                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
3072                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
3073             } else {
3074                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
3075             }
3076          } else {
3077             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
3078          }
3079       } else {
3080          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
3081       }
3082
3083       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3084          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3085          attrib_offset = attrib_offset % attrib_stride;
3086       }
3087
3088       Operand soffset(0u);
3089       if (attrib_offset >= 4096) {
3090          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3091          attrib_offset = 0;
3092       }
3093
3094       aco_opcode opcode;
3095       switch (num_channels) {
3096       case 1:
3097          opcode = aco_opcode::tbuffer_load_format_x;
3098          break;
3099       case 2:
3100          opcode = aco_opcode::tbuffer_load_format_xy;
3101          break;
3102       case 3:
3103          opcode = aco_opcode::tbuffer_load_format_xyz;
3104          break;
3105       case 4:
3106          opcode = aco_opcode::tbuffer_load_format_xyzw;
3107          break;
3108       default:
3109          unreachable("Unimplemented load_input vector size");
3110       }
3111
3112       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3113
3114       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3115       mubuf->operands[0] = Operand(index);
3116       mubuf->operands[1] = Operand(list);
3117       mubuf->operands[2] = soffset;
3118       mubuf->definitions[0] = Definition(tmp);
3119       mubuf->idxen = true;
3120       mubuf->can_reorder = true;
3121       mubuf->dfmt = dfmt;
3122       mubuf->nfmt = nfmt;
3123       assert(attrib_offset < 4096);
3124       mubuf->offset = attrib_offset;
3125       ctx->block->instructions.emplace_back(std::move(mubuf));
3126
3127       emit_split_vector(ctx, tmp, tmp.size());
3128
3129       if (tmp.id() != dst.id()) {
3130          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3131                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3132
3133          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3134          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3135          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3136
3137          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3138          for (unsigned i = 0; i < dst.size(); i++) {
3139             unsigned idx = i + component;
3140             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3141                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3142                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3143             } else if (idx < num_channels) {
3144                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3145             } else if (is_float && idx == 3) {
3146                vec->operands[i] = Operand(0x3f800000u);
3147             } else if (!is_float && idx == 3) {
3148                vec->operands[i] = Operand(1u);
3149             } else {
3150                vec->operands[i] = Operand(0u);
3151             }
3152          }
3153          vec->definitions[0] = Definition(dst);
3154          ctx->block->instructions.emplace_back(std::move(vec));
3155          emit_split_vector(ctx, dst, dst.size());
3156       }
3157
3158    } else if (ctx->stage == fragment_fs) {
3159       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3160       if (off_instr->type != nir_instr_type_load_const ||
3161           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3162          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3163          nir_print_instr(off_instr, stderr);
3164          fprintf(stderr, "\n");
3165       }
3166
3167       Temp prim_mask = ctx->prim_mask;
3168       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3169       if (offset) {
3170          assert(offset->u32 == 0);
3171       } else {
3172          /* the lower 15bit of the prim_mask contain the offset into LDS
3173           * while the upper bits contain the number of prims */
3174          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3175          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3176          Builder bld(ctx->program, ctx->block);
3177          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3178          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3179          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3180          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3181          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3182       }
3183
3184       unsigned idx = nir_intrinsic_base(instr);
3185       unsigned component = nir_intrinsic_component(instr);
3186
3187       if (dst.size() == 1) {
3188          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3189       } else {
3190          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3191          for (unsigned i = 0; i < dst.size(); i++)
3192             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3193          vec->definitions[0] = Definition(dst);
3194          bld.insert(std::move(vec));
3195       }
3196
3197    } else {
3198       unreachable("Shader stage not implemented");
3199    }
3200 }
3201
3202 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3203 {
3204    if (ctx->program->info->need_indirect_descriptor_sets) {
3205       Builder bld(ctx->program, ctx->block);
3206       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
3207       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3208    }
3209
3210    return ctx->descriptor_sets[desc_set];
3211 }
3212
3213
3214 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3215 {
3216    Builder bld(ctx->program, ctx->block);
3217    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3218    if (!ctx->divergent_vals[instr->dest.ssa.index])
3219       index = bld.as_uniform(index);
3220    unsigned desc_set = nir_intrinsic_desc_set(instr);
3221    unsigned binding = nir_intrinsic_binding(instr);
3222
3223    Temp desc_ptr;
3224    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3225    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3226    unsigned offset = layout->binding[binding].offset;
3227    unsigned stride;
3228    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3229        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3230       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3231       desc_ptr = ctx->push_constants;
3232       offset = pipeline_layout->push_constant_size + 16 * idx;
3233       stride = 16;
3234    } else {
3235       desc_ptr = load_desc_ptr(ctx, desc_set);
3236       stride = layout->binding[binding].size;
3237    }
3238
3239    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3240    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3241    if (stride != 1) {
3242       if (nir_const_index) {
3243          const_index = const_index * stride;
3244       } else if (index.type() == RegType::vgpr) {
3245          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3246          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3247       } else {
3248          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3249       }
3250    }
3251    if (offset) {
3252       if (nir_const_index) {
3253          const_index = const_index + offset;
3254       } else if (index.type() == RegType::vgpr) {
3255          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3256       } else {
3257          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3258       }
3259    }
3260
3261    if (nir_const_index && const_index == 0) {
3262       index = desc_ptr;
3263    } else if (index.type() == RegType::vgpr) {
3264       index = bld.vadd32(bld.def(v1),
3265                          nir_const_index ? Operand(const_index) : Operand(index),
3266                          Operand(desc_ptr));
3267    } else {
3268       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3269                        nir_const_index ? Operand(const_index) : Operand(index),
3270                        Operand(desc_ptr));
3271    }
3272
3273    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3274 }
3275
3276 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3277                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3278 {
3279    Builder bld(ctx->program, ctx->block);
3280
3281    unsigned num_bytes = dst.size() * 4;
3282    bool dlc = glc && ctx->options->chip_class >= GFX10;
3283
3284    aco_opcode op;
3285    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3286       if (ctx->options->chip_class < GFX8)
3287          offset = as_vgpr(ctx, offset);
3288
3289       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3290       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3291       unsigned const_offset = 0;
3292
3293       Temp lower = Temp();
3294       if (num_bytes > 16) {
3295          assert(num_components == 3 || num_components == 4);
3296          op = aco_opcode::buffer_load_dwordx4;
3297          lower = bld.tmp(v4);
3298          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3299          mubuf->definitions[0] = Definition(lower);
3300          mubuf->operands[0] = vaddr;
3301          mubuf->operands[1] = Operand(rsrc);
3302          mubuf->operands[2] = soffset;
3303          mubuf->offen = (offset.type() == RegType::vgpr);
3304          mubuf->glc = glc;
3305          mubuf->dlc = dlc;
3306          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3307          mubuf->can_reorder = readonly;
3308          bld.insert(std::move(mubuf));
3309          emit_split_vector(ctx, lower, 2);
3310          num_bytes -= 16;
3311          const_offset = 16;
3312       }
3313
3314       switch (num_bytes) {
3315          case 4:
3316             op = aco_opcode::buffer_load_dword;
3317             break;
3318          case 8:
3319             op = aco_opcode::buffer_load_dwordx2;
3320             break;
3321          case 12:
3322             op = aco_opcode::buffer_load_dwordx3;
3323             break;
3324          case 16:
3325             op = aco_opcode::buffer_load_dwordx4;
3326             break;
3327          default:
3328             unreachable("Load SSBO not implemented for this size.");
3329       }
3330       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3331       mubuf->operands[0] = vaddr;
3332       mubuf->operands[1] = Operand(rsrc);
3333       mubuf->operands[2] = soffset;
3334       mubuf->offen = (offset.type() == RegType::vgpr);
3335       mubuf->glc = glc;
3336       mubuf->dlc = dlc;
3337       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3338       mubuf->can_reorder = readonly;
3339       mubuf->offset = const_offset;
3340       aco_ptr<Instruction> instr = std::move(mubuf);
3341
3342       if (dst.size() > 4) {
3343          assert(lower != Temp());
3344          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3345          instr->definitions[0] = Definition(upper);
3346          bld.insert(std::move(instr));
3347          if (dst.size() == 8)
3348             emit_split_vector(ctx, upper, 2);
3349          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3350          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3351          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3352          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3353          if (dst.size() == 8)
3354             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3355       }
3356
3357       if (dst.type() == RegType::sgpr) {
3358          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3359          instr->definitions[0] = Definition(vec);
3360          bld.insert(std::move(instr));
3361          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3362       } else {
3363          instr->definitions[0] = Definition(dst);
3364          bld.insert(std::move(instr));
3365       }
3366    } else {
3367       switch (num_bytes) {
3368          case 4:
3369             op = aco_opcode::s_buffer_load_dword;
3370             break;
3371          case 8:
3372             op = aco_opcode::s_buffer_load_dwordx2;
3373             break;
3374          case 12:
3375          case 16:
3376             op = aco_opcode::s_buffer_load_dwordx4;
3377             break;
3378          case 24:
3379          case 32:
3380             op = aco_opcode::s_buffer_load_dwordx8;
3381             break;
3382          default:
3383             unreachable("Load SSBO not implemented for this size.");
3384       }
3385       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3386       load->operands[0] = Operand(rsrc);
3387       load->operands[1] = Operand(bld.as_uniform(offset));
3388       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3389       load->definitions[0] = Definition(dst);
3390       load->glc = glc;
3391       load->dlc = dlc;
3392       load->barrier = readonly ? barrier_none : barrier_buffer;
3393       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3394       assert(ctx->options->chip_class >= GFX8 || !glc);
3395
3396       /* trim vector */
3397       if (dst.size() == 3) {
3398          Temp vec = bld.tmp(s4);
3399          load->definitions[0] = Definition(vec);
3400          bld.insert(std::move(load));
3401          emit_split_vector(ctx, vec, 4);
3402
3403          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3404                     emit_extract_vector(ctx, vec, 0, s1),
3405                     emit_extract_vector(ctx, vec, 1, s1),
3406                     emit_extract_vector(ctx, vec, 2, s1));
3407       } else if (dst.size() == 6) {
3408          Temp vec = bld.tmp(s8);
3409          load->definitions[0] = Definition(vec);
3410          bld.insert(std::move(load));
3411          emit_split_vector(ctx, vec, 4);
3412
3413          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3414                     emit_extract_vector(ctx, vec, 0, s2),
3415                     emit_extract_vector(ctx, vec, 1, s2),
3416                     emit_extract_vector(ctx, vec, 2, s2));
3417       } else {
3418          bld.insert(std::move(load));
3419       }
3420
3421    }
3422    emit_split_vector(ctx, dst, num_components);
3423 }
3424
3425 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3426 {
3427    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3428    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3429
3430    Builder bld(ctx->program, ctx->block);
3431
3432    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3433    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3434    unsigned binding = nir_intrinsic_binding(idx_instr);
3435    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3436
3437    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3438       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3439                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3440                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3441                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3442       if (ctx->options->chip_class >= GFX10) {
3443          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3444                       S_008F0C_OOB_SELECT(3) |
3445                       S_008F0C_RESOURCE_LEVEL(1);
3446       } else {
3447          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3448                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3449       }
3450       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3451                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3452                                      Operand(0xFFFFFFFFu),
3453                                      Operand(desc_type));
3454       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3455                         rsrc, upper_dwords);
3456    } else {
3457       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3458       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3459    }
3460
3461    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3462 }
3463
3464 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3465 {
3466    Builder bld(ctx->program, ctx->block);
3467    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3468
3469    unsigned offset = nir_intrinsic_base(instr);
3470    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3471    if (index_cv && instr->dest.ssa.bit_size == 32) {
3472
3473       unsigned count = instr->dest.ssa.num_components;
3474       unsigned start = (offset + index_cv->u32) / 4u;
3475       start -= ctx->base_inline_push_consts;
3476       if (start + count <= ctx->num_inline_push_consts) {
3477          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3478          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3479          for (unsigned i = 0; i < count; ++i) {
3480             elems[i] = ctx->inline_push_consts[start + i];
3481             vec->operands[i] = Operand{elems[i]};
3482          }
3483          vec->definitions[0] = Definition(dst);
3484          ctx->block->instructions.emplace_back(std::move(vec));
3485          ctx->allocated_vec.emplace(dst.id(), elems);
3486          return;
3487       }
3488    }
3489
3490    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3491    if (offset != 0) // TODO check if index != 0 as well
3492       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3493    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3494    Temp vec = dst;
3495    bool trim = false;
3496    aco_opcode op;
3497
3498    switch (dst.size()) {
3499    case 1:
3500       op = aco_opcode::s_load_dword;
3501       break;
3502    case 2:
3503       op = aco_opcode::s_load_dwordx2;
3504       break;
3505    case 3:
3506       vec = bld.tmp(s4);
3507       trim = true;
3508    case 4:
3509       op = aco_opcode::s_load_dwordx4;
3510       break;
3511    case 6:
3512       vec = bld.tmp(s8);
3513       trim = true;
3514    case 8:
3515       op = aco_opcode::s_load_dwordx8;
3516       break;
3517    default:
3518       unreachable("unimplemented or forbidden load_push_constant.");
3519    }
3520
3521    bld.smem(op, Definition(vec), ptr, index);
3522
3523    if (trim) {
3524       emit_split_vector(ctx, vec, 4);
3525       RegClass rc = dst.size() == 3 ? s1 : s2;
3526       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3527                  emit_extract_vector(ctx, vec, 0, rc),
3528                  emit_extract_vector(ctx, vec, 1, rc),
3529                  emit_extract_vector(ctx, vec, 2, rc));
3530
3531    }
3532    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3533 }
3534
3535 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3536 {
3537    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3538
3539    Builder bld(ctx->program, ctx->block);
3540
3541    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3542                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3543                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3544                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3545    if (ctx->options->chip_class >= GFX10) {
3546       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3547                    S_008F0C_OOB_SELECT(3) |
3548                    S_008F0C_RESOURCE_LEVEL(1);
3549    } else {
3550       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3551                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3552    }
3553
3554    unsigned base = nir_intrinsic_base(instr);
3555    unsigned range = nir_intrinsic_range(instr);
3556
3557    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3558    if (base && offset.type() == RegType::sgpr)
3559       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3560    else if (base && offset.type() == RegType::vgpr)
3561       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3562
3563    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3564                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3565                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3566                           Operand(desc_type));
3567
3568    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3569 }
3570
3571 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3572 {
3573    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3574       ctx->cf_info.exec_potentially_empty = true;
3575
3576    ctx->program->needs_exact = true;
3577
3578    // TODO: optimize uniform conditions
3579    Builder bld(ctx->program, ctx->block);
3580    Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
3581    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3582    bld.pseudo(aco_opcode::p_discard_if, src);
3583    ctx->block->kind |= block_kind_uses_discard_if;
3584    return;
3585 }
3586
3587 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3588 {
3589    Builder bld(ctx->program, ctx->block);
3590
3591    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3592       ctx->cf_info.exec_potentially_empty = true;
3593
3594    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3595                     ctx->cf_info.parent_loop.has_divergent_continue;
3596
3597    if (ctx->block->loop_nest_depth &&
3598        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3599       /* we handle discards the same way as jump instructions */
3600       append_logical_end(ctx->block);
3601
3602       /* in loops, discard behaves like break */
3603       Block *linear_target = ctx->cf_info.parent_loop.exit;
3604       ctx->block->kind |= block_kind_discard;
3605
3606       if (!divergent) {
3607          /* uniform discard - loop ends here */
3608          assert(nir_instr_is_last(&instr->instr));
3609          ctx->block->kind |= block_kind_uniform;
3610          ctx->cf_info.has_branch = true;
3611          bld.branch(aco_opcode::p_branch);
3612          add_linear_edge(ctx->block->index, linear_target);
3613          return;
3614       }
3615
3616       /* we add a break right behind the discard() instructions */
3617       ctx->block->kind |= block_kind_break;
3618       unsigned idx = ctx->block->index;
3619
3620       /* remove critical edges from linear CFG */
3621       bld.branch(aco_opcode::p_branch);
3622       Block* break_block = ctx->program->create_and_insert_block();
3623       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3624       break_block->kind |= block_kind_uniform;
3625       add_linear_edge(idx, break_block);
3626       add_linear_edge(break_block->index, linear_target);
3627       bld.reset(break_block);
3628       bld.branch(aco_opcode::p_branch);
3629
3630       Block* continue_block = ctx->program->create_and_insert_block();
3631       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3632       add_linear_edge(idx, continue_block);
3633       append_logical_start(continue_block);
3634       ctx->block = continue_block;
3635
3636       return;
3637    }
3638
3639    /* it can currently happen that NIR doesn't remove the unreachable code */
3640    if (!nir_instr_is_last(&instr->instr)) {
3641       ctx->program->needs_exact = true;
3642       /* save exec somewhere temporarily so that it doesn't get
3643        * overwritten before the discard from outer exec masks */
3644       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3645       bld.pseudo(aco_opcode::p_discard_if, cond);
3646       ctx->block->kind |= block_kind_uses_discard_if;
3647       return;
3648    }
3649
3650    /* This condition is incorrect for uniformly branched discards in a loop
3651     * predicated by a divergent condition, but the above code catches that case
3652     * and the discard would end up turning into a discard_if.
3653     * For example:
3654     * if (divergent) {
3655     *    while (...) {
3656     *       if (uniform) {
3657     *          discard;
3658     *       }
3659     *    }
3660     * }
3661     */
3662    if (!ctx->cf_info.parent_if.is_divergent) {
3663       /* program just ends here */
3664       ctx->block->kind |= block_kind_uniform;
3665       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3666               0 /* enabled mask */, 9 /* dest */,
3667               false /* compressed */, true/* done */, true /* valid mask */);
3668       bld.sopp(aco_opcode::s_endpgm);
3669       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3670    } else {
3671       ctx->block->kind |= block_kind_discard;
3672       /* branch and linear edge is added by visit_if() */
3673    }
3674 }
3675
3676 enum aco_descriptor_type {
3677    ACO_DESC_IMAGE,
3678    ACO_DESC_FMASK,
3679    ACO_DESC_SAMPLER,
3680    ACO_DESC_BUFFER,
3681    ACO_DESC_PLANE_0,
3682    ACO_DESC_PLANE_1,
3683    ACO_DESC_PLANE_2,
3684 };
3685
3686 static bool
3687 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3688    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3689       return false;
3690    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3691    return dim == ac_image_cube ||
3692           dim == ac_image_1darray ||
3693           dim == ac_image_2darray ||
3694           dim == ac_image_2darraymsaa;
3695 }
3696
3697 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3698                       enum aco_descriptor_type desc_type,
3699                       const nir_tex_instr *tex_instr, bool image, bool write)
3700 {
3701 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3702    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3703    if (it != ctx->tex_desc.end())
3704       return it->second;
3705 */
3706    Temp index = Temp();
3707    bool index_set = false;
3708    unsigned constant_index = 0;
3709    unsigned descriptor_set;
3710    unsigned base_index;
3711    Builder bld(ctx->program, ctx->block);
3712
3713    if (!deref_instr) {
3714       assert(tex_instr && !image);
3715       descriptor_set = 0;
3716       base_index = tex_instr->sampler_index;
3717    } else {
3718       while(deref_instr->deref_type != nir_deref_type_var) {
3719          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3720          if (!array_size)
3721             array_size = 1;
3722
3723          assert(deref_instr->deref_type == nir_deref_type_array);
3724          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3725          if (const_value) {
3726             constant_index += array_size * const_value->u32;
3727          } else {
3728             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3729             if (indirect.type() == RegType::vgpr)
3730                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3731
3732             if (array_size != 1)
3733                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3734
3735             if (!index_set) {
3736                index = indirect;
3737                index_set = true;
3738             } else {
3739                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3740             }
3741          }
3742
3743          deref_instr = nir_src_as_deref(deref_instr->parent);
3744       }
3745       descriptor_set = deref_instr->var->data.descriptor_set;
3746       base_index = deref_instr->var->data.binding;
3747    }
3748
3749    Temp list = load_desc_ptr(ctx, descriptor_set);
3750    list = convert_pointer_to_64_bit(ctx, list);
3751
3752    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3753    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3754    unsigned offset = binding->offset;
3755    unsigned stride = binding->size;
3756    aco_opcode opcode;
3757    RegClass type;
3758
3759    assert(base_index < layout->binding_count);
3760
3761    switch (desc_type) {
3762    case ACO_DESC_IMAGE:
3763       type = s8;
3764       opcode = aco_opcode::s_load_dwordx8;
3765       break;
3766    case ACO_DESC_FMASK:
3767       type = s8;
3768       opcode = aco_opcode::s_load_dwordx8;
3769       offset += 32;
3770       break;
3771    case ACO_DESC_SAMPLER:
3772       type = s4;
3773       opcode = aco_opcode::s_load_dwordx4;
3774       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3775          offset += radv_combined_image_descriptor_sampler_offset(binding);
3776       break;
3777    case ACO_DESC_BUFFER:
3778       type = s4;
3779       opcode = aco_opcode::s_load_dwordx4;
3780       break;
3781    case ACO_DESC_PLANE_0:
3782    case ACO_DESC_PLANE_1:
3783       type = s8;
3784       opcode = aco_opcode::s_load_dwordx8;
3785       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3786       break;
3787    case ACO_DESC_PLANE_2:
3788       type = s4;
3789       opcode = aco_opcode::s_load_dwordx4;
3790       offset += 64;
3791       break;
3792    default:
3793       unreachable("invalid desc_type\n");
3794    }
3795
3796    offset += constant_index * stride;
3797
3798    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3799       (!index_set || binding->immutable_samplers_equal)) {
3800       if (binding->immutable_samplers_equal)
3801          constant_index = 0;
3802
3803       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3804       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3805                         Operand(samplers[constant_index * 4 + 0]),
3806                         Operand(samplers[constant_index * 4 + 1]),
3807                         Operand(samplers[constant_index * 4 + 2]),
3808                         Operand(samplers[constant_index * 4 + 3]));
3809    }
3810
3811    Operand off;
3812    if (!index_set) {
3813       off = Operand(offset);
3814    } else {
3815       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3816                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3817    }
3818
3819    Temp res = bld.smem(opcode, bld.def(type), list, off);
3820
3821    if (desc_type == ACO_DESC_PLANE_2) {
3822       Temp components[8];
3823       for (unsigned i = 0; i < 8; i++)
3824          components[i] = bld.tmp(s1);
3825       bld.pseudo(aco_opcode::p_split_vector,
3826                  Definition(components[0]),
3827                  Definition(components[1]),
3828                  Definition(components[2]),
3829                  Definition(components[3]),
3830                  res);
3831
3832       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3833       bld.pseudo(aco_opcode::p_split_vector,
3834                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3835                  Definition(components[4]),
3836                  Definition(components[5]),
3837                  Definition(components[6]),
3838                  Definition(components[7]),
3839                  desc2);
3840
3841       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3842                        components[0], components[1], components[2], components[3],
3843                        components[4], components[5], components[6], components[7]);
3844    }
3845
3846    return res;
3847 }
3848
3849 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3850 {
3851    switch (dim) {
3852    case GLSL_SAMPLER_DIM_BUF:
3853       return 1;
3854    case GLSL_SAMPLER_DIM_1D:
3855       return array ? 2 : 1;
3856    case GLSL_SAMPLER_DIM_2D:
3857       return array ? 3 : 2;
3858    case GLSL_SAMPLER_DIM_MS:
3859       return array ? 4 : 3;
3860    case GLSL_SAMPLER_DIM_3D:
3861    case GLSL_SAMPLER_DIM_CUBE:
3862       return 3;
3863    case GLSL_SAMPLER_DIM_RECT:
3864    case GLSL_SAMPLER_DIM_SUBPASS:
3865       return 2;
3866    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3867       return 3;
3868    default:
3869       break;
3870    }
3871    return 0;
3872 }
3873
3874
3875 /* Adjust the sample index according to FMASK.
3876  *
3877  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3878  * which is the identity mapping. Each nibble says which physical sample
3879  * should be fetched to get that sample.
3880  *
3881  * For example, 0x11111100 means there are only 2 samples stored and
3882  * the second sample covers 3/4 of the pixel. When reading samples 0
3883  * and 1, return physical sample 0 (determined by the first two 0s
3884  * in FMASK), otherwise return physical sample 1.
3885  *
3886  * The sample index should be adjusted as follows:
3887  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3888  */
3889 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3890 {
3891    Builder bld(ctx->program, ctx->block);
3892    Temp fmask = bld.tmp(v1);
3893    unsigned dim = ctx->options->chip_class >= GFX10
3894                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3895                   : 0;
3896
3897    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3898    load->operands[0] = Operand(coords);
3899    load->operands[1] = Operand(fmask_desc_ptr);
3900    load->definitions[0] = Definition(fmask);
3901    load->glc = false;
3902    load->dlc = false;
3903    load->dmask = 0x1;
3904    load->unrm = true;
3905    load->da = da;
3906    load->dim = dim;
3907    load->can_reorder = true; /* fmask images shouldn't be modified */
3908    ctx->block->instructions.emplace_back(std::move(load));
3909
3910    Operand sample_index4;
3911    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3912       sample_index4 = Operand(sample_index.constantValue() << 2);
3913    } else if (sample_index.regClass() == s1) {
3914       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3915    } else {
3916       assert(sample_index.regClass() == v1);
3917       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3918    }
3919
3920    Temp final_sample;
3921    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3922       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3923    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3924       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3925    else
3926       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3927
3928    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3929     * resource descriptor is 0 (invalid),
3930     */
3931    Temp compare = bld.tmp(s2);
3932    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3933                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3934
3935    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3936
3937    /* Replace the MSAA sample index. */
3938    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3939 }
3940
3941 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3942 {
3943
3944    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3945    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3946    bool is_array = glsl_sampler_type_is_array(type);
3947    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3948    assert(!add_frag_pos && "Input attachments should be lowered.");
3949    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3950    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3951    int count = image_type_to_components_count(dim, is_array);
3952    std::vector<Operand> coords(count);
3953
3954    if (is_ms) {
3955       Operand sample_index;
3956       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3957       if (sample_cv)
3958          sample_index = Operand(sample_cv->u32);
3959       else
3960          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3961
3962       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3963          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3964          for (unsigned i = 0; i < vec->operands.size(); i++)
3965             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3966          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3967          vec->definitions[0] = Definition(fmask_load_address);
3968          ctx->block->instructions.emplace_back(std::move(vec));
3969
3970          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3971          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3972       }
3973       count--;
3974       coords[count] = sample_index;
3975    }
3976
3977    if (count == 1 && !gfx9_1d)
3978       return emit_extract_vector(ctx, src0, 0, v1);
3979
3980    if (gfx9_1d) {
3981       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3982       coords.resize(coords.size() + 1);
3983       coords[1] = Operand((uint32_t) 0);
3984       if (is_array)
3985          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3986    } else {
3987       for (int i = 0; i < count; i++)
3988          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3989    }
3990
3991    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3992    for (unsigned i = 0; i < coords.size(); i++)
3993       vec->operands[i] = coords[i];
3994    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3995    vec->definitions[0] = Definition(res);
3996    ctx->block->instructions.emplace_back(std::move(vec));
3997    return res;
3998 }
3999
4000
4001 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4002 {
4003    Builder bld(ctx->program, ctx->block);
4004    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4005    const struct glsl_type *type = glsl_without_array(var->type);
4006    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4007    bool is_array = glsl_sampler_type_is_array(type);
4008    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4009
4010    if (dim == GLSL_SAMPLER_DIM_BUF) {
4011       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4012       unsigned num_channels = util_last_bit(mask);
4013       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4014       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4015
4016       aco_opcode opcode;
4017       switch (num_channels) {
4018       case 1:
4019          opcode = aco_opcode::buffer_load_format_x;
4020          break;
4021       case 2:
4022          opcode = aco_opcode::buffer_load_format_xy;
4023          break;
4024       case 3:
4025          opcode = aco_opcode::buffer_load_format_xyz;
4026          break;
4027       case 4:
4028          opcode = aco_opcode::buffer_load_format_xyzw;
4029          break;
4030       default:
4031          unreachable(">4 channel buffer image load");
4032       }
4033       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4034       load->operands[0] = Operand(vindex);
4035       load->operands[1] = Operand(rsrc);
4036       load->operands[2] = Operand((uint32_t) 0);
4037       Temp tmp;
4038       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4039          tmp = dst;
4040       else
4041          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4042       load->definitions[0] = Definition(tmp);
4043       load->idxen = true;
4044       load->barrier = barrier_image;
4045       ctx->block->instructions.emplace_back(std::move(load));
4046
4047       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4048       return;
4049    }
4050
4051    Temp coords = get_image_coords(ctx, instr, type);
4052    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4053
4054    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4055    unsigned num_components = util_bitcount(dmask);
4056    Temp tmp;
4057    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4058       tmp = dst;
4059    else
4060       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4061
4062    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4063    load->operands[0] = Operand(coords);
4064    load->operands[1] = Operand(resource);
4065    load->definitions[0] = Definition(tmp);
4066    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4067    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4068    load->dmask = dmask;
4069    load->unrm = true;
4070    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4071    load->barrier = barrier_image;
4072    ctx->block->instructions.emplace_back(std::move(load));
4073
4074    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4075    return;
4076 }
4077
4078 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4079 {
4080    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4081    const struct glsl_type *type = glsl_without_array(var->type);
4082    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4083    bool is_array = glsl_sampler_type_is_array(type);
4084    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4085
4086    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4087
4088    if (dim == GLSL_SAMPLER_DIM_BUF) {
4089       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4090       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4091       aco_opcode opcode;
4092       switch (data.size()) {
4093       case 1:
4094          opcode = aco_opcode::buffer_store_format_x;
4095          break;
4096       case 2:
4097          opcode = aco_opcode::buffer_store_format_xy;
4098          break;
4099       case 3:
4100          opcode = aco_opcode::buffer_store_format_xyz;
4101          break;
4102       case 4:
4103          opcode = aco_opcode::buffer_store_format_xyzw;
4104          break;
4105       default:
4106          unreachable(">4 channel buffer image store");
4107       }
4108       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4109       store->operands[0] = Operand(vindex);
4110       store->operands[1] = Operand(rsrc);
4111       store->operands[2] = Operand((uint32_t) 0);
4112       store->operands[3] = Operand(data);
4113       store->idxen = true;
4114       store->glc = glc;
4115       store->dlc = false;
4116       store->disable_wqm = true;
4117       store->barrier = barrier_image;
4118       ctx->program->needs_exact = true;
4119       ctx->block->instructions.emplace_back(std::move(store));
4120       return;
4121    }
4122
4123    assert(data.type() == RegType::vgpr);
4124    Temp coords = get_image_coords(ctx, instr, type);
4125    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4126
4127    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4128    store->operands[0] = Operand(coords);
4129    store->operands[1] = Operand(resource);
4130    store->operands[2] = Operand(s4);
4131    store->operands[3] = Operand(data);
4132    store->glc = glc;
4133    store->dlc = false;
4134    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4135    store->dmask = (1 << data.size()) - 1;
4136    store->unrm = true;
4137    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4138    store->disable_wqm = true;
4139    store->barrier = barrier_image;
4140    ctx->program->needs_exact = true;
4141    ctx->block->instructions.emplace_back(std::move(store));
4142    return;
4143 }
4144
4145 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4146 {
4147    /* return the previous value if dest is ever used */
4148    bool return_previous = false;
4149    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4150       return_previous = true;
4151       break;
4152    }
4153    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4154       return_previous = true;
4155       break;
4156    }
4157
4158    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4159    const struct glsl_type *type = glsl_without_array(var->type);
4160    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4161    bool is_array = glsl_sampler_type_is_array(type);
4162    Builder bld(ctx->program, ctx->block);
4163
4164    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4165    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4166
4167    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4168       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4169
4170    aco_opcode buf_op, image_op;
4171    switch (instr->intrinsic) {
4172       case nir_intrinsic_image_deref_atomic_add:
4173          buf_op = aco_opcode::buffer_atomic_add;
4174          image_op = aco_opcode::image_atomic_add;
4175          break;
4176       case nir_intrinsic_image_deref_atomic_umin:
4177          buf_op = aco_opcode::buffer_atomic_umin;
4178          image_op = aco_opcode::image_atomic_umin;
4179          break;
4180       case nir_intrinsic_image_deref_atomic_imin:
4181          buf_op = aco_opcode::buffer_atomic_smin;
4182          image_op = aco_opcode::image_atomic_smin;
4183          break;
4184       case nir_intrinsic_image_deref_atomic_umax:
4185          buf_op = aco_opcode::buffer_atomic_umax;
4186          image_op = aco_opcode::image_atomic_umax;
4187          break;
4188       case nir_intrinsic_image_deref_atomic_imax:
4189          buf_op = aco_opcode::buffer_atomic_smax;
4190          image_op = aco_opcode::image_atomic_smax;
4191          break;
4192       case nir_intrinsic_image_deref_atomic_and:
4193          buf_op = aco_opcode::buffer_atomic_and;
4194          image_op = aco_opcode::image_atomic_and;
4195          break;
4196       case nir_intrinsic_image_deref_atomic_or:
4197          buf_op = aco_opcode::buffer_atomic_or;
4198          image_op = aco_opcode::image_atomic_or;
4199          break;
4200       case nir_intrinsic_image_deref_atomic_xor:
4201          buf_op = aco_opcode::buffer_atomic_xor;
4202          image_op = aco_opcode::image_atomic_xor;
4203          break;
4204       case nir_intrinsic_image_deref_atomic_exchange:
4205          buf_op = aco_opcode::buffer_atomic_swap;
4206          image_op = aco_opcode::image_atomic_swap;
4207          break;
4208       case nir_intrinsic_image_deref_atomic_comp_swap:
4209          buf_op = aco_opcode::buffer_atomic_cmpswap;
4210          image_op = aco_opcode::image_atomic_cmpswap;
4211          break;
4212       default:
4213          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4214    }
4215
4216    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4217
4218    if (dim == GLSL_SAMPLER_DIM_BUF) {
4219       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4220       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4221       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4222       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4223       mubuf->operands[0] = Operand(vindex);
4224       mubuf->operands[1] = Operand(resource);
4225       mubuf->operands[2] = Operand((uint32_t)0);
4226       mubuf->operands[3] = Operand(data);
4227       if (return_previous)
4228          mubuf->definitions[0] = Definition(dst);
4229       mubuf->offset = 0;
4230       mubuf->idxen = true;
4231       mubuf->glc = return_previous;
4232       mubuf->dlc = false; /* Not needed for atomics */
4233       mubuf->disable_wqm = true;
4234       mubuf->barrier = barrier_image;
4235       ctx->program->needs_exact = true;
4236       ctx->block->instructions.emplace_back(std::move(mubuf));
4237       return;
4238    }
4239
4240    Temp coords = get_image_coords(ctx, instr, type);
4241    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4242    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4243    mimg->operands[0] = Operand(coords);
4244    mimg->operands[1] = Operand(resource);
4245    mimg->operands[2] = Operand(s4); /* no sampler */
4246    mimg->operands[3] = Operand(data);
4247    if (return_previous)
4248       mimg->definitions[0] = Definition(dst);
4249    mimg->glc = return_previous;
4250    mimg->dlc = false; /* Not needed for atomics */
4251    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4252    mimg->dmask = (1 << data.size()) - 1;
4253    mimg->unrm = true;
4254    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4255    mimg->disable_wqm = true;
4256    mimg->barrier = barrier_image;
4257    ctx->program->needs_exact = true;
4258    ctx->block->instructions.emplace_back(std::move(mimg));
4259    return;
4260 }
4261
4262 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4263 {
4264    if (in_elements && ctx->options->chip_class == GFX8) {
4265       Builder bld(ctx->program, ctx->block);
4266
4267       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4268       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4269       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4270       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4271
4272       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4273       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4274
4275       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4276       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4277       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4278
4279       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4280       /* idea
4281        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4282        * in case 12 (or 3?), we have to divide by 3:
4283        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4284        * use v_mul_hi_u32 with magic number to divide
4285        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4286        * disable v_skip
4287        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4288        */
4289
4290    } else {
4291       emit_extract_vector(ctx, desc, 2, dst);
4292    }
4293 }
4294
4295 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4296 {
4297    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4298    const struct glsl_type *type = glsl_without_array(var->type);
4299    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4300    bool is_array = glsl_sampler_type_is_array(type);
4301    Builder bld(ctx->program, ctx->block);
4302
4303    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4304       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4305       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4306    }
4307
4308    /* LOD */
4309    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4310
4311    /* Resource */
4312    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4313
4314    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4315
4316    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4317    mimg->operands[0] = Operand(lod);
4318    mimg->operands[1] = Operand(resource);
4319    unsigned& dmask = mimg->dmask;
4320    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4321    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4322    mimg->da = glsl_sampler_type_is_array(type);
4323    mimg->can_reorder = true;
4324    Definition& def = mimg->definitions[0];
4325    ctx->block->instructions.emplace_back(std::move(mimg));
4326
4327    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4328        glsl_sampler_type_is_array(type)) {
4329
4330       assert(instr->dest.ssa.num_components == 3);
4331       Temp tmp = {ctx->program->allocateId(), v3};
4332       def = Definition(tmp);
4333       emit_split_vector(ctx, tmp, 3);
4334
4335       /* divide 3rd value by 6 by multiplying with magic number */
4336       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4337       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4338
4339       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4340                  emit_extract_vector(ctx, tmp, 0, v1),
4341                  emit_extract_vector(ctx, tmp, 1, v1),
4342                  by_6);
4343
4344    } else if (ctx->options->chip_class == GFX9 &&
4345               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4346               glsl_sampler_type_is_array(type)) {
4347       assert(instr->dest.ssa.num_components == 2);
4348       def = Definition(dst);
4349       dmask = 0x5;
4350    } else {
4351       def = Definition(dst);
4352    }
4353
4354    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4355 }
4356
4357 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4358 {
4359    Builder bld(ctx->program, ctx->block);
4360    unsigned num_components = instr->num_components;
4361
4362    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4363    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4364    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4365
4366    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4367    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4368 }
4369
4370 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4371 {
4372    Builder bld(ctx->program, ctx->block);
4373    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4374    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4375    unsigned writemask = nir_intrinsic_write_mask(instr);
4376
4377    Temp offset;
4378    if (ctx->options->chip_class < GFX8)
4379       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4380    else
4381       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4382
4383    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4384    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4385
4386    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4387                ctx->options->chip_class >= GFX8;
4388    if (smem)
4389       offset = bld.as_uniform(offset);
4390    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4391
4392    while (writemask) {
4393       int start, count;
4394       u_bit_scan_consecutive_range(&writemask, &start, &count);
4395       if (count == 3 && smem) {
4396          writemask |= 1u << (start + 2);
4397          count = 2;
4398       }
4399       int num_bytes = count * elem_size_bytes;
4400
4401       if (num_bytes > 16) {
4402          assert(elem_size_bytes == 8);
4403          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4404          count = 2;
4405          num_bytes = 16;
4406       }
4407
4408       // TODO: check alignment of sub-dword stores
4409       // TODO: split 3 bytes. there is no store instruction for that
4410
4411       Temp write_data;
4412       if (count != instr->num_components) {
4413          emit_split_vector(ctx, data, instr->num_components);
4414          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4415          for (int i = 0; i < count; i++) {
4416             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4417             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4418          }
4419          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4420          vec->definitions[0] = Definition(write_data);
4421          ctx->block->instructions.emplace_back(std::move(vec));
4422       } else if (!smem && data.type() != RegType::vgpr) {
4423          assert(num_bytes % 4 == 0);
4424          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4425       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4426          assert(num_bytes % 4 == 0);
4427          write_data = bld.as_uniform(data);
4428       } else {
4429          write_data = data;
4430       }
4431
4432       aco_opcode vmem_op, smem_op;
4433       switch (num_bytes) {
4434          case 4:
4435             vmem_op = aco_opcode::buffer_store_dword;
4436             smem_op = aco_opcode::s_buffer_store_dword;
4437             break;
4438          case 8:
4439             vmem_op = aco_opcode::buffer_store_dwordx2;
4440             smem_op = aco_opcode::s_buffer_store_dwordx2;
4441             break;
4442          case 12:
4443             vmem_op = aco_opcode::buffer_store_dwordx3;
4444             smem_op = aco_opcode::last_opcode;
4445             assert(!smem);
4446             break;
4447          case 16:
4448             vmem_op = aco_opcode::buffer_store_dwordx4;
4449             smem_op = aco_opcode::s_buffer_store_dwordx4;
4450             break;
4451          default:
4452             unreachable("Store SSBO not implemented for this size.");
4453       }
4454       if (ctx->stage == fragment_fs)
4455          smem_op = aco_opcode::p_fs_buffer_store_smem;
4456
4457       if (smem) {
4458          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4459          store->operands[0] = Operand(rsrc);
4460          if (start) {
4461             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4462                                 offset, Operand(start * elem_size_bytes));
4463             store->operands[1] = Operand(off);
4464          } else {
4465             store->operands[1] = Operand(offset);
4466          }
4467          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4468             store->operands[1].setFixed(m0);
4469          store->operands[2] = Operand(write_data);
4470          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4471          store->dlc = false;
4472          store->disable_wqm = true;
4473          store->barrier = barrier_buffer;
4474          ctx->block->instructions.emplace_back(std::move(store));
4475          ctx->program->wb_smem_l1_on_end = true;
4476          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4477             ctx->block->kind |= block_kind_needs_lowering;
4478             ctx->program->needs_exact = true;
4479          }
4480       } else {
4481          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4482          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4483          store->operands[1] = Operand(rsrc);
4484          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4485          store->operands[3] = Operand(write_data);
4486          store->offset = start * elem_size_bytes;
4487          store->offen = (offset.type() == RegType::vgpr);
4488          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4489          store->dlc = false;
4490          store->disable_wqm = true;
4491          store->barrier = barrier_buffer;
4492          ctx->program->needs_exact = true;
4493          ctx->block->instructions.emplace_back(std::move(store));
4494       }
4495    }
4496 }
4497
4498 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4499 {
4500    /* return the previous value if dest is ever used */
4501    bool return_previous = false;
4502    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4503       return_previous = true;
4504       break;
4505    }
4506    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4507       return_previous = true;
4508       break;
4509    }
4510
4511    Builder bld(ctx->program, ctx->block);
4512    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4513
4514    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4515       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4516                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4517
4518    Temp offset;
4519    if (ctx->options->chip_class < GFX8)
4520       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4521    else
4522       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4523
4524    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4525    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4526
4527    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4528
4529    aco_opcode op32, op64;
4530    switch (instr->intrinsic) {
4531       case nir_intrinsic_ssbo_atomic_add:
4532          op32 = aco_opcode::buffer_atomic_add;
4533          op64 = aco_opcode::buffer_atomic_add_x2;
4534          break;
4535       case nir_intrinsic_ssbo_atomic_imin:
4536          op32 = aco_opcode::buffer_atomic_smin;
4537          op64 = aco_opcode::buffer_atomic_smin_x2;
4538          break;
4539       case nir_intrinsic_ssbo_atomic_umin:
4540          op32 = aco_opcode::buffer_atomic_umin;
4541          op64 = aco_opcode::buffer_atomic_umin_x2;
4542          break;
4543       case nir_intrinsic_ssbo_atomic_imax:
4544          op32 = aco_opcode::buffer_atomic_smax;
4545          op64 = aco_opcode::buffer_atomic_smax_x2;
4546          break;
4547       case nir_intrinsic_ssbo_atomic_umax:
4548          op32 = aco_opcode::buffer_atomic_umax;
4549          op64 = aco_opcode::buffer_atomic_umax_x2;
4550          break;
4551       case nir_intrinsic_ssbo_atomic_and:
4552          op32 = aco_opcode::buffer_atomic_and;
4553          op64 = aco_opcode::buffer_atomic_and_x2;
4554          break;
4555       case nir_intrinsic_ssbo_atomic_or:
4556          op32 = aco_opcode::buffer_atomic_or;
4557          op64 = aco_opcode::buffer_atomic_or_x2;
4558          break;
4559       case nir_intrinsic_ssbo_atomic_xor:
4560          op32 = aco_opcode::buffer_atomic_xor;
4561          op64 = aco_opcode::buffer_atomic_xor_x2;
4562          break;
4563       case nir_intrinsic_ssbo_atomic_exchange:
4564          op32 = aco_opcode::buffer_atomic_swap;
4565          op64 = aco_opcode::buffer_atomic_swap_x2;
4566          break;
4567       case nir_intrinsic_ssbo_atomic_comp_swap:
4568          op32 = aco_opcode::buffer_atomic_cmpswap;
4569          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4570          break;
4571       default:
4572          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4573    }
4574    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4575    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4576    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4577    mubuf->operands[1] = Operand(rsrc);
4578    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4579    mubuf->operands[3] = Operand(data);
4580    if (return_previous)
4581       mubuf->definitions[0] = Definition(dst);
4582    mubuf->offset = 0;
4583    mubuf->offen = (offset.type() == RegType::vgpr);
4584    mubuf->glc = return_previous;
4585    mubuf->dlc = false; /* Not needed for atomics */
4586    mubuf->disable_wqm = true;
4587    mubuf->barrier = barrier_buffer;
4588    ctx->program->needs_exact = true;
4589    ctx->block->instructions.emplace_back(std::move(mubuf));
4590 }
4591
4592 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4593
4594    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4595    Builder bld(ctx->program, ctx->block);
4596    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4597    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4598 }
4599
4600 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4601 {
4602    Builder bld(ctx->program, ctx->block);
4603    unsigned num_components = instr->num_components;
4604    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4605
4606    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4607    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4608
4609    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4610    bool dlc = glc && ctx->options->chip_class >= GFX10;
4611    aco_opcode op;
4612    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4613       bool global = ctx->options->chip_class >= GFX9;
4614       aco_opcode op;
4615       switch (num_bytes) {
4616       case 4:
4617          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4618          break;
4619       case 8:
4620          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4621          break;
4622       case 12:
4623          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4624          break;
4625       case 16:
4626          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4627          break;
4628       default:
4629          unreachable("load_global not implemented for this size.");
4630       }
4631       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4632       flat->operands[0] = Operand(addr);
4633       flat->operands[1] = Operand(s1);
4634       flat->glc = glc;
4635       flat->dlc = dlc;
4636
4637       if (dst.type() == RegType::sgpr) {
4638          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4639          flat->definitions[0] = Definition(vec);
4640          ctx->block->instructions.emplace_back(std::move(flat));
4641          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4642       } else {
4643          flat->definitions[0] = Definition(dst);
4644          ctx->block->instructions.emplace_back(std::move(flat));
4645       }
4646       emit_split_vector(ctx, dst, num_components);
4647    } else {
4648       switch (num_bytes) {
4649          case 4:
4650             op = aco_opcode::s_load_dword;
4651             break;
4652          case 8:
4653             op = aco_opcode::s_load_dwordx2;
4654             break;
4655          case 12:
4656          case 16:
4657             op = aco_opcode::s_load_dwordx4;
4658             break;
4659          default:
4660             unreachable("load_global not implemented for this size.");
4661       }
4662       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4663       load->operands[0] = Operand(addr);
4664       load->operands[1] = Operand(0u);
4665       load->definitions[0] = Definition(dst);
4666       load->glc = glc;
4667       load->dlc = dlc;
4668       load->barrier = barrier_buffer;
4669       assert(ctx->options->chip_class >= GFX8 || !glc);
4670
4671       if (dst.size() == 3) {
4672          /* trim vector */
4673          Temp vec = bld.tmp(s4);
4674          load->definitions[0] = Definition(vec);
4675          ctx->block->instructions.emplace_back(std::move(load));
4676          emit_split_vector(ctx, vec, 4);
4677
4678          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4679                     emit_extract_vector(ctx, vec, 0, s1),
4680                     emit_extract_vector(ctx, vec, 1, s1),
4681                     emit_extract_vector(ctx, vec, 2, s1));
4682       } else {
4683          ctx->block->instructions.emplace_back(std::move(load));
4684       }
4685    }
4686 }
4687
4688 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4689 {
4690    Builder bld(ctx->program, ctx->block);
4691    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4692
4693    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4694    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4695
4696    unsigned writemask = nir_intrinsic_write_mask(instr);
4697    while (writemask) {
4698       int start, count;
4699       u_bit_scan_consecutive_range(&writemask, &start, &count);
4700       unsigned num_bytes = count * elem_size_bytes;
4701
4702       Temp write_data = data;
4703       if (count != instr->num_components) {
4704          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4705          for (int i = 0; i < count; i++)
4706             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4707          write_data = bld.tmp(RegType::vgpr, count);
4708          vec->definitions[0] = Definition(write_data);
4709          ctx->block->instructions.emplace_back(std::move(vec));
4710       }
4711
4712       unsigned offset = start * elem_size_bytes;
4713       if (offset > 0 && ctx->options->chip_class < GFX9) {
4714          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4715          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4716          Temp carry = bld.tmp(s2);
4717          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4718
4719          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4720                   Operand(offset), addr0);
4721          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4722                   Operand(0u), addr1,
4723                   carry).def(1).setHint(vcc);
4724
4725          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4726
4727          offset = 0;
4728       }
4729
4730       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4731       bool global = ctx->options->chip_class >= GFX9;
4732       aco_opcode op;
4733       switch (num_bytes) {
4734       case 4:
4735          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4736          break;
4737       case 8:
4738          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4739          break;
4740       case 12:
4741          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4742          break;
4743       case 16:
4744          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4745          break;
4746       default:
4747          unreachable("store_global not implemented for this size.");
4748       }
4749       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4750       flat->operands[0] = Operand(addr);
4751       flat->operands[1] = Operand(s1);
4752       flat->operands[2] = Operand(data);
4753       flat->glc = glc;
4754       flat->dlc = false;
4755       flat->offset = offset;
4756       ctx->block->instructions.emplace_back(std::move(flat));
4757    }
4758 }
4759
4760 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4761    Builder bld(ctx->program, ctx->block);
4762    switch(instr->intrinsic) {
4763       case nir_intrinsic_group_memory_barrier:
4764       case nir_intrinsic_memory_barrier:
4765          bld.barrier(aco_opcode::p_memory_barrier_all);
4766          break;
4767       case nir_intrinsic_memory_barrier_atomic_counter:
4768          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4769          break;
4770       case nir_intrinsic_memory_barrier_buffer:
4771          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4772          break;
4773       case nir_intrinsic_memory_barrier_image:
4774          bld.barrier(aco_opcode::p_memory_barrier_image);
4775          break;
4776       case nir_intrinsic_memory_barrier_shared:
4777          bld.barrier(aco_opcode::p_memory_barrier_shared);
4778          break;
4779       default:
4780          unreachable("Unimplemented memory barrier intrinsic");
4781          break;
4782    }
4783 }
4784
4785 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4786 {
4787    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4788    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4789    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4790    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4791    Builder bld(ctx->program, ctx->block);
4792
4793    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4794    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4795    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4796 }
4797
4798 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4799 {
4800    unsigned writemask = nir_intrinsic_write_mask(instr);
4801    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4802    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4803    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4804    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4805
4806    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4807    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4808 }
4809
4810 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4811 {
4812    unsigned offset = nir_intrinsic_base(instr);
4813    Operand m = load_lds_size_m0(ctx);
4814    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4815    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4816
4817    unsigned num_operands = 3;
4818    aco_opcode op32, op64, op32_rtn, op64_rtn;
4819    switch(instr->intrinsic) {
4820       case nir_intrinsic_shared_atomic_add:
4821          op32 = aco_opcode::ds_add_u32;
4822          op64 = aco_opcode::ds_add_u64;
4823          op32_rtn = aco_opcode::ds_add_rtn_u32;
4824          op64_rtn = aco_opcode::ds_add_rtn_u64;
4825          break;
4826       case nir_intrinsic_shared_atomic_imin:
4827          op32 = aco_opcode::ds_min_i32;
4828          op64 = aco_opcode::ds_min_i64;
4829          op32_rtn = aco_opcode::ds_min_rtn_i32;
4830          op64_rtn = aco_opcode::ds_min_rtn_i64;
4831          break;
4832       case nir_intrinsic_shared_atomic_umin:
4833          op32 = aco_opcode::ds_min_u32;
4834          op64 = aco_opcode::ds_min_u64;
4835          op32_rtn = aco_opcode::ds_min_rtn_u32;
4836          op64_rtn = aco_opcode::ds_min_rtn_u64;
4837          break;
4838       case nir_intrinsic_shared_atomic_imax:
4839          op32 = aco_opcode::ds_max_i32;
4840          op64 = aco_opcode::ds_max_i64;
4841          op32_rtn = aco_opcode::ds_max_rtn_i32;
4842          op64_rtn = aco_opcode::ds_max_rtn_i64;
4843          break;
4844       case nir_intrinsic_shared_atomic_umax:
4845          op32 = aco_opcode::ds_max_u32;
4846          op64 = aco_opcode::ds_max_u64;
4847          op32_rtn = aco_opcode::ds_max_rtn_u32;
4848          op64_rtn = aco_opcode::ds_max_rtn_u64;
4849          break;
4850       case nir_intrinsic_shared_atomic_and:
4851          op32 = aco_opcode::ds_and_b32;
4852          op64 = aco_opcode::ds_and_b64;
4853          op32_rtn = aco_opcode::ds_and_rtn_b32;
4854          op64_rtn = aco_opcode::ds_and_rtn_b64;
4855          break;
4856       case nir_intrinsic_shared_atomic_or:
4857          op32 = aco_opcode::ds_or_b32;
4858          op64 = aco_opcode::ds_or_b64;
4859          op32_rtn = aco_opcode::ds_or_rtn_b32;
4860          op64_rtn = aco_opcode::ds_or_rtn_b64;
4861          break;
4862       case nir_intrinsic_shared_atomic_xor:
4863          op32 = aco_opcode::ds_xor_b32;
4864          op64 = aco_opcode::ds_xor_b64;
4865          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4866          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4867          break;
4868       case nir_intrinsic_shared_atomic_exchange:
4869          op32 = aco_opcode::ds_write_b32;
4870          op64 = aco_opcode::ds_write_b64;
4871          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4872          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4873          break;
4874       case nir_intrinsic_shared_atomic_comp_swap:
4875          op32 = aco_opcode::ds_cmpst_b32;
4876          op64 = aco_opcode::ds_cmpst_b64;
4877          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4878          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4879          num_operands = 4;
4880          break;
4881       default:
4882          unreachable("Unhandled shared atomic intrinsic");
4883    }
4884
4885    /* return the previous value if dest is ever used */
4886    bool return_previous = false;
4887    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4888       return_previous = true;
4889       break;
4890    }
4891    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4892       return_previous = true;
4893       break;
4894    }
4895
4896    aco_opcode op;
4897    if (data.size() == 1) {
4898       assert(instr->dest.ssa.bit_size == 32);
4899       op = return_previous ? op32_rtn : op32;
4900    } else {
4901       assert(instr->dest.ssa.bit_size == 64);
4902       op = return_previous ? op64_rtn : op64;
4903    }
4904
4905    if (offset > 65535) {
4906       Builder bld(ctx->program, ctx->block);
4907       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4908       offset = 0;
4909    }
4910
4911    aco_ptr<DS_instruction> ds;
4912    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4913    ds->operands[0] = Operand(address);
4914    ds->operands[1] = Operand(data);
4915    if (num_operands == 4)
4916       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4917    ds->operands[num_operands - 1] = m;
4918    ds->offset0 = offset;
4919    if (return_previous)
4920       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4921    ctx->block->instructions.emplace_back(std::move(ds));
4922 }
4923
4924 Temp get_scratch_resource(isel_context *ctx)
4925 {
4926    Builder bld(ctx->program, ctx->block);
4927    Temp scratch_addr = ctx->program->private_segment_buffer;
4928    if (ctx->stage != compute_cs)
4929       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
4930
4931    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
4932                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
4933
4934    if (ctx->program->chip_class >= GFX10) {
4935       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4936                    S_008F0C_OOB_SELECT(3) |
4937                    S_008F0C_RESOURCE_LEVEL(1);
4938    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
4939       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4940                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4941    }
4942
4943    /* older generations need element size = 16 bytes. element size removed in GFX9 */
4944    if (ctx->program->chip_class <= GFX8)
4945       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
4946
4947    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4948 }
4949
4950 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4951    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4952    Builder bld(ctx->program, ctx->block);
4953    Temp rsrc = get_scratch_resource(ctx);
4954    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4955    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4956
4957    aco_opcode op;
4958    switch (dst.size()) {
4959       case 1:
4960          op = aco_opcode::buffer_load_dword;
4961          break;
4962       case 2:
4963          op = aco_opcode::buffer_load_dwordx2;
4964          break;
4965       case 3:
4966          op = aco_opcode::buffer_load_dwordx3;
4967          break;
4968       case 4:
4969          op = aco_opcode::buffer_load_dwordx4;
4970          break;
4971       case 6:
4972       case 8: {
4973          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4974          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4975                                 bld.def(v4), offset, rsrc,
4976                                 ctx->program->scratch_offset, 0, true);
4977          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4978                                                   aco_opcode::buffer_load_dwordx4,
4979                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4980                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
4981          emit_split_vector(ctx, lower, 2);
4982          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4983          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4984          if (dst.size() == 8) {
4985             emit_split_vector(ctx, upper, 2);
4986             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4987             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4988          } else {
4989             elems[2] = upper;
4990          }
4991
4992          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4993                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4994          for (unsigned i = 0; i < dst.size() / 2; i++)
4995             vec->operands[i] = Operand(elems[i]);
4996          vec->definitions[0] = Definition(dst);
4997          bld.insert(std::move(vec));
4998          ctx->allocated_vec.emplace(dst.id(), elems);
4999          return;
5000       }
5001       default:
5002          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5003    }
5004
5005    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5006    emit_split_vector(ctx, dst, instr->num_components);
5007 }
5008
5009 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5010    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5011    Builder bld(ctx->program, ctx->block);
5012    Temp rsrc = get_scratch_resource(ctx);
5013    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5014    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5015
5016    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5017    unsigned writemask = nir_intrinsic_write_mask(instr);
5018
5019    while (writemask) {
5020       int start, count;
5021       u_bit_scan_consecutive_range(&writemask, &start, &count);
5022       int num_bytes = count * elem_size_bytes;
5023
5024       if (num_bytes > 16) {
5025          assert(elem_size_bytes == 8);
5026          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5027          count = 2;
5028          num_bytes = 16;
5029       }
5030
5031       // TODO: check alignment of sub-dword stores
5032       // TODO: split 3 bytes. there is no store instruction for that
5033
5034       Temp write_data;
5035       if (count != instr->num_components) {
5036          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5037          for (int i = 0; i < count; i++) {
5038             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5039             vec->operands[i] = Operand(elem);
5040          }
5041          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5042          vec->definitions[0] = Definition(write_data);
5043          ctx->block->instructions.emplace_back(std::move(vec));
5044       } else {
5045          write_data = data;
5046       }
5047
5048       aco_opcode op;
5049       switch (num_bytes) {
5050          case 4:
5051             op = aco_opcode::buffer_store_dword;
5052             break;
5053          case 8:
5054             op = aco_opcode::buffer_store_dwordx2;
5055             break;
5056          case 12:
5057             op = aco_opcode::buffer_store_dwordx3;
5058             break;
5059          case 16:
5060             op = aco_opcode::buffer_store_dwordx4;
5061             break;
5062          default:
5063             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5064       }
5065
5066       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5067    }
5068 }
5069
5070 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5071    uint8_t log2_ps_iter_samples;
5072    if (ctx->program->info->ps.force_persample) {
5073       log2_ps_iter_samples =
5074          util_logbase2(ctx->options->key.fs.num_samples);
5075    } else {
5076       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5077    }
5078
5079    /* The bit pattern matches that used by fixed function fragment
5080     * processing. */
5081    static const unsigned ps_iter_masks[] = {
5082       0xffff, /* not used */
5083       0x5555,
5084       0x1111,
5085       0x0101,
5086       0x0001,
5087    };
5088    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5089
5090    Builder bld(ctx->program, ctx->block);
5091
5092    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
5093    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5094    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5095    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5096    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
5097 }
5098
5099 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5100 {
5101    Builder bld(ctx->program, ctx->block);
5102
5103    if (cluster_size == 1) {
5104       return src;
5105    } if (op == nir_op_iand && cluster_size == 4) {
5106       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5107       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5108       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
5109                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
5110    } else if (op == nir_op_ior && cluster_size == 4) {
5111       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5112       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
5113                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
5114    } else if (op == nir_op_iand && cluster_size == 64) {
5115       //subgroupAnd(val) -> (exec & ~val) == 0
5116       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5117       return bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), tmp, Operand(0u));
5118    } else if (op == nir_op_ior && cluster_size == 64) {
5119       //subgroupOr(val) -> (val & exec) != 0
5120       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
5121    } else if (op == nir_op_ixor && cluster_size == 64) {
5122       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5123       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5124       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
5125       return bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5126    } else {
5127       //subgroupClustered{And,Or,Xor}(val, n) ->
5128       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5129       //cluster_offset = ~(n - 1) & lane_id
5130       //cluster_mask = ((1 << n) - 1)
5131       //subgroupClusteredAnd():
5132       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5133       //subgroupClusteredOr():
5134       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5135       //subgroupClusteredXor():
5136       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5137       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5138                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5139       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5140
5141       Temp tmp;
5142       if (op == nir_op_iand)
5143          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5144       else
5145          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5146
5147       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5148       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5149       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5150       if (cluster_mask != 0xffffffff)
5151          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5152
5153       Definition cmp_def = Definition();
5154       if (op == nir_op_iand) {
5155          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5156       } else if (op == nir_op_ior) {
5157          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5158       } else if (op == nir_op_ixor) {
5159          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5160                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5161          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5162       }
5163       cmp_def.setHint(vcc);
5164       return cmp_def.getTemp();
5165    }
5166 }
5167
5168 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5169 {
5170    Builder bld(ctx->program, ctx->block);
5171
5172    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5173    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5174    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5175    Temp tmp;
5176    if (op == nir_op_iand)
5177       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5178    else
5179       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5180
5181    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5182    Temp lo = lohi.def(0).getTemp();
5183    Temp hi = lohi.def(1).getTemp();
5184    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5185                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5186
5187    Definition cmp_def = Definition();
5188    if (op == nir_op_iand)
5189       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5190    else if (op == nir_op_ior)
5191       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5192    else if (op == nir_op_ixor)
5193       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5194                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5195    cmp_def.setHint(vcc);
5196    return cmp_def.getTemp();
5197 }
5198
5199 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5200 {
5201    Builder bld(ctx->program, ctx->block);
5202
5203    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5204    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5205    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5206    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5207    if (op == nir_op_iand)
5208       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5209    else if (op == nir_op_ior)
5210       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5211    else if (op == nir_op_ixor)
5212       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5213
5214    assert(false);
5215    return Temp();
5216 }
5217
5218 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5219 {
5220    Builder bld(ctx->program, ctx->block);
5221    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5222    if (src.regClass().type() == RegType::vgpr) {
5223       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5224    } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5225       bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(dst), Operand(0u), Operand(src));
5226    } else if (src.regClass() == s1) {
5227       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5228    } else if (src.regClass() == s2) {
5229       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5230    } else {
5231       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5232       nir_print_instr(&instr->instr, stderr);
5233       fprintf(stderr, "\n");
5234    }
5235 }
5236
5237 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5238 {
5239    Builder bld(ctx->program, ctx->block);
5240    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5241    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5242
5243    /* Build DD X/Y */
5244    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5245    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5246    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5247    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5248    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5249    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5250
5251    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5252    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5253    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5254    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5255    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5256    Temp wqm1 = bld.tmp(v1);
5257    emit_wqm(ctx, tmp1, wqm1, true);
5258    Temp wqm2 = bld.tmp(v1);
5259    emit_wqm(ctx, tmp2, wqm2, true);
5260    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5261    return;
5262 }
5263
5264 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5265 {
5266    Builder bld(ctx->program, ctx->block);
5267    switch(instr->intrinsic) {
5268    case nir_intrinsic_load_barycentric_sample:
5269    case nir_intrinsic_load_barycentric_pixel:
5270    case nir_intrinsic_load_barycentric_centroid: {
5271       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5272       fs_input input = get_interp_input(instr->intrinsic, mode);
5273
5274       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5275       if (input == fs_input::max_inputs) {
5276          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5277                     Operand(0u), Operand(0u));
5278       } else {
5279          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5280                     ctx->fs_inputs[input],
5281                     ctx->fs_inputs[input + 1]);
5282       }
5283       emit_split_vector(ctx, dst, 2);
5284       break;
5285    }
5286    case nir_intrinsic_load_barycentric_at_sample: {
5287       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5288       switch (ctx->options->key.fs.num_samples) {
5289          case 2: sample_pos_offset += 1 << 3; break;
5290          case 4: sample_pos_offset += 3 << 3; break;
5291          case 8: sample_pos_offset += 7 << 3; break;
5292          default: break;
5293       }
5294       Temp sample_pos;
5295       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5296       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5297       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5298       if (addr.type() == RegType::sgpr) {
5299          Operand offset;
5300          if (const_addr) {
5301             sample_pos_offset += const_addr->u32 << 3;
5302             offset = Operand(sample_pos_offset);
5303          } else if (ctx->options->chip_class >= GFX9) {
5304             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5305          } else {
5306             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5307             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5308          }
5309          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5310
5311       } else if (ctx->options->chip_class >= GFX9) {
5312          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5313          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5314       } else {
5315          /* addr += private_segment_buffer + sample_pos_offset */
5316          Temp tmp0 = bld.tmp(s1);
5317          Temp tmp1 = bld.tmp(s1);
5318          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5319          Definition scc_tmp = bld.def(s1, scc);
5320          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5321          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5322          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5323          Temp pck0 = bld.tmp(v1);
5324          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5325          tmp1 = as_vgpr(ctx, tmp1);
5326          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5327          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5328
5329          /* sample_pos = flat_load_dwordx2 addr */
5330          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5331       }
5332
5333       /* sample_pos -= 0.5 */
5334       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5335       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5336       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5337       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5338       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5339
5340       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5341       break;
5342    }
5343    case nir_intrinsic_load_barycentric_at_offset: {
5344       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5345       RegClass rc = RegClass(offset.type(), 1);
5346       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5347       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5348       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5349       break;
5350    }
5351    case nir_intrinsic_load_front_face: {
5352       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5353                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5354       break;
5355    }
5356    case nir_intrinsic_load_view_index:
5357    case nir_intrinsic_load_layer_id: {
5358       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5359          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5360          bld.copy(Definition(dst), Operand(ctx->view_index));
5361          break;
5362       }
5363
5364       unsigned idx = nir_intrinsic_base(instr);
5365       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5366                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5367       break;
5368    }
5369    case nir_intrinsic_load_frag_coord: {
5370       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5371       break;
5372    }
5373    case nir_intrinsic_load_sample_pos: {
5374       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5375       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5376       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5377                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5378                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5379       break;
5380    }
5381    case nir_intrinsic_load_interpolated_input:
5382       visit_load_interpolated_input(ctx, instr);
5383       break;
5384    case nir_intrinsic_store_output:
5385       visit_store_output(ctx, instr);
5386       break;
5387    case nir_intrinsic_load_input:
5388       visit_load_input(ctx, instr);
5389       break;
5390    case nir_intrinsic_load_ubo:
5391       visit_load_ubo(ctx, instr);
5392       break;
5393    case nir_intrinsic_load_push_constant:
5394       visit_load_push_constant(ctx, instr);
5395       break;
5396    case nir_intrinsic_load_constant:
5397       visit_load_constant(ctx, instr);
5398       break;
5399    case nir_intrinsic_vulkan_resource_index:
5400       visit_load_resource(ctx, instr);
5401       break;
5402    case nir_intrinsic_discard:
5403       visit_discard(ctx, instr);
5404       break;
5405    case nir_intrinsic_discard_if:
5406       visit_discard_if(ctx, instr);
5407       break;
5408    case nir_intrinsic_load_shared:
5409       visit_load_shared(ctx, instr);
5410       break;
5411    case nir_intrinsic_store_shared:
5412       visit_store_shared(ctx, instr);
5413       break;
5414    case nir_intrinsic_shared_atomic_add:
5415    case nir_intrinsic_shared_atomic_imin:
5416    case nir_intrinsic_shared_atomic_umin:
5417    case nir_intrinsic_shared_atomic_imax:
5418    case nir_intrinsic_shared_atomic_umax:
5419    case nir_intrinsic_shared_atomic_and:
5420    case nir_intrinsic_shared_atomic_or:
5421    case nir_intrinsic_shared_atomic_xor:
5422    case nir_intrinsic_shared_atomic_exchange:
5423    case nir_intrinsic_shared_atomic_comp_swap:
5424       visit_shared_atomic(ctx, instr);
5425       break;
5426    case nir_intrinsic_image_deref_load:
5427       visit_image_load(ctx, instr);
5428       break;
5429    case nir_intrinsic_image_deref_store:
5430       visit_image_store(ctx, instr);
5431       break;
5432    case nir_intrinsic_image_deref_atomic_add:
5433    case nir_intrinsic_image_deref_atomic_umin:
5434    case nir_intrinsic_image_deref_atomic_imin:
5435    case nir_intrinsic_image_deref_atomic_umax:
5436    case nir_intrinsic_image_deref_atomic_imax:
5437    case nir_intrinsic_image_deref_atomic_and:
5438    case nir_intrinsic_image_deref_atomic_or:
5439    case nir_intrinsic_image_deref_atomic_xor:
5440    case nir_intrinsic_image_deref_atomic_exchange:
5441    case nir_intrinsic_image_deref_atomic_comp_swap:
5442       visit_image_atomic(ctx, instr);
5443       break;
5444    case nir_intrinsic_image_deref_size:
5445       visit_image_size(ctx, instr);
5446       break;
5447    case nir_intrinsic_load_ssbo:
5448       visit_load_ssbo(ctx, instr);
5449       break;
5450    case nir_intrinsic_store_ssbo:
5451       visit_store_ssbo(ctx, instr);
5452       break;
5453    case nir_intrinsic_load_global:
5454       visit_load_global(ctx, instr);
5455       break;
5456    case nir_intrinsic_store_global:
5457       visit_store_global(ctx, instr);
5458       break;
5459    case nir_intrinsic_ssbo_atomic_add:
5460    case nir_intrinsic_ssbo_atomic_imin:
5461    case nir_intrinsic_ssbo_atomic_umin:
5462    case nir_intrinsic_ssbo_atomic_imax:
5463    case nir_intrinsic_ssbo_atomic_umax:
5464    case nir_intrinsic_ssbo_atomic_and:
5465    case nir_intrinsic_ssbo_atomic_or:
5466    case nir_intrinsic_ssbo_atomic_xor:
5467    case nir_intrinsic_ssbo_atomic_exchange:
5468    case nir_intrinsic_ssbo_atomic_comp_swap:
5469       visit_atomic_ssbo(ctx, instr);
5470       break;
5471    case nir_intrinsic_load_scratch:
5472       visit_load_scratch(ctx, instr);
5473       break;
5474    case nir_intrinsic_store_scratch:
5475       visit_store_scratch(ctx, instr);
5476       break;
5477    case nir_intrinsic_get_buffer_size:
5478       visit_get_buffer_size(ctx, instr);
5479       break;
5480    case nir_intrinsic_barrier: {
5481       unsigned* bsize = ctx->program->info->cs.block_size;
5482       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5483       if (workgroup_size > 64)
5484          bld.sopp(aco_opcode::s_barrier);
5485       break;
5486    }
5487    case nir_intrinsic_group_memory_barrier:
5488    case nir_intrinsic_memory_barrier:
5489    case nir_intrinsic_memory_barrier_atomic_counter:
5490    case nir_intrinsic_memory_barrier_buffer:
5491    case nir_intrinsic_memory_barrier_image:
5492    case nir_intrinsic_memory_barrier_shared:
5493       emit_memory_barrier(ctx, instr);
5494       break;
5495    case nir_intrinsic_load_num_work_groups:
5496    case nir_intrinsic_load_work_group_id:
5497    case nir_intrinsic_load_local_invocation_id: {
5498       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5499       Temp* ids;
5500       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5501          ids = ctx->num_workgroups;
5502       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5503          ids = ctx->workgroup_ids;
5504       else
5505          ids = ctx->local_invocation_ids;
5506       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5507                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5508                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5509                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5510       emit_split_vector(ctx, dst, 3);
5511       break;
5512    }
5513    case nir_intrinsic_load_local_invocation_index: {
5514       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5515                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5516       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5517       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5518       break;
5519    }
5520    case nir_intrinsic_load_subgroup_id: {
5521       if (ctx->stage == compute_cs) {
5522          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5523          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5524       } else {
5525          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5526       }
5527       break;
5528    }
5529    case nir_intrinsic_load_subgroup_invocation: {
5530       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5531                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5532       break;
5533    }
5534    case nir_intrinsic_load_num_subgroups: {
5535       if (ctx->stage == compute_cs)
5536          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5537       else
5538          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5539       break;
5540    }
5541    case nir_intrinsic_ballot: {
5542       Definition tmp = bld.def(s2);
5543       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5544       if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s2) {
5545          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5546       } else if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s1) {
5547          bld.sop2(aco_opcode::s_cselect_b64, tmp, Operand(exec, s2), Operand(0u), bld.scc(src));
5548       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5549          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5550       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5551          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5552       } else {
5553          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5554          nir_print_instr(&instr->instr, stderr);
5555          fprintf(stderr, "\n");
5556       }
5557       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5558       break;
5559    }
5560    case nir_intrinsic_shuffle:
5561    case nir_intrinsic_read_invocation: {
5562       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5563       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5564          emit_uniform_subgroup(ctx, instr, src);
5565       } else {
5566          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5567          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5568             tid = bld.as_uniform(tid);
5569          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5570          if (src.regClass() == v1) {
5571             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5572          } else if (src.regClass() == v2) {
5573             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5574             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5575             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5576             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5577             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5578             emit_split_vector(ctx, dst, 2);
5579          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2 && tid.regClass() == s1) {
5580             emit_wqm(ctx, bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, tid), dst);
5581          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5582             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5583             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5584             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5585             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5586          } else {
5587             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5588             nir_print_instr(&instr->instr, stderr);
5589             fprintf(stderr, "\n");
5590          }
5591       }
5592       break;
5593    }
5594    case nir_intrinsic_load_sample_id: {
5595       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5596                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5597       break;
5598    }
5599    case nir_intrinsic_load_sample_mask_in: {
5600       visit_load_sample_mask_in(ctx, instr);
5601       break;
5602    }
5603    case nir_intrinsic_read_first_invocation: {
5604       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5605       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5606       if (src.regClass() == v1) {
5607          emit_wqm(ctx,
5608                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5609                   dst);
5610       } else if (src.regClass() == v2) {
5611          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5612          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5613          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5614          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5615          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5616          emit_split_vector(ctx, dst, 2);
5617       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5618          emit_wqm(ctx,
5619                   bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5620                            bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2))),
5621                   dst);
5622       } else if (src.regClass() == s1) {
5623          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5624       } else if (src.regClass() == s2) {
5625          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5626       } else {
5627          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5628          nir_print_instr(&instr->instr, stderr);
5629          fprintf(stderr, "\n");
5630       }
5631       break;
5632    }
5633    case nir_intrinsic_vote_all: {
5634       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5635       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5636       assert(src.regClass() == s2);
5637       assert(dst.regClass() == s1);
5638
5639       Definition tmp = bld.def(s1);
5640       bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(tmp),
5641                bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)),
5642                Operand(exec, s2));
5643       emit_wqm(ctx, tmp.getTemp(), dst);
5644       break;
5645    }
5646    case nir_intrinsic_vote_any: {
5647       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5648       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5649       assert(src.regClass() == s2);
5650       assert(dst.regClass() == s1);
5651
5652       Definition tmp = bld.def(s1);
5653       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(tmp), src, Operand(exec, s2));
5654       emit_wqm(ctx, tmp.getTemp(), dst);
5655       break;
5656    }
5657    case nir_intrinsic_reduce:
5658    case nir_intrinsic_inclusive_scan:
5659    case nir_intrinsic_exclusive_scan: {
5660       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5661       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5662       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5663       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5664          nir_intrinsic_cluster_size(instr) : 0;
5665       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5666
5667       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5668          emit_uniform_subgroup(ctx, instr, src);
5669       } else if (instr->dest.ssa.bit_size == 1) {
5670          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5671             op = nir_op_iand;
5672          else if (op == nir_op_iadd)
5673             op = nir_op_ixor;
5674          else if (op == nir_op_umax || op == nir_op_imax)
5675             op = nir_op_ior;
5676          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5677
5678          switch (instr->intrinsic) {
5679          case nir_intrinsic_reduce:
5680             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5681             break;
5682          case nir_intrinsic_exclusive_scan:
5683             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5684             break;
5685          case nir_intrinsic_inclusive_scan:
5686             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5687             break;
5688          default:
5689             assert(false);
5690          }
5691       } else if (cluster_size == 1) {
5692          bld.copy(Definition(dst), src);
5693       } else {
5694          src = as_vgpr(ctx, src);
5695
5696          ReduceOp reduce_op;
5697          switch (op) {
5698          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5699             CASE(iadd)
5700             CASE(imul)
5701             CASE(fadd)
5702             CASE(fmul)
5703             CASE(imin)
5704             CASE(umin)
5705             CASE(fmin)
5706             CASE(imax)
5707             CASE(umax)
5708             CASE(fmax)
5709             CASE(iand)
5710             CASE(ior)
5711             CASE(ixor)
5712             default:
5713                unreachable("unknown reduction op");
5714          #undef CASE
5715          }
5716
5717          aco_opcode aco_op;
5718          switch (instr->intrinsic) {
5719             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5720             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5721             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5722             default:
5723                unreachable("unknown reduce intrinsic");
5724          }
5725
5726          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5727          reduce->operands[0] = Operand(src);
5728          // filled in by aco_reduce_assign.cpp, used internally as part of the
5729          // reduce sequence
5730          assert(dst.size() == 1 || dst.size() == 2);
5731          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5732          reduce->operands[2] = Operand(v1.as_linear());
5733
5734          Temp tmp_dst = bld.tmp(dst.regClass());
5735          reduce->definitions[0] = Definition(tmp_dst);
5736          reduce->definitions[1] = bld.def(s2); // used internally
5737          reduce->definitions[2] = Definition();
5738          reduce->definitions[3] = Definition(scc, s1);
5739          reduce->definitions[4] = Definition();
5740          reduce->reduce_op = reduce_op;
5741          reduce->cluster_size = cluster_size;
5742          ctx->block->instructions.emplace_back(std::move(reduce));
5743
5744          emit_wqm(ctx, tmp_dst, dst);
5745       }
5746       break;
5747    }
5748    case nir_intrinsic_quad_broadcast: {
5749       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5750       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5751          emit_uniform_subgroup(ctx, instr, src);
5752       } else {
5753          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5754          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5755          if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5756             uint32_t half_mask = 0x11111111u << lane;
5757             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5758             Temp tmp = bld.tmp(s2);
5759             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5760                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5761                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5762             emit_wqm(ctx, tmp, dst);
5763          } else if (instr->dest.ssa.bit_size == 32) {
5764             emit_wqm(ctx,
5765                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5766                                   dpp_quad_perm(lane, lane, lane, lane)),
5767                      dst);
5768          } else if (instr->dest.ssa.bit_size == 64) {
5769             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5770             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5771             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5772             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5773             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5774             emit_split_vector(ctx, dst, 2);
5775          } else {
5776             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5777             nir_print_instr(&instr->instr, stderr);
5778             fprintf(stderr, "\n");
5779          }
5780       }
5781       break;
5782    }
5783    case nir_intrinsic_quad_swap_horizontal:
5784    case nir_intrinsic_quad_swap_vertical:
5785    case nir_intrinsic_quad_swap_diagonal:
5786    case nir_intrinsic_quad_swizzle_amd: {
5787       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5788       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5789          emit_uniform_subgroup(ctx, instr, src);
5790          break;
5791       }
5792       uint16_t dpp_ctrl = 0;
5793       switch (instr->intrinsic) {
5794       case nir_intrinsic_quad_swap_horizontal:
5795          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5796          break;
5797       case nir_intrinsic_quad_swap_vertical:
5798          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5799          break;
5800       case nir_intrinsic_quad_swap_diagonal:
5801          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5802          break;
5803       case nir_intrinsic_quad_swizzle_amd: {
5804          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5805          break;
5806       }
5807       default:
5808          break;
5809       }
5810
5811       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5812       if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5813          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5814          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5815          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5816          emit_wqm(ctx, tmp, dst);
5817       } else if (instr->dest.ssa.bit_size == 32) {
5818          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5819          emit_wqm(ctx, tmp, dst);
5820       } else if (instr->dest.ssa.bit_size == 64) {
5821          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5822          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5823          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5824          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5825          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5826          emit_split_vector(ctx, dst, 2);
5827       } else {
5828          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5829          nir_print_instr(&instr->instr, stderr);
5830          fprintf(stderr, "\n");
5831       }
5832       break;
5833    }
5834    case nir_intrinsic_masked_swizzle_amd: {
5835       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5836       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5837          emit_uniform_subgroup(ctx, instr, src);
5838          break;
5839       }
5840       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5841       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5842       if (dst.regClass() == v1) {
5843          emit_wqm(ctx,
5844                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5845                   dst);
5846       } else if (dst.regClass() == v2) {
5847          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5848          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5849          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5850          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5851          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5852          emit_split_vector(ctx, dst, 2);
5853       } else {
5854          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5855          nir_print_instr(&instr->instr, stderr);
5856          fprintf(stderr, "\n");
5857       }
5858       break;
5859    }
5860    case nir_intrinsic_write_invocation_amd: {
5861       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5862       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5863       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5864       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5865       if (dst.regClass() == v1) {
5866          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5867          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5868       } else if (dst.regClass() == v2) {
5869          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5870          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5871          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5872          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5873          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5874          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5875          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5876          emit_split_vector(ctx, dst, 2);
5877       } else {
5878          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5879          nir_print_instr(&instr->instr, stderr);
5880          fprintf(stderr, "\n");
5881       }
5882       break;
5883    }
5884    case nir_intrinsic_mbcnt_amd: {
5885       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5886       RegClass rc = RegClass(src.type(), 1);
5887       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5888       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5889       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5890       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5891       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5892       emit_wqm(ctx, wqm_tmp, dst);
5893       break;
5894    }
5895    case nir_intrinsic_load_helper_invocation: {
5896       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5897       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5898       ctx->block->kind |= block_kind_needs_lowering;
5899       ctx->program->needs_exact = true;
5900       break;
5901    }
5902    case nir_intrinsic_is_helper_invocation: {
5903       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5904       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5905       ctx->block->kind |= block_kind_needs_lowering;
5906       ctx->program->needs_exact = true;
5907       break;
5908    }
5909    case nir_intrinsic_demote:
5910       bld.pseudo(aco_opcode::p_demote_to_helper);
5911       ctx->block->kind |= block_kind_uses_demote;
5912       ctx->program->needs_exact = true;
5913       break;
5914    case nir_intrinsic_demote_if: {
5915       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc),
5916                            as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false),
5917                            Operand(exec, s2));
5918       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5919       ctx->block->kind |= block_kind_uses_demote;
5920       ctx->program->needs_exact = true;
5921       break;
5922    }
5923    case nir_intrinsic_first_invocation: {
5924       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5925                get_ssa_temp(ctx, &instr->dest.ssa));
5926       break;
5927    }
5928    case nir_intrinsic_shader_clock:
5929       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
5930       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
5931       break;
5932    case nir_intrinsic_load_vertex_id_zero_base: {
5933       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5934       bld.copy(Definition(dst), ctx->vertex_id);
5935       break;
5936    }
5937    case nir_intrinsic_load_first_vertex: {
5938       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5939       bld.copy(Definition(dst), ctx->base_vertex);
5940       break;
5941    }
5942    case nir_intrinsic_load_base_instance: {
5943       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5944       bld.copy(Definition(dst), ctx->start_instance);
5945       break;
5946    }
5947    case nir_intrinsic_load_instance_id: {
5948       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5949       bld.copy(Definition(dst), ctx->instance_id);
5950       break;
5951    }
5952    case nir_intrinsic_load_draw_id: {
5953       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5954       bld.copy(Definition(dst), ctx->draw_id);
5955       break;
5956    }
5957    default:
5958       fprintf(stderr, "Unimplemented intrinsic instr: ");
5959       nir_print_instr(&instr->instr, stderr);
5960       fprintf(stderr, "\n");
5961       abort();
5962
5963       break;
5964    }
5965 }
5966
5967
5968 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5969                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5970                     enum glsl_base_type *stype)
5971 {
5972    nir_deref_instr *texture_deref_instr = NULL;
5973    nir_deref_instr *sampler_deref_instr = NULL;
5974    int plane = -1;
5975
5976    for (unsigned i = 0; i < instr->num_srcs; i++) {
5977       switch (instr->src[i].src_type) {
5978       case nir_tex_src_texture_deref:
5979          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
5980          break;
5981       case nir_tex_src_sampler_deref:
5982          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
5983          break;
5984       case nir_tex_src_plane:
5985          plane = nir_src_as_int(instr->src[i].src);
5986          break;
5987       default:
5988          break;
5989       }
5990    }
5991
5992    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
5993
5994    if (!sampler_deref_instr)
5995       sampler_deref_instr = texture_deref_instr;
5996
5997    if (plane >= 0) {
5998       assert(instr->op != nir_texop_txf_ms &&
5999              instr->op != nir_texop_samples_identical);
6000       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6001       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6002    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6003       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6004    } else {
6005       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6006    }
6007    if (samp_ptr) {
6008       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6009       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6010          fprintf(stderr, "Unimplemented sampler descriptor: ");
6011          nir_print_instr(&instr->instr, stderr);
6012          fprintf(stderr, "\n");
6013          abort();
6014          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
6015       }
6016    }
6017    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6018                      instr->op == nir_texop_samples_identical))
6019       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6020 }
6021
6022 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6023                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6024 {
6025    Builder bld(ctx->program, ctx->block);
6026
6027    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6028    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6029    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6030
6031    Operand neg_one(0xbf800000u);
6032    Operand one(0x3f800000u);
6033    Operand two(0x40000000u);
6034    Operand four(0x40800000u);
6035
6036    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
6037    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6038    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6039
6040    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
6041    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6042    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
6043    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
6044
6045    // select sc
6046    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6047    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6048                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6049                        one, is_ma_y);
6050    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6051
6052    // select tc
6053    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6054    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6055    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6056
6057    // select ma
6058    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6059                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6060                   deriv_z, is_ma_z);
6061    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6062    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6063 }
6064
6065 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6066 {
6067    Builder bld(ctx->program, ctx->block);
6068    Temp coord_args[4], ma, tc, sc, id;
6069    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6070       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6071
6072    if (is_array) {
6073       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6074
6075       // see comment in ac_prepare_cube_coords()
6076       if (ctx->options->chip_class <= GFX8)
6077          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6078    }
6079
6080    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6081
6082    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6083    vop3a->operands[0] = Operand(ma);
6084    vop3a->abs[0] = true;
6085    Temp invma = bld.tmp(v1);
6086    vop3a->definitions[0] = Definition(invma);
6087    ctx->block->instructions.emplace_back(std::move(vop3a));
6088
6089    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6090    if (!is_deriv)
6091       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6092
6093    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6094    if (!is_deriv)
6095       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6096
6097    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6098
6099    if (is_deriv) {
6100       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6101       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6102
6103       for (unsigned i = 0; i < 2; i++) {
6104          // see comment in ac_prepare_cube_coords()
6105          Temp deriv_ma;
6106          Temp deriv_sc, deriv_tc;
6107          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6108                            &deriv_ma, &deriv_sc, &deriv_tc);
6109
6110          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6111
6112          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6113                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6114                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6115          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6116                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6117                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6118          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6119       }
6120
6121       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6122       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6123    }
6124
6125    if (is_array)
6126       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6127    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6128
6129 }
6130
6131 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6132 {
6133    Temp coord_vec[3];
6134    for (unsigned i = 0; i < coords.size(); i++)
6135       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6136
6137    Builder bld(ctx->program, ctx->block);
6138    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6139
6140    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6141    for (unsigned i = 0; i < coords.size(); i++)
6142       vec->operands[i] = Operand(coord_vec[i]);
6143    Temp res = bld.tmp(RegType::vgpr, coords.size());
6144    vec->definitions[0] = Definition(res);
6145    ctx->block->instructions.emplace_back(std::move(vec));
6146    return res;
6147 }
6148
6149 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6150 {
6151    if (vec->parent_instr->type != nir_instr_type_alu)
6152       return;
6153    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6154    if (vec_instr->op != nir_op_vec(vec->num_components))
6155       return;
6156
6157    for (unsigned i = 0; i < vec->num_components; i++) {
6158       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6159               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6160    }
6161 }
6162
6163 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6164 {
6165    Builder bld(ctx->program, ctx->block);
6166    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6167         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6168    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6169         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6170    nir_const_value *sample_index_cv = NULL;
6171    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6172    enum glsl_base_type stype;
6173    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6174
6175    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6176                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6177    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6178                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6179
6180    for (unsigned i = 0; i < instr->num_srcs; i++) {
6181       switch (instr->src[i].src_type) {
6182       case nir_tex_src_coord:
6183          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6184          break;
6185       case nir_tex_src_bias:
6186          if (instr->op == nir_texop_txb) {
6187             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6188             has_bias = true;
6189          }
6190          break;
6191       case nir_tex_src_lod: {
6192          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6193
6194          if (val && val->f32 <= 0.0) {
6195             level_zero = true;
6196          } else {
6197             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6198             has_lod = true;
6199          }
6200          break;
6201       }
6202       case nir_tex_src_comparator:
6203          if (instr->is_shadow) {
6204             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6205             has_compare = true;
6206          }
6207          break;
6208       case nir_tex_src_offset:
6209          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6210          get_const_vec(instr->src[i].src.ssa, const_offset);
6211          has_offset = true;
6212          break;
6213       case nir_tex_src_ddx:
6214          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6215          has_ddx = true;
6216          break;
6217       case nir_tex_src_ddy:
6218          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6219          has_ddy = true;
6220          break;
6221       case nir_tex_src_ms_index:
6222          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6223          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6224          has_sample_index = true;
6225          break;
6226       case nir_tex_src_texture_offset:
6227       case nir_tex_src_sampler_offset:
6228       default:
6229          break;
6230       }
6231    }
6232 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6233    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6234       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6235
6236    if (instr->op == nir_texop_texture_samples) {
6237       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6238
6239       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6240       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6241       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6242       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6243
6244       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6245                samples, Operand(1u), bld.scc(is_msaa));
6246       return;
6247    }
6248
6249    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6250       aco_ptr<Instruction> tmp_instr;
6251       Temp acc, pack = Temp();
6252
6253       uint32_t pack_const = 0;
6254       for (unsigned i = 0; i < offset.size(); i++) {
6255          if (!const_offset[i])
6256             continue;
6257          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6258       }
6259
6260       if (offset.type() == RegType::sgpr) {
6261          for (unsigned i = 0; i < offset.size(); i++) {
6262             if (const_offset[i])
6263                continue;
6264
6265             acc = emit_extract_vector(ctx, offset, i, s1);
6266             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6267
6268             if (i) {
6269                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6270             }
6271
6272             if (pack == Temp()) {
6273                pack = acc;
6274             } else {
6275                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6276             }
6277          }
6278
6279          if (pack_const && pack != Temp())
6280             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6281       } else {
6282          for (unsigned i = 0; i < offset.size(); i++) {
6283             if (const_offset[i])
6284                continue;
6285
6286             acc = emit_extract_vector(ctx, offset, i, v1);
6287             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6288
6289             if (i) {
6290                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6291             }
6292
6293             if (pack == Temp()) {
6294                pack = acc;
6295             } else {
6296                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6297             }
6298          }
6299
6300          if (pack_const && pack != Temp())
6301             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6302       }
6303       if (pack_const && pack == Temp())
6304          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6305       else if (pack == Temp())
6306          has_offset = false;
6307       else
6308          offset = pack;
6309    }
6310
6311    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6312       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6313
6314    /* pack derivatives */
6315    if (has_ddx || has_ddy) {
6316       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6317          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6318                              ddx, Operand(0u), ddy, Operand(0u));
6319       } else {
6320          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6321       }
6322       has_derivs = true;
6323    }
6324
6325    if (instr->coord_components > 1 &&
6326        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6327        instr->is_array &&
6328        instr->op != nir_texop_txf)
6329       coords = apply_round_slice(ctx, coords, 1);
6330
6331    if (instr->coord_components > 2 &&
6332       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6333        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6334        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6335        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6336        instr->is_array &&
6337        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6338       coords = apply_round_slice(ctx, coords, 2);
6339
6340    if (ctx->options->chip_class == GFX9 &&
6341        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6342        instr->op != nir_texop_lod && instr->coord_components) {
6343       assert(coords.size() > 0 && coords.size() < 3);
6344
6345       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6346       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6347       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6348       if (coords.size() > 1)
6349          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6350       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6351       vec->definitions[0] = Definition(coords);
6352       ctx->block->instructions.emplace_back(std::move(vec));
6353    }
6354
6355    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6356
6357    if (instr->op == nir_texop_samples_identical)
6358       resource = fmask_ptr;
6359
6360    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6361              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6362             instr->op != nir_texop_txs) {
6363       assert(has_sample_index);
6364       Operand op(sample_index);
6365       if (sample_index_cv)
6366          op = Operand(sample_index_cv->u32);
6367       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6368    }
6369
6370    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6371       Temp split_coords[coords.size()];
6372       emit_split_vector(ctx, coords, coords.size());
6373       for (unsigned i = 0; i < coords.size(); i++)
6374          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6375
6376       unsigned i = 0;
6377       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6378          Temp off = emit_extract_vector(ctx, offset, i, v1);
6379          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6380       }
6381
6382       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6383       for (unsigned i = 0; i < coords.size(); i++)
6384          vec->operands[i] = Operand(split_coords[i]);
6385       coords = bld.tmp(coords.regClass());
6386       vec->definitions[0] = Definition(coords);
6387       ctx->block->instructions.emplace_back(std::move(vec));
6388
6389       has_offset = false;
6390    }
6391
6392    /* Build tex instruction */
6393    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6394    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6395                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6396                   : 0;
6397    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6398    Temp tmp_dst = dst;
6399
6400    /* gather4 selects the component by dmask and always returns vec4 */
6401    if (instr->op == nir_texop_tg4) {
6402       assert(instr->dest.ssa.num_components == 4);
6403       if (instr->is_shadow)
6404          dmask = 1;
6405       else
6406          dmask = 1 << instr->component;
6407       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6408          tmp_dst = bld.tmp(v4);
6409    } else if (instr->op == nir_texop_samples_identical) {
6410       tmp_dst = bld.tmp(v1);
6411    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6412       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6413    }
6414
6415    aco_ptr<MIMG_instruction> tex;
6416    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6417       if (!has_lod)
6418          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6419
6420       bool div_by_6 = instr->op == nir_texop_txs &&
6421                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6422                       instr->is_array &&
6423                       (dmask & (1 << 2));
6424       if (tmp_dst.id() == dst.id() && div_by_6)
6425          tmp_dst = bld.tmp(tmp_dst.regClass());
6426
6427       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6428       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6429       tex->operands[1] = Operand(resource);
6430       if (ctx->options->chip_class == GFX9 &&
6431           instr->op == nir_texop_txs &&
6432           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6433           instr->is_array) {
6434          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6435       } else if (instr->op == nir_texop_query_levels) {
6436          tex->dmask = 1 << 3;
6437       } else {
6438          tex->dmask = dmask;
6439       }
6440       tex->da = da;
6441       tex->definitions[0] = Definition(tmp_dst);
6442       tex->dim = dim;
6443       tex->can_reorder = true;
6444       ctx->block->instructions.emplace_back(std::move(tex));
6445
6446       if (div_by_6) {
6447          /* divide 3rd value by 6 by multiplying with magic number */
6448          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6449          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6450          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6451          assert(instr->dest.ssa.num_components == 3);
6452          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6453          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6454                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6455                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6456                               by_6);
6457
6458       }
6459
6460       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6461       return;
6462    }
6463
6464    Temp tg4_compare_cube_wa64 = Temp();
6465
6466    if (tg4_integer_workarounds) {
6467       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6468       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6469       tex->operands[1] = Operand(resource);
6470       tex->dim = dim;
6471       tex->dmask = 0x3;
6472       tex->da = da;
6473       Temp size = bld.tmp(v2);
6474       tex->definitions[0] = Definition(size);
6475       tex->can_reorder = true;
6476       ctx->block->instructions.emplace_back(std::move(tex));
6477       emit_split_vector(ctx, size, size.size());
6478
6479       Temp half_texel[2];
6480       for (unsigned i = 0; i < 2; i++) {
6481          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6482          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6483          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6484          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6485       }
6486
6487       Temp orig_coords[2] = {
6488          emit_extract_vector(ctx, coords, 0, v1),
6489          emit_extract_vector(ctx, coords, 1, v1)};
6490       Temp new_coords[2] = {
6491          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6492          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6493       };
6494
6495       if (tg4_integer_cube_workaround) {
6496          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6497          Temp desc[resource.size()];
6498          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6499                                                                            Format::PSEUDO, 1, resource.size())};
6500          split->operands[0] = Operand(resource);
6501          for (unsigned i = 0; i < resource.size(); i++) {
6502             desc[i] = bld.tmp(s1);
6503             split->definitions[i] = Definition(desc[i]);
6504          }
6505          ctx->block->instructions.emplace_back(std::move(split));
6506
6507          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6508          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6509                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6510
6511          Temp nfmt;
6512          if (stype == GLSL_TYPE_UINT) {
6513             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6514                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6515                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6516                             bld.scc(compare_cube_wa));
6517          } else {
6518             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6519                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6520                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6521                             bld.scc(compare_cube_wa));
6522          }
6523          tg4_compare_cube_wa64 = as_divergent_bool(ctx, compare_cube_wa, true);
6524          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6525
6526          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6527                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6528          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6529
6530          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6531                                                                          Format::PSEUDO, resource.size(), 1)};
6532          for (unsigned i = 0; i < resource.size(); i++)
6533             vec->operands[i] = Operand(desc[i]);
6534          resource = bld.tmp(resource.regClass());
6535          vec->definitions[0] = Definition(resource);
6536          ctx->block->instructions.emplace_back(std::move(vec));
6537
6538          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6539                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6540          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6541                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6542       }
6543
6544       if (coords.size() == 3) {
6545          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6546                              new_coords[0], new_coords[1],
6547                              emit_extract_vector(ctx, coords, 2, v1));
6548       } else {
6549          assert(coords.size() == 2);
6550          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6551                              new_coords[0], new_coords[1]);
6552       }
6553    }
6554
6555    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6556        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6557        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6558       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6559
6560    std::vector<Operand> args;
6561    if (has_offset)
6562       args.emplace_back(Operand(offset));
6563    if (has_bias)
6564       args.emplace_back(Operand(bias));
6565    if (has_compare)
6566       args.emplace_back(Operand(compare));
6567    if (has_derivs)
6568       args.emplace_back(Operand(derivs));
6569    args.emplace_back(Operand(coords));
6570    if (has_sample_index)
6571       args.emplace_back(Operand(sample_index));
6572    if (has_lod)
6573       args.emplace_back(lod);
6574
6575    Operand arg;
6576    if (args.size() > 1) {
6577       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6578       unsigned size = 0;
6579       for (unsigned i = 0; i < args.size(); i++) {
6580          size += args[i].size();
6581          vec->operands[i] = args[i];
6582       }
6583       RegClass rc = RegClass(RegType::vgpr, size);
6584       Temp tmp = bld.tmp(rc);
6585       vec->definitions[0] = Definition(tmp);
6586       ctx->block->instructions.emplace_back(std::move(vec));
6587       arg = Operand(tmp);
6588    } else {
6589       assert(args[0].isTemp());
6590       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6591    }
6592
6593    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6594       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6595
6596       assert(coords.size() == 1);
6597       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6598       aco_opcode op;
6599       switch (last_bit) {
6600       case 1:
6601          op = aco_opcode::buffer_load_format_x; break;
6602       case 2:
6603          op = aco_opcode::buffer_load_format_xy; break;
6604       case 3:
6605          op = aco_opcode::buffer_load_format_xyz; break;
6606       case 4:
6607          op = aco_opcode::buffer_load_format_xyzw; break;
6608       default:
6609          unreachable("Tex instruction loads more than 4 components.");
6610       }
6611
6612       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6613       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6614          tmp_dst = dst;
6615       else
6616          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6617
6618       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6619       mubuf->operands[0] = Operand(coords);
6620       mubuf->operands[1] = Operand(resource);
6621       mubuf->operands[2] = Operand((uint32_t) 0);
6622       mubuf->definitions[0] = Definition(tmp_dst);
6623       mubuf->idxen = true;
6624       mubuf->can_reorder = true;
6625       ctx->block->instructions.emplace_back(std::move(mubuf));
6626
6627       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6628       return;
6629    }
6630
6631
6632    if (instr->op == nir_texop_txf ||
6633        instr->op == nir_texop_txf_ms ||
6634        instr->op == nir_texop_samples_identical) {
6635       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6636       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6637       tex->operands[0] = Operand(arg);
6638       tex->operands[1] = Operand(resource);
6639       tex->dim = dim;
6640       tex->dmask = dmask;
6641       tex->unrm = true;
6642       tex->da = da;
6643       tex->definitions[0] = Definition(tmp_dst);
6644       tex->can_reorder = true;
6645       ctx->block->instructions.emplace_back(std::move(tex));
6646
6647       if (instr->op == nir_texop_samples_identical) {
6648          assert(dmask == 1 && dst.regClass() == v1);
6649          assert(dst.id() != tmp_dst.id());
6650
6651          Temp tmp = bld.tmp(s2);
6652          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6653          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6654
6655       } else {
6656          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6657       }
6658       return;
6659    }
6660
6661    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6662    aco_opcode opcode = aco_opcode::image_sample;
6663    if (has_offset) { /* image_sample_*_o */
6664       if (has_compare) {
6665          opcode = aco_opcode::image_sample_c_o;
6666          if (has_derivs)
6667             opcode = aco_opcode::image_sample_c_d_o;
6668          if (has_bias)
6669             opcode = aco_opcode::image_sample_c_b_o;
6670          if (level_zero)
6671             opcode = aco_opcode::image_sample_c_lz_o;
6672          if (has_lod)
6673             opcode = aco_opcode::image_sample_c_l_o;
6674       } else {
6675          opcode = aco_opcode::image_sample_o;
6676          if (has_derivs)
6677             opcode = aco_opcode::image_sample_d_o;
6678          if (has_bias)
6679             opcode = aco_opcode::image_sample_b_o;
6680          if (level_zero)
6681             opcode = aco_opcode::image_sample_lz_o;
6682          if (has_lod)
6683             opcode = aco_opcode::image_sample_l_o;
6684       }
6685    } else { /* no offset */
6686       if (has_compare) {
6687          opcode = aco_opcode::image_sample_c;
6688          if (has_derivs)
6689             opcode = aco_opcode::image_sample_c_d;
6690          if (has_bias)
6691             opcode = aco_opcode::image_sample_c_b;
6692          if (level_zero)
6693             opcode = aco_opcode::image_sample_c_lz;
6694          if (has_lod)
6695             opcode = aco_opcode::image_sample_c_l;
6696       } else {
6697          opcode = aco_opcode::image_sample;
6698          if (has_derivs)
6699             opcode = aco_opcode::image_sample_d;
6700          if (has_bias)
6701             opcode = aco_opcode::image_sample_b;
6702          if (level_zero)
6703             opcode = aco_opcode::image_sample_lz;
6704          if (has_lod)
6705             opcode = aco_opcode::image_sample_l;
6706       }
6707    }
6708
6709    if (instr->op == nir_texop_tg4) {
6710       if (has_offset) {
6711          opcode = aco_opcode::image_gather4_lz_o;
6712          if (has_compare)
6713             opcode = aco_opcode::image_gather4_c_lz_o;
6714       } else {
6715          opcode = aco_opcode::image_gather4_lz;
6716          if (has_compare)
6717             opcode = aco_opcode::image_gather4_c_lz;
6718       }
6719    } else if (instr->op == nir_texop_lod) {
6720       opcode = aco_opcode::image_get_lod;
6721    }
6722
6723    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6724    tex->operands[0] = arg;
6725    tex->operands[1] = Operand(resource);
6726    tex->operands[2] = Operand(sampler);
6727    tex->dim = dim;
6728    tex->dmask = dmask;
6729    tex->da = da;
6730    tex->definitions[0] = Definition(tmp_dst);
6731    tex->can_reorder = true;
6732    ctx->block->instructions.emplace_back(std::move(tex));
6733
6734    if (tg4_integer_cube_workaround) {
6735       assert(tmp_dst.id() != dst.id());
6736       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6737
6738       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6739       Temp val[4];
6740       for (unsigned i = 0; i < dst.size(); i++) {
6741          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6742          Temp cvt_val;
6743          if (stype == GLSL_TYPE_UINT)
6744             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6745          else
6746             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6747          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6748       }
6749       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6750       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6751                            val[0], val[1], val[2], val[3]);
6752    }
6753    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6754    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6755
6756 }
6757
6758
6759 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6760 {
6761    Temp tmp = get_ssa_temp(ctx, ssa);
6762    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6763       return Operand(tmp.regClass());
6764    else
6765       return Operand(tmp);
6766 }
6767
6768 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6769 {
6770    aco_ptr<Pseudo_instruction> phi;
6771    unsigned num_src = exec_list_length(&instr->srcs);
6772    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6773
6774    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6775
6776    std::map<unsigned, nir_ssa_def*> phi_src;
6777    bool all_undef = true;
6778    nir_foreach_phi_src(src, instr) {
6779       phi_src[src->pred->index] = src->src.ssa;
6780       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6781          all_undef = false;
6782    }
6783    if (all_undef) {
6784       Builder bld(ctx->program, ctx->block);
6785       if (dst.regClass() == s1) {
6786          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6787       } else if (dst.regClass() == v1) {
6788          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6789       } else {
6790          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6791          for (unsigned i = 0; i < dst.size(); i++)
6792             vec->operands[i] = Operand(0u);
6793          vec->definitions[0] = Definition(dst);
6794          ctx->block->instructions.emplace_back(std::move(vec));
6795       }
6796       return;
6797    }
6798
6799    /* try to scalarize vector phis */
6800    if (dst.size() > 1) {
6801       // TODO: scalarize linear phis on divergent ifs
6802       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6803       std::array<Temp, 4> new_vec;
6804       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6805          Operand src = get_phi_operand(ctx, pair.second);
6806          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6807             can_scalarize = false;
6808             break;
6809          }
6810       }
6811       if (can_scalarize) {
6812          unsigned num_components = instr->dest.ssa.num_components;
6813          assert(dst.size() % num_components == 0);
6814          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6815
6816          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6817          for (unsigned k = 0; k < num_components; k++) {
6818             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6819             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6820             for (unsigned i = 0; i < num_src; i++) {
6821                Operand src = get_phi_operand(ctx, it->second);
6822                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6823                ++it;
6824             }
6825             Temp phi_dst = {ctx->program->allocateId(), rc};
6826             phi->definitions[0] = Definition(phi_dst);
6827             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6828             new_vec[k] = phi_dst;
6829             vec->operands[k] = Operand(phi_dst);
6830          }
6831          vec->definitions[0] = Definition(dst);
6832          ctx->block->instructions.emplace_back(std::move(vec));
6833          ctx->allocated_vec.emplace(dst.id(), new_vec);
6834          return;
6835       }
6836    }
6837
6838    unsigned extra_src = 0;
6839    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6840        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6841       extra_src++;
6842    }
6843
6844    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6845
6846    /* if we have a linear phi on a divergent if, we know that one src is undef */
6847    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6848       assert(extra_src == 0);
6849       Block* block;
6850       /* we place the phi either in the invert-block or in the current block */
6851       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6852          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6853          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6854          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6855          assert(block->kind & block_kind_invert);
6856          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6857       } else {
6858          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6859          block = ctx->block;
6860          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6861       }
6862       phi->operands[1] = Operand(dst.regClass());
6863       phi->definitions[0] = Definition(dst);
6864       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6865       return;
6866    }
6867
6868    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6869    for (unsigned i = 0; i < num_src; i++) {
6870       phi->operands[i] = get_phi_operand(ctx, it->second);
6871       ++it;
6872    }
6873    for (unsigned i = 0; i < extra_src; i++)
6874       phi->operands[num_src + i] = Operand(dst.regClass());
6875    phi->definitions[0] = Definition(dst);
6876    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6877 }
6878
6879
6880 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6881 {
6882    Temp dst = get_ssa_temp(ctx, &instr->def);
6883
6884    assert(dst.type() == RegType::sgpr);
6885
6886    if (dst.size() == 1) {
6887       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6888    } else {
6889       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6890       for (unsigned i = 0; i < dst.size(); i++)
6891          vec->operands[i] = Operand(0u);
6892       vec->definitions[0] = Definition(dst);
6893       ctx->block->instructions.emplace_back(std::move(vec));
6894    }
6895 }
6896
6897 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6898 {
6899    Builder bld(ctx->program, ctx->block);
6900    Block *logical_target;
6901    append_logical_end(ctx->block);
6902    unsigned idx = ctx->block->index;
6903
6904    switch (instr->type) {
6905    case nir_jump_break:
6906       logical_target = ctx->cf_info.parent_loop.exit;
6907       add_logical_edge(idx, logical_target);
6908       ctx->block->kind |= block_kind_break;
6909
6910       if (!ctx->cf_info.parent_if.is_divergent &&
6911           !ctx->cf_info.parent_loop.has_divergent_continue) {
6912          /* uniform break - directly jump out of the loop */
6913          ctx->block->kind |= block_kind_uniform;
6914          ctx->cf_info.has_branch = true;
6915          bld.branch(aco_opcode::p_branch);
6916          add_linear_edge(idx, logical_target);
6917          return;
6918       }
6919       ctx->cf_info.parent_loop.has_divergent_branch = true;
6920       break;
6921    case nir_jump_continue:
6922       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6923       add_logical_edge(idx, logical_target);
6924       ctx->block->kind |= block_kind_continue;
6925
6926       if (ctx->cf_info.parent_if.is_divergent) {
6927          /* for potential uniform breaks after this continue,
6928             we must ensure that they are handled correctly */
6929          ctx->cf_info.parent_loop.has_divergent_continue = true;
6930          ctx->cf_info.parent_loop.has_divergent_branch = true;
6931       } else {
6932          /* uniform continue - directly jump to the loop header */
6933          ctx->block->kind |= block_kind_uniform;
6934          ctx->cf_info.has_branch = true;
6935          bld.branch(aco_opcode::p_branch);
6936          add_linear_edge(idx, logical_target);
6937          return;
6938       }
6939       break;
6940    default:
6941       fprintf(stderr, "Unknown NIR jump instr: ");
6942       nir_print_instr(&instr->instr, stderr);
6943       fprintf(stderr, "\n");
6944       abort();
6945    }
6946
6947    /* remove critical edges from linear CFG */
6948    bld.branch(aco_opcode::p_branch);
6949    Block* break_block = ctx->program->create_and_insert_block();
6950    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6951    break_block->kind |= block_kind_uniform;
6952    add_linear_edge(idx, break_block);
6953    /* the loop_header pointer might be invalidated by this point */
6954    if (instr->type == nir_jump_continue)
6955       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6956    add_linear_edge(break_block->index, logical_target);
6957    bld.reset(break_block);
6958    bld.branch(aco_opcode::p_branch);
6959
6960    Block* continue_block = ctx->program->create_and_insert_block();
6961    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6962    add_linear_edge(idx, continue_block);
6963    append_logical_start(continue_block);
6964    ctx->block = continue_block;
6965    return;
6966 }
6967
6968 void visit_block(isel_context *ctx, nir_block *block)
6969 {
6970    nir_foreach_instr(instr, block) {
6971       switch (instr->type) {
6972       case nir_instr_type_alu:
6973          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6974          break;
6975       case nir_instr_type_load_const:
6976          visit_load_const(ctx, nir_instr_as_load_const(instr));
6977          break;
6978       case nir_instr_type_intrinsic:
6979          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
6980          break;
6981       case nir_instr_type_tex:
6982          visit_tex(ctx, nir_instr_as_tex(instr));
6983          break;
6984       case nir_instr_type_phi:
6985          visit_phi(ctx, nir_instr_as_phi(instr));
6986          break;
6987       case nir_instr_type_ssa_undef:
6988          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
6989          break;
6990       case nir_instr_type_deref:
6991          break;
6992       case nir_instr_type_jump:
6993          visit_jump(ctx, nir_instr_as_jump(instr));
6994          break;
6995       default:
6996          fprintf(stderr, "Unknown NIR instr type: ");
6997          nir_print_instr(instr, stderr);
6998          fprintf(stderr, "\n");
6999          //abort();
7000       }
7001    }
7002 }
7003
7004
7005
7006 static void visit_loop(isel_context *ctx, nir_loop *loop)
7007 {
7008    append_logical_end(ctx->block);
7009    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7010    Builder bld(ctx->program, ctx->block);
7011    bld.branch(aco_opcode::p_branch);
7012    unsigned loop_preheader_idx = ctx->block->index;
7013
7014    Block loop_exit = Block();
7015    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7016    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7017
7018    Block* loop_header = ctx->program->create_and_insert_block();
7019    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7020    loop_header->kind |= block_kind_loop_header;
7021    add_edge(loop_preheader_idx, loop_header);
7022    ctx->block = loop_header;
7023
7024    /* emit loop body */
7025    unsigned loop_header_idx = loop_header->index;
7026    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7027    append_logical_start(ctx->block);
7028    visit_cf_list(ctx, &loop->body);
7029
7030    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7031    if (!ctx->cf_info.has_branch) {
7032       append_logical_end(ctx->block);
7033       if (ctx->cf_info.exec_potentially_empty) {
7034          /* Discards can result in code running with an empty exec mask.
7035           * This would result in divergent breaks not ever being taken. As a
7036           * workaround, break the loop when the loop mask is empty instead of
7037           * always continuing. */
7038          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7039          unsigned block_idx = ctx->block->index;
7040
7041          /* create helper blocks to avoid critical edges */
7042          Block *break_block = ctx->program->create_and_insert_block();
7043          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7044          break_block->kind = block_kind_uniform;
7045          bld.reset(break_block);
7046          bld.branch(aco_opcode::p_branch);
7047          add_linear_edge(block_idx, break_block);
7048          add_linear_edge(break_block->index, &loop_exit);
7049
7050          Block *continue_block = ctx->program->create_and_insert_block();
7051          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7052          continue_block->kind = block_kind_uniform;
7053          bld.reset(continue_block);
7054          bld.branch(aco_opcode::p_branch);
7055          add_linear_edge(block_idx, continue_block);
7056          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7057
7058          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7059          ctx->block = &ctx->program->blocks[block_idx];
7060       } else {
7061          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7062          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7063             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7064          else
7065             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7066       }
7067
7068       bld.reset(ctx->block);
7069       bld.branch(aco_opcode::p_branch);
7070    }
7071
7072    /* fixup phis in loop header from unreachable blocks */
7073    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7074       bool linear = ctx->cf_info.has_branch;
7075       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7076       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7077          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7078              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7079             /* the last operand should be the one that needs to be removed */
7080             instr->operands.pop_back();
7081          } else if (!is_phi(instr)) {
7082             break;
7083          }
7084       }
7085    }
7086
7087    ctx->cf_info.has_branch = false;
7088
7089    // TODO: if the loop has not a single exit, we must add one °°
7090    /* emit loop successor block */
7091    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7092    append_logical_start(ctx->block);
7093
7094    #if 0
7095    // TODO: check if it is beneficial to not branch on continues
7096    /* trim linear phis in loop header */
7097    for (auto&& instr : loop_entry->instructions) {
7098       if (instr->opcode == aco_opcode::p_linear_phi) {
7099          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7100          new_phi->definitions[0] = instr->definitions[0];
7101          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7102             new_phi->operands[i] = instr->operands[i];
7103          /* check that the remaining operands are all the same */
7104          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7105             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7106          instr.swap(new_phi);
7107       } else if (instr->opcode == aco_opcode::p_phi) {
7108          continue;
7109       } else {
7110          break;
7111       }
7112    }
7113    #endif
7114 }
7115
7116 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7117 {
7118    ic->cond = cond;
7119
7120    append_logical_end(ctx->block);
7121    ctx->block->kind |= block_kind_branch;
7122
7123    /* branch to linear then block */
7124    assert(cond.regClass() == s2);
7125    aco_ptr<Pseudo_branch_instruction> branch;
7126    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7127    branch->operands[0] = Operand(cond);
7128    ctx->block->instructions.push_back(std::move(branch));
7129
7130    ic->BB_if_idx = ctx->block->index;
7131    ic->BB_invert = Block();
7132    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7133    /* Invert blocks are intentionally not marked as top level because they
7134     * are not part of the logical cfg. */
7135    ic->BB_invert.kind |= block_kind_invert;
7136    ic->BB_endif = Block();
7137    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7138    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7139
7140    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7141    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7142    ctx->cf_info.parent_if.is_divergent = true;
7143    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7144
7145    /** emit logical then block */
7146    Block* BB_then_logical = ctx->program->create_and_insert_block();
7147    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7148    add_edge(ic->BB_if_idx, BB_then_logical);
7149    ctx->block = BB_then_logical;
7150    append_logical_start(BB_then_logical);
7151 }
7152
7153 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7154 {
7155    Block *BB_then_logical = ctx->block;
7156    append_logical_end(BB_then_logical);
7157     /* branch from logical then block to invert block */
7158    aco_ptr<Pseudo_branch_instruction> branch;
7159    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7160    BB_then_logical->instructions.emplace_back(std::move(branch));
7161    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7162    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7163       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7164    BB_then_logical->kind |= block_kind_uniform;
7165    assert(!ctx->cf_info.has_branch);
7166    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7167    ctx->cf_info.parent_loop.has_divergent_branch = false;
7168
7169    /** emit linear then block */
7170    Block* BB_then_linear = ctx->program->create_and_insert_block();
7171    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7172    BB_then_linear->kind |= block_kind_uniform;
7173    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7174    /* branch from linear then block to invert block */
7175    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7176    BB_then_linear->instructions.emplace_back(std::move(branch));
7177    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7178
7179    /** emit invert merge block */
7180    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7181    ic->invert_idx = ctx->block->index;
7182
7183    /* branch to linear else block (skip else) */
7184    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7185    branch->operands[0] = Operand(ic->cond);
7186    ctx->block->instructions.push_back(std::move(branch));
7187
7188    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7189    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7190
7191    /** emit logical else block */
7192    Block* BB_else_logical = ctx->program->create_and_insert_block();
7193    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7194    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7195    add_linear_edge(ic->invert_idx, BB_else_logical);
7196    ctx->block = BB_else_logical;
7197    append_logical_start(BB_else_logical);
7198 }
7199
7200 static void end_divergent_if(isel_context *ctx, if_context *ic)
7201 {
7202    Block *BB_else_logical = ctx->block;
7203    append_logical_end(BB_else_logical);
7204
7205    /* branch from logical else block to endif block */
7206    aco_ptr<Pseudo_branch_instruction> branch;
7207    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7208    BB_else_logical->instructions.emplace_back(std::move(branch));
7209    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7210    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7211       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7212    BB_else_logical->kind |= block_kind_uniform;
7213
7214    assert(!ctx->cf_info.has_branch);
7215    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7216
7217
7218    /** emit linear else block */
7219    Block* BB_else_linear = ctx->program->create_and_insert_block();
7220    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7221    BB_else_linear->kind |= block_kind_uniform;
7222    add_linear_edge(ic->invert_idx, BB_else_linear);
7223
7224    /* branch from linear else block to endif block */
7225    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7226    BB_else_linear->instructions.emplace_back(std::move(branch));
7227    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7228
7229
7230    /** emit endif merge block */
7231    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7232    append_logical_start(ctx->block);
7233
7234
7235    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7236    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7237    /* uniform control flow never has an empty exec-mask */
7238    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7239       ctx->cf_info.exec_potentially_empty = false;
7240 }
7241
7242 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7243 {
7244    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7245    Builder bld(ctx->program, ctx->block);
7246    aco_ptr<Pseudo_branch_instruction> branch;
7247
7248    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7249       /**
7250        * Uniform conditionals are represented in the following way*) :
7251        *
7252        * The linear and logical CFG:
7253        *                        BB_IF
7254        *                        /    \
7255        *       BB_THEN (logical)      BB_ELSE (logical)
7256        *                        \    /
7257        *                        BB_ENDIF
7258        *
7259        * *) Exceptions may be due to break and continue statements within loops
7260        *    If a break/continue happens within uniform control flow, it branches
7261        *    to the loop exit/entry block. Otherwise, it branches to the next
7262        *    merge block.
7263        **/
7264       append_logical_end(ctx->block);
7265       ctx->block->kind |= block_kind_uniform;
7266
7267       /* emit branch */
7268       if (cond.regClass() == s2) {
7269          // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7270          cond = as_uniform_bool(ctx, cond);
7271       }
7272       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7273       branch->operands[0] = Operand(cond);
7274       branch->operands[0].setFixed(scc);
7275       ctx->block->instructions.emplace_back(std::move(branch));
7276
7277       unsigned BB_if_idx = ctx->block->index;
7278       Block BB_endif = Block();
7279       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7280       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7281
7282       /** emit then block */
7283       Block* BB_then = ctx->program->create_and_insert_block();
7284       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7285       add_edge(BB_if_idx, BB_then);
7286       append_logical_start(BB_then);
7287       ctx->block = BB_then;
7288       visit_cf_list(ctx, &if_stmt->then_list);
7289       BB_then = ctx->block;
7290       bool then_branch = ctx->cf_info.has_branch;
7291       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7292
7293       if (!then_branch) {
7294          append_logical_end(BB_then);
7295          /* branch from then block to endif block */
7296          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7297          BB_then->instructions.emplace_back(std::move(branch));
7298          add_linear_edge(BB_then->index, &BB_endif);
7299          if (!then_branch_divergent)
7300             add_logical_edge(BB_then->index, &BB_endif);
7301          BB_then->kind |= block_kind_uniform;
7302       }
7303
7304       ctx->cf_info.has_branch = false;
7305       ctx->cf_info.parent_loop.has_divergent_branch = false;
7306
7307       /** emit else block */
7308       Block* BB_else = ctx->program->create_and_insert_block();
7309       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7310       add_edge(BB_if_idx, BB_else);
7311       append_logical_start(BB_else);
7312       ctx->block = BB_else;
7313       visit_cf_list(ctx, &if_stmt->else_list);
7314       BB_else = ctx->block;
7315
7316       if (!ctx->cf_info.has_branch) {
7317          append_logical_end(BB_else);
7318          /* branch from then block to endif block */
7319          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7320          BB_else->instructions.emplace_back(std::move(branch));
7321          add_linear_edge(BB_else->index, &BB_endif);
7322          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7323             add_logical_edge(BB_else->index, &BB_endif);
7324          BB_else->kind |= block_kind_uniform;
7325       }
7326
7327       ctx->cf_info.has_branch &= then_branch;
7328       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7329
7330       /** emit endif merge block */
7331       if (!ctx->cf_info.has_branch) {
7332          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7333          append_logical_start(ctx->block);
7334       }
7335    } else { /* non-uniform condition */
7336       /**
7337        * To maintain a logical and linear CFG without critical edges,
7338        * non-uniform conditionals are represented in the following way*) :
7339        *
7340        * The linear CFG:
7341        *                        BB_IF
7342        *                        /    \
7343        *       BB_THEN (logical)      BB_THEN (linear)
7344        *                        \    /
7345        *                        BB_INVERT (linear)
7346        *                        /    \
7347        *       BB_ELSE (logical)      BB_ELSE (linear)
7348        *                        \    /
7349        *                        BB_ENDIF
7350        *
7351        * The logical CFG:
7352        *                        BB_IF
7353        *                        /    \
7354        *       BB_THEN (logical)      BB_ELSE (logical)
7355        *                        \    /
7356        *                        BB_ENDIF
7357        *
7358        * *) Exceptions may be due to break and continue statements within loops
7359        **/
7360
7361       if_context ic;
7362
7363       begin_divergent_if_then(ctx, &ic, cond);
7364       visit_cf_list(ctx, &if_stmt->then_list);
7365
7366       begin_divergent_if_else(ctx, &ic);
7367       visit_cf_list(ctx, &if_stmt->else_list);
7368
7369       end_divergent_if(ctx, &ic);
7370    }
7371 }
7372
7373 static void visit_cf_list(isel_context *ctx,
7374                           struct exec_list *list)
7375 {
7376    foreach_list_typed(nir_cf_node, node, node, list) {
7377       switch (node->type) {
7378       case nir_cf_node_block:
7379          visit_block(ctx, nir_cf_node_as_block(node));
7380          break;
7381       case nir_cf_node_if:
7382          visit_if(ctx, nir_cf_node_as_if(node));
7383          break;
7384       case nir_cf_node_loop:
7385          visit_loop(ctx, nir_cf_node_as_loop(node));
7386          break;
7387       default:
7388          unreachable("unimplemented cf list type");
7389       }
7390    }
7391 }
7392
7393 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7394 {
7395    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7396    uint64_t mask = ctx->vs_output.mask[slot];
7397    if (!is_pos && !mask)
7398       return;
7399    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7400       return;
7401    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7402    exp->enabled_mask = mask;
7403    for (unsigned i = 0; i < 4; ++i) {
7404       if (mask & (1 << i))
7405          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7406       else
7407          exp->operands[i] = Operand(v1);
7408    }
7409    exp->valid_mask = false;
7410    exp->done = false;
7411    exp->compressed = false;
7412    if (is_pos)
7413       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7414    else
7415       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7416    ctx->block->instructions.emplace_back(std::move(exp));
7417 }
7418
7419 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7420 {
7421    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7422    exp->enabled_mask = 0;
7423    for (unsigned i = 0; i < 4; ++i)
7424       exp->operands[i] = Operand(v1);
7425    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7426       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7427       exp->enabled_mask |= 0x1;
7428    }
7429    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7430       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7431       exp->enabled_mask |= 0x4;
7432    }
7433    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7434       if (ctx->options->chip_class < GFX9) {
7435          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7436          exp->enabled_mask |= 0x8;
7437       } else {
7438          Builder bld(ctx->program, ctx->block);
7439
7440          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7441                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7442          if (exp->operands[2].isTemp())
7443             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7444
7445          exp->operands[2] = Operand(out);
7446          exp->enabled_mask |= 0x4;
7447       }
7448    }
7449    exp->valid_mask = false;
7450    exp->done = false;
7451    exp->compressed = false;
7452    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7453    ctx->block->instructions.emplace_back(std::move(exp));
7454 }
7455
7456 static void create_vs_exports(isel_context *ctx)
7457 {
7458    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7459
7460    if (outinfo->export_prim_id) {
7461       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7462       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7463    }
7464
7465    if (ctx->options->key.has_multiview_view_index) {
7466       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7467       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7468    }
7469
7470    /* the order these position exports are created is important */
7471    int next_pos = 0;
7472    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7473    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7474       export_vs_psiz_layer_viewport(ctx, &next_pos);
7475    }
7476    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7477       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7478    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7479       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7480
7481    if (ctx->options->key.vs_common_out.export_clip_dists) {
7482       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7483          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7484       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7485          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7486    }
7487
7488    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7489       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7490           i != VARYING_SLOT_PRIMITIVE_ID)
7491          continue;
7492
7493       export_vs_varying(ctx, i, false, NULL);
7494    }
7495 }
7496
7497 static void emit_stream_output(isel_context *ctx,
7498                                Temp const *so_buffers,
7499                                Temp const *so_write_offset,
7500                                const struct radv_stream_output *output)
7501 {
7502    unsigned num_comps = util_bitcount(output->component_mask);
7503    unsigned loc = output->location;
7504    unsigned buf = output->buffer;
7505    unsigned offset = output->offset;
7506
7507    assert(num_comps && num_comps <= 4);
7508    if (!num_comps || num_comps > 4)
7509       return;
7510
7511    unsigned start = ffs(output->component_mask) - 1;
7512
7513    Temp out[4];
7514    bool all_undef = true;
7515    assert(ctx->stage == vertex_vs);
7516    for (unsigned i = 0; i < num_comps; i++) {
7517       out[i] = ctx->vs_output.outputs[loc][start + i];
7518       all_undef = all_undef && !out[i].id();
7519    }
7520    if (all_undef)
7521       return;
7522
7523    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7524    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7525    for (unsigned i = 0; i < num_comps; ++i)
7526       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7527    vec->definitions[0] = Definition(write_data);
7528    ctx->block->instructions.emplace_back(std::move(vec));
7529
7530    aco_opcode opcode;
7531    switch (num_comps) {
7532    case 1:
7533       opcode = aco_opcode::buffer_store_dword;
7534       break;
7535    case 2:
7536       opcode = aco_opcode::buffer_store_dwordx2;
7537       break;
7538    case 3:
7539       opcode = aco_opcode::buffer_store_dwordx3;
7540       break;
7541    case 4:
7542       opcode = aco_opcode::buffer_store_dwordx4;
7543       break;
7544    }
7545
7546    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7547    store->operands[0] = Operand(so_write_offset[buf]);
7548    store->operands[1] = Operand(so_buffers[buf]);
7549    store->operands[2] = Operand((uint32_t) 0);
7550    store->operands[3] = Operand(write_data);
7551    if (offset > 4095) {
7552       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7553       Builder bld(ctx->program, ctx->block);
7554       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7555    } else {
7556       store->offset = offset;
7557    }
7558    store->offen = true;
7559    store->glc = true;
7560    store->dlc = false;
7561    store->slc = true;
7562    store->can_reorder = true;
7563    ctx->block->instructions.emplace_back(std::move(store));
7564 }
7565
7566 static void emit_streamout(isel_context *ctx, unsigned stream)
7567 {
7568    Builder bld(ctx->program, ctx->block);
7569
7570    Temp so_buffers[4];
7571    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7572    for (unsigned i = 0; i < 4; i++) {
7573       unsigned stride = ctx->program->info->so.strides[i];
7574       if (!stride)
7575          continue;
7576
7577       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7578    }
7579
7580    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7581                                 ctx->streamout_config, Operand(0x70010u));
7582
7583    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7584                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7585
7586    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7587
7588    if_context ic;
7589    begin_divergent_if_then(ctx, &ic, can_emit);
7590
7591    bld.reset(ctx->block);
7592
7593    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7594
7595    Temp so_write_offset[4];
7596
7597    for (unsigned i = 0; i < 4; i++) {
7598       unsigned stride = ctx->program->info->so.strides[i];
7599       if (!stride)
7600          continue;
7601
7602       if (stride == 1) {
7603          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7604                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7605          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7606
7607          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7608       } else {
7609          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7610          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7611          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7612       }
7613    }
7614
7615    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7616       struct radv_stream_output *output =
7617          &ctx->program->info->so.outputs[i];
7618       if (stream != output->stream)
7619          continue;
7620
7621       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7622    }
7623
7624    begin_divergent_if_else(ctx, &ic);
7625    end_divergent_if(ctx, &ic);
7626 }
7627
7628 } /* end namespace */
7629
7630 void handle_bc_optimize(isel_context *ctx)
7631 {
7632    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7633    Builder bld(ctx->program, ctx->block);
7634    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7635    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7636    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7637    if (uses_center && uses_centroid) {
7638       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7639
7640       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7641          for (unsigned i = 0; i < 2; i++) {
7642             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7643                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7644                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7645                                       sel);
7646             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7647          }
7648       }
7649
7650       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7651          for (unsigned i = 0; i < 2; i++) {
7652             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7653                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7654                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7655                                       sel);
7656             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7657          }
7658       }
7659    }
7660 }
7661
7662 void select_program(Program *program,
7663                     unsigned shader_count,
7664                     struct nir_shader *const *shaders,
7665                     ac_shader_config* config,
7666                     struct radv_shader_info *info,
7667                     struct radv_nir_compiler_options *options)
7668 {
7669    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7670
7671    for (unsigned i = 0; i < shader_count; i++) {
7672       nir_shader *nir = shaders[i];
7673       init_context(&ctx, nir);
7674
7675       if (!i) {
7676          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7677          append_logical_start(ctx.block);
7678       }
7679
7680       if_context ic;
7681       if (shader_count >= 2) {
7682          Builder bld(ctx.program, ctx.block);
7683          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7684          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7685                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7686          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7687
7688          begin_divergent_if_then(&ctx, &ic, cond);
7689       }
7690
7691       if (i) {
7692          Builder bld(ctx.program, ctx.block);
7693          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7694          bld.sopp(aco_opcode::s_barrier);
7695       }
7696
7697       if (ctx.stage == fragment_fs)
7698          handle_bc_optimize(&ctx);
7699
7700       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7701       visit_cf_list(&ctx, &func->body);
7702
7703       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7704          emit_streamout(&ctx, 0);
7705
7706       if (ctx.stage == vertex_vs)
7707          create_vs_exports(&ctx);
7708
7709       if (shader_count >= 2) {
7710          begin_divergent_if_else(&ctx, &ic);
7711          end_divergent_if(&ctx, &ic);
7712       }
7713
7714       ralloc_free(ctx.divergent_vals);
7715    }
7716
7717    append_logical_end(ctx.block);
7718    ctx.block->kind |= block_kind_uniform;
7719    Builder bld(ctx.program, ctx.block);
7720    if (ctx.program->wb_smem_l1_on_end)
7721       bld.smem(aco_opcode::s_dcache_wb, false);
7722    bld.sopp(aco_opcode::s_endpgm);
7723
7724    /* cleanup CFG */
7725    for (Block& BB : program->blocks) {
7726       for (unsigned idx : BB.linear_preds)
7727          program->blocks[idx].linear_succs.emplace_back(BB.index);
7728       for (unsigned idx : BB.logical_preds)
7729          program->blocks[idx].logical_succs.emplace_back(BB.index);
7730    }
7731 }
7732 }